From 0cb7c0d9441e9a1de2f60fca77811cdb70070ba4 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 6 Oct 2026 17:15:15 +0100 Subject: [PATCH 1/8] feat(io): support refreshable storage credentials for S3 and GCS Add `StorageCredentialProvider`, which storage backends call to obtain and refresh short-lived vended credentials, and wire it into the OpenDAL S3 and GCS backends. - `StorageCredential` holds backend storage properties for locations under a prefix, matched like Java's `StorageCredential`. - `FileIOBuilder::with_credential_provider` attaches a provider, passed to `StorageFactory::build_with_credential_provider`. Factories that cannot use it ignore it. - `FileIO::serialize_all` serializes the provider's `StorageCredentialProviderFactory` and rebuilds it on deserialization, falling back with a warning when it cannot. - OpenDAL S3 and GCS sign with the provider's credentials, reloading them before they expire, and keep bulk deletes within one credential scope. - Add the refresh endpoint and expiry property constants used by catalogs that vend credentials. --- Cargo.lock | 69 +- crates/iceberg/public-api.txt | 37 + crates/iceberg/src/io/file_io.rs | 319 ++++++++- crates/iceberg/src/io/storage/config/azdls.rs | 8 + crates/iceberg/src/io/storage/config/gcs.rs | 6 + crates/iceberg/src/io/storage/config/s3.rs | 6 + crates/iceberg/src/io/storage/mod.rs | 178 ++++- crates/storage/opendal/Cargo.toml | 9 +- crates/storage/opendal/public-api.txt | 4 + crates/storage/opendal/src/gcs.rs | 181 ++++- crates/storage/opendal/src/lib.rs | 663 +++++++++++++++++- crates/storage/opendal/src/resolving.rs | 124 +++- crates/storage/opendal/src/s3.rs | 156 ++++- crates/storage/opendal/src/utils.rs | 294 ++++++++ 14 files changed, 1986 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b2115a713..c52fb6769d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -511,6 +511,12 @@ dependencies = [ "syn", ] +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + [[package]] name = "atoi" version = "2.0.0" @@ -1407,7 +1413,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2007,7 +2013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2662,7 +2668,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -2937,14 +2943,17 @@ dependencies = [ "iceberg", "iceberg-property-macro", "iceberg_test_utils", + "mockito", "opendal", "reqsign-aws-v4", "reqsign-core", + "reqsign-google", "reqwest 0.12.28", "serde", "serde_json", "tempfile", "tokio", + "tracing", "typetag", "url", ] @@ -3940,7 +3949,7 @@ dependencies = [ "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "serde", "serde_json", @@ -4020,7 +4029,7 @@ dependencies = [ "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4064,7 +4073,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -4099,7 +4108,7 @@ dependencies = [ "http 1.5.0", "log", "opendal-core", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -4119,7 +4128,7 @@ dependencies = [ "log", "md-5 0.11.0", "opendal-core", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -4627,6 +4636,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.11" @@ -4640,7 +4659,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -4679,9 +4698,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4925,9 +4944,9 @@ dependencies = [ [[package]] name = "reqsign-aws-core" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" dependencies = [ "bytes", "form_urlencoded", @@ -4935,7 +4954,7 @@ dependencies = [ "http 1.5.0", "log", "percent-encoding", - "quick-xml", + "quick-xml 0.42.0", "reqsign-core", "rust-ini", "serde", @@ -4946,14 +4965,14 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.2.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" dependencies = [ "bytes", "http 1.5.0", "log", - "quick-xml", + "quick-xml 0.42.0", "reqsign-aws-core", "reqsign-core", "serde", @@ -4982,14 +5001,14 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.3.0" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" dependencies = [ "anyhow", + "asyncband", "base64 0.23.0", "bytes", - "futures", "hex", "hmac 0.13.0", "http 1.5.0", @@ -5263,7 +5282,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5321,7 +5340,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6193,10 +6212,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7069,7 +7088,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index bb4d350e5a..ed1641a7a7 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -792,6 +792,7 @@ pub fn iceberg::io::FileIO::new_output(&self, path: impl core::convert::AsRef Self pub fn iceberg::io::FileIO::new_with_memory() -> Self pub fn iceberg::io::FileIO::serialize_all(&self) -> iceberg::Result> +pub fn iceberg::io::FileIO::without_credential_provider(&self) -> Self impl core::clone::Clone for iceberg::io::FileIO pub fn iceberg::io::FileIO::clone(&self) -> iceberg::io::FileIO impl core::fmt::Debug for iceberg::io::FileIO @@ -801,6 +802,7 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_credential_provider(self, provider: alloc::sync::Arc) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder @@ -904,6 +906,7 @@ impl core::fmt::Debug for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::LocalFsStorageFactory @@ -942,6 +945,7 @@ impl core::fmt::Debug for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::MemoryStorageFactory @@ -1054,14 +1058,34 @@ impl serde_core::ser::Serialize for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::StorageCredential +impl iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageCredential::covers(&self, location: &str) -> bool +pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::io::StorageCredential::prefix(&self) -> &str +impl core::clone::Clone for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential +impl core::cmp::Eq for iceberg::io::StorageCredential +impl core::cmp::PartialEq for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::eq(&self, other: &iceberg::io::StorageCredential) -> bool +impl core::fmt::Debug for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential pub const iceberg::io::ADLS_ACCOUNT_KEY: &str pub const iceberg::io::ADLS_ACCOUNT_NAME: &str pub const iceberg::io::ADLS_AUTHORITY_HOST: &str pub const iceberg::io::ADLS_CLIENT_ID: &str pub const iceberg::io::ADLS_CLIENT_SECRET: &str pub const iceberg::io::ADLS_CONNECTION_STRING: &str +pub const iceberg::io::ADLS_REFRESH_CREDENTIALS_ENABLED: &str +pub const iceberg::io::ADLS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::ADLS_SAS_TOKEN: &str +pub const iceberg::io::ADLS_SAS_TOKEN_EXPIRES_AT_MS_PREFIX: &str +pub const iceberg::io::ADLS_SAS_TOKEN_PREFIX: &str pub const iceberg::io::ADLS_TENANT_ID: &str +pub const iceberg::io::AWS_REFRESH_CREDENTIALS_ENABLED: &str +pub const iceberg::io::AWS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::CLIENT_REGION: &str pub const iceberg::io::GCS_ALLOW_ANONYMOUS: &str pub const iceberg::io::GCS_CREDENTIALS_JSON: &str @@ -1069,8 +1093,11 @@ pub const iceberg::io::GCS_DISABLE_CONFIG_LOAD: &str pub const iceberg::io::GCS_DISABLE_VM_METADATA: &str pub const iceberg::io::GCS_NO_AUTH: &str pub const iceberg::io::GCS_PROJECT_ID: &str +pub const iceberg::io::GCS_REFRESH_CREDENTIALS_ENABLED: &str +pub const iceberg::io::GCS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::GCS_SERVICE_HOST: &str pub const iceberg::io::GCS_TOKEN: &str +pub const iceberg::io::GCS_TOKEN_EXPIRES_AT: &str pub const iceberg::io::GCS_USER_PROJECT: &str pub const iceberg::io::HF_ENDPOINT: &str pub const iceberg::io::HF_REVISION: &str @@ -1090,6 +1117,7 @@ pub const iceberg::io::S3_PATH_STYLE_ACCESS: &str pub const iceberg::io::S3_REGION: &str pub const iceberg::io::S3_SECRET_ACCESS_KEY: &str pub const iceberg::io::S3_SESSION_TOKEN: &str +pub const iceberg::io::S3_SESSION_TOKEN_EXPIRES_AT_MS: &str pub const iceberg::io::S3_SSE_KEY: &str pub const iceberg::io::S3_SSE_MD5: &str pub const iceberg::io::S3_SSE_TYPE: &str @@ -1141,12 +1169,21 @@ pub fn iceberg::io::MemoryStorage::read<'life0, 'life1, 'async_trait>(&'life0 se pub fn iceberg::io::MemoryStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg::io::MemoryStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg::io::MemoryStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub trait iceberg::io::StorageCredentialProvider: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg::io::StorageCredentialProvider::factory(&self) -> iceberg::Result> +pub fn iceberg::io::StorageCredentialProvider::load_credential<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::StorageCredentialProvider::supports_path(&self, path: &str) -> bool +pub trait iceberg::io::StorageCredentialProviderFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize +pub fn iceberg::io::StorageCredentialProviderFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> pub trait iceberg::io::StorageFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize pub fn iceberg::io::StorageFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::StorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> pub mod iceberg::memory pub struct iceberg::memory::MemoryCatalog impl core::fmt::Debug for iceberg::memory::MemoryCatalog diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 42eec4db56..e61ed8de57 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -22,7 +22,8 @@ use bytes::Bytes; use futures::{Stream, StreamExt}; use super::storage::{ - LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, + LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageCredentialProvider, + StorageCredentialProviderFactory, StorageFactory, }; use crate::Result; @@ -65,6 +66,8 @@ pub struct FileIO { config: StorageConfig, /// Factory for creating storage instances factory: Arc, + /// Optional provider of refreshable, backend-specific credentials + credential_provider: Option>, /// Cached storage instance (lazily initialized) storage: Arc>>, } @@ -80,15 +83,33 @@ mod _serde { pub(super) struct SerializableFileIO<'a> { pub(super) config: &'a StorageConfig, pub(super) factory: &'a Arc, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) credential_provider: Option, } #[derive(Deserialize)] pub(super) struct DeserializedFileIO { pub(super) config: StorageConfig, pub(super) factory: Arc, + /// Kept as JSON, so that a provider factory the receiving binary does + /// not link can be dropped instead of failing the whole `FileIO`. + #[serde(default)] + pub(super) credential_provider: Option, } } +/// Warns that a `FileIO` crossing a process boundary loses its credential +/// provider. Engines may serialize a `FileIO` per task, so each cause, tracked +/// by its own `warned`, is reported once. +fn warn_once_without_provider(warned: &'static std::sync::Once, reason: impl FnOnce() -> String) { + warned.call_once(|| { + tracing::warn!( + "{}; its credentials will not be refreshed after deserialization", + reason() + ) + }); +} + impl FileIO { /// Create a new FileIO backed by in-memory storage. /// @@ -97,6 +118,7 @@ impl FileIO { Self { config: StorageConfig::new(), factory: Arc::new(MemoryStorageFactory), + credential_provider: None, storage: Arc::new(OnceLock::new()), } } @@ -108,6 +130,7 @@ impl FileIO { Self { config: StorageConfig::new(), factory: Arc::new(LocalFsStorageFactory), + credential_provider: None, storage: Arc::new(OnceLock::new()), } } @@ -123,14 +146,38 @@ impl FileIO { /// /// All storage configuration properties are included in the serialized representation. These /// properties may contain credentials or other sensitive values, so the returned bytes must be - /// protected in transit and at rest by the application embedding this crate. + /// protected in transit and at rest by the application embedding this crate. A serialized + /// credential provider may likewise carry catalog authentication and vended credentials. /// /// Storage factories are serialized through [`typetag`](https://docs.rs/typetag). Third-party /// factories must use `#[typetag::serde]` on their [`StorageFactory`] implementation. + /// + /// A credential provider is serialized as the + /// [`StorageCredentialProviderFactory`] returned by + /// [`StorageCredentialProvider::factory`], and rebuilt on deserialization. A provider that + /// cannot be rebuilt in another process is left out with a warning, as with + /// [`FileIO::without_credential_provider`]: the deserialized `FileIO` uses the credentials in + /// its configuration, which are not refreshed. Storage factories that hold their own + /// credential sources, such as a custom AWS credential loader, still fail to serialize, as + /// leaving them out would leave no credentials at all. pub fn serialize_all(&self) -> Result> { + let credential_provider = self.credential_provider.as_ref().and_then(|provider| { + provider + .factory() + .and_then(|factory| Ok(serde_json::to_value(factory)?)) + .inspect_err(|error| { + static WARNED: std::sync::Once = std::sync::Once::new(); + warn_once_without_provider(&WARNED, || { + format!("serializing FileIO without its credential provider: {error}") + }) + }) + .ok() + }); + Ok(serde_json::to_vec(&_serde::SerializableFileIO { config: &self.config, factory: &self.factory, + credential_provider, })?) } @@ -139,15 +186,73 @@ impl FileIO { /// The receiving binary must use a compatible crate version and link the concrete factory /// implementation so it is registered with `typetag`. Backend-specific requirements are /// documented by each storage factory implementation. + /// + /// A credential provider is rebuilt when the binary links its factory implementation, such as + /// the REST catalog's. Otherwise, or when rebuilding fails, the `FileIO` is deserialized without + /// it and logs a warning, as [`FileIO::serialize_all`] does. pub fn deserialize_all(bytes: &[u8]) -> Result { - let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; + let _serde::DeserializedFileIO { + config, + factory, + credential_provider, + } = serde_json::from_slice(bytes)?; + let credential_provider = credential_provider.and_then(|provider_factory| { + // The factory may hold credentials, and serde errors quote the + // offending value, so only its type is reported. + let factory_type = provider_factory + .get("type") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown") + .to_owned(); + match serde_json::from_value::>( + provider_factory, + ) { + Ok(provider_factory) => provider_factory + .build(&config) + .inspect_err(|error| { + static WARNED: std::sync::Once = std::sync::Once::new(); + warn_once_without_provider(&WARNED, || { + format!( + "deserializing FileIO without its credential provider, which \ + could not be rebuilt: {error}" + ) + }) + }) + .ok(), + Err(_) => { + static WARNED: std::sync::Once = std::sync::Once::new(); + warn_once_without_provider(&WARNED, || { + format!( + "deserializing FileIO without its credential provider, whose \ + factory {factory_type} is not linked or does not match this version" + ) + }); + None + } + } + }); Ok(Self { config, factory, + credential_provider, storage: Arc::new(OnceLock::new()), }) } + /// Returns a copy of this `FileIO` without its credential provider. + /// + /// The copy uses only the credentials in its storage configuration, which are not refreshed. + /// Use this to serialize a `FileIO` without its credential provider, even when the provider + /// could be rebuilt in another process. + pub fn without_credential_provider(&self) -> Self { + Self { + config: self.config.clone(), + factory: Arc::clone(&self.factory), + credential_provider: None, + storage: Arc::new(OnceLock::new()), + } + } + /// Get the storage configuration. pub fn config(&self) -> &StorageConfig { &self.config @@ -163,8 +268,11 @@ impl FileIO { return Ok(storage.clone()); } - // Build the storage - let storage = self.factory.build(&self.config)?; + // Build the storage, passing any credential provider so backends that + // support refreshable credentials can wire it into their operators. + let storage = self + .factory + .build_with_credential_provider(&self.config, self.credential_provider.clone())?; // Try to set it (another thread might have set it first) let _ = self.storage.set(storage.clone()); @@ -247,6 +355,8 @@ pub struct FileIOBuilder { factory: Arc, /// Storage configuration config: StorageConfig, + /// Optional provider of refreshable, backend-specific credentials + credential_provider: Option>, } impl FileIOBuilder { @@ -255,6 +365,7 @@ impl FileIOBuilder { Self { factory, config: StorageConfig::new(), + credential_provider: None, } } @@ -280,11 +391,24 @@ impl FileIOBuilder { &self.config } + /// Attach a provider of refreshable, backend-specific credentials. + /// + /// Storage factories that cannot use the provider ignore it and use the + /// credentials in the configuration. + pub fn with_credential_provider( + mut self, + provider: Arc, + ) -> Self { + self.credential_provider = Some(provider); + self + } + /// Builds [`FileIO`]. pub fn build(self) -> FileIO { FileIO { config: self.config, factory: self.factory, + credential_provider: self.credential_provider, storage: Arc::new(OnceLock::new()), } } @@ -453,7 +577,64 @@ mod tests { use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; - use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; + use crate::Result; + use crate::io::{ + GCS_TOKEN, LocalFsStorageFactory, MemoryStorageFactory, StorageConfig, StorageCredential, + StorageCredentialProvider, StorageCredentialProviderFactory, + }; + + #[derive(Debug)] + struct TestCredentialProvider; + + #[async_trait::async_trait] + impl StorageCredentialProvider for TestCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + unreachable!("unsupported factories must ignore the provider") + } + } + + /// A provider rebuilt from the `FileIO` configuration, like a catalog provider. + #[derive(Debug)] + struct PortableCredentialProvider { + endpoint: Option, + } + + #[async_trait::async_trait] + impl StorageCredentialProvider for PortableCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + Ok(StorageCredential::new( + "gs", + std::collections::HashMap::from([( + GCS_TOKEN.to_string(), + self.endpoint.clone().unwrap_or_default(), + )]), + )) + } + + fn factory(&self) -> Result> { + Ok(Arc::new(PortableCredentialProviderFactory)) + } + } + + #[derive(Debug, serde::Serialize, serde::Deserialize)] + struct PortableCredentialProviderFactory; + + #[typetag::serde] + impl StorageCredentialProviderFactory for PortableCredentialProviderFactory { + fn build(&self, config: &StorageConfig) -> Result> { + Ok(Arc::new(PortableCredentialProvider { + endpoint: config.get("endpoint").cloned(), + })) + } + } fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -601,6 +782,132 @@ mod tests { assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + #[tokio::test] + async fn test_file_io_ignores_credentials_for_unsupported_factory() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + file_io + .new_output("memory://file") + .unwrap() + .write("data".into()) + .await + .unwrap(); + assert!(file_io.exists("memory://file").await.unwrap()); + } + + #[test] + fn test_file_io_drops_unserializable_credential_provider() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("s3.access-key-id", "static-key") + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(deserialized.credential_provider.is_none()); + assert_eq!( + deserialized.config().get("s3.access-key-id"), + Some(&"static-key".to_string()) + ); + assert_eq!( + file_io.serialize_all().unwrap(), + file_io + .without_credential_provider() + .serialize_all() + .unwrap() + ); + } + + /// A provider whose factory cannot be serialized. + #[derive(Debug)] + struct UnserializableFactoryProvider; + + #[async_trait::async_trait] + impl StorageCredentialProvider for UnserializableFactoryProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + unreachable!("serialization tests never load credentials") + } + + fn factory(&self) -> Result> { + Ok(Arc::new(UnserializableFactory)) + } + } + + #[derive(Debug, serde::Deserialize)] + struct UnserializableFactory; + + impl serde::Serialize for UnserializableFactory { + fn serialize( + &self, + _serializer: S, + ) -> std::result::Result { + Err(serde::ser::Error::custom("cannot serialize")) + } + } + + #[typetag::serde] + impl StorageCredentialProviderFactory for UnserializableFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + unreachable!("never serialized") + } + } + + #[test] + fn test_file_io_drops_provider_whose_factory_fails_to_serialize() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_credential_provider(Arc::new(UnserializableFactoryProvider)) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(deserialized.credential_provider.is_none()); + } + + #[test] + fn test_file_io_deserializes_without_unknown_credential_provider() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("s3.access-key-id", "static-key") + .with_credential_provider(Arc::new(PortableCredentialProvider { endpoint: None })) + .build(); + // A receiving binary that does not link the provider's factory. + let mut serialized: serde_json::Value = + serde_json::from_slice(&file_io.serialize_all().unwrap()).unwrap(); + serialized["credential_provider"]["type"] = "UnlinkedProviderFactory".into(); + + let deserialized = + FileIO::deserialize_all(&serde_json::to_vec(&serialized).unwrap()).unwrap(); + assert!(deserialized.credential_provider.is_none()); + assert_eq!( + deserialized.config().get("s3.access-key-id"), + Some(&"static-key".to_string()) + ); + } + + #[tokio::test] + async fn test_file_io_rebuilds_credential_provider_after_serialization() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("endpoint", "https://catalog/credentials") + .with_credential_provider(Arc::new(PortableCredentialProvider { endpoint: None })) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + let credential = deserialized + .credential_provider + .unwrap() + .load_credential("gs://bucket/file") + .await + .unwrap(); + // The provider is rebuilt from the deserialized configuration. + assert_eq!( + credential.config().get(GCS_TOKEN).map(String::as_str), + Some("https://catalog/credentials") + ); + } + #[tokio::test] async fn test_memory_file_io_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) diff --git a/crates/iceberg/src/io/storage/config/azdls.rs b/crates/iceberg/src/io/storage/config/azdls.rs index a9541e7791..7d49a354dc 100644 --- a/crates/iceberg/src/io/storage/config/azdls.rs +++ b/crates/iceberg/src/io/storage/config/azdls.rs @@ -36,6 +36,14 @@ pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name"; pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key"; /// The shared access signature. pub const ADLS_SAS_TOKEN: &str = "adls.sas-token"; +/// Prefix for account-specific shared access signatures vended by a REST catalog. +pub const ADLS_SAS_TOKEN_PREFIX: &str = "adls.sas-token."; +/// Prefix for the epoch-millisecond expiration of an account-specific SAS token. +pub const ADLS_SAS_TOKEN_EXPIRES_AT_MS_PREFIX: &str = "adls.sas-token-expires-at-ms."; +/// Table property naming the endpoint used to refresh vended ADLS credentials. +pub const ADLS_REFRESH_CREDENTIALS_ENDPOINT: &str = "adls.refresh-credentials-endpoint"; +/// Table property controlling whether vended ADLS credentials are refreshed. +pub const ADLS_REFRESH_CREDENTIALS_ENABLED: &str = "adls.refresh-credentials-enabled"; /// The tenant-id. pub const ADLS_TENANT_ID: &str = "adls.tenant-id"; /// The client-id. diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs index 99867e5dfc..6d06ab6a34 100644 --- a/crates/iceberg/src/io/storage/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -41,6 +41,12 @@ pub const GCS_NO_AUTH: &str = "gcs.no-auth"; pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; /// Google Cloud Storage token. pub const GCS_TOKEN: &str = "gcs.oauth2.token"; +/// Epoch-millisecond timestamp at which the vended GCS OAuth2 token expires. +pub const GCS_TOKEN_EXPIRES_AT: &str = "gcs.oauth2.token-expires-at"; +/// Endpoint used to fetch and refresh vended GCS OAuth2 credentials. +pub const GCS_REFRESH_CREDENTIALS_ENDPOINT: &str = "gcs.oauth2.refresh-credentials-endpoint"; +/// Whether vended GCS OAuth2 credentials should be refreshed. Defaults to `true`. +pub const GCS_REFRESH_CREDENTIALS_ENABLED: &str = "gcs.oauth2.refresh-credentials-enabled"; /// Option to skip signing requests (e.g. for public buckets/folders). pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; /// Option to skip loading the credential from GCE metadata server. diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index ae3bc4ba3b..b0dcc115c1 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -36,10 +36,16 @@ pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id"; pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key"; /// S3 session token (required when using temporary credentials). pub const S3_SESSION_TOKEN: &str = "s3.session-token"; +/// Epoch-millisecond timestamp at which the vended S3 session token expires. +pub const S3_SESSION_TOKEN_EXPIRES_AT_MS: &str = "s3.session-token-expires-at-ms"; /// S3 region. pub const S3_REGION: &str = "s3.region"; /// Region to use for the S3 client (takes precedence over [`S3_REGION`]). pub const CLIENT_REGION: &str = "client.region"; +/// Endpoint used to fetch and refresh vended AWS credentials. +pub const AWS_REFRESH_CREDENTIALS_ENDPOINT: &str = "client.refresh-credentials-endpoint"; +/// Whether vended AWS credentials should be refreshed. Defaults to `true`. +pub const AWS_REFRESH_CREDENTIALS_ENABLED: &str = "client.refresh-credentials-enabled"; /// S3 Path Style Access. pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access"; /// S3 Server Side Encryption Type. diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index 5276c7771f..7de00854ac 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -21,6 +21,7 @@ mod config; mod local_fs; mod memory; +use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -32,7 +33,7 @@ pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; pub use memory::{MemoryStorage, MemoryStorageFactory}; use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; -use crate::Result; +use crate::{Error, ErrorKind, Result}; /// Trait for storage operations in Iceberg. /// @@ -139,4 +140,179 @@ pub trait StorageFactory: Debug + Send + Sync { /// A `Result` containing an `Arc` on success, or an error /// if the storage could not be created. fn build(&self, config: &StorageConfig) -> Result>; + + /// Build a new Storage instance, optionally supplying a credential provider + /// that the backend can call to obtain and refresh short-lived credentials. + /// + /// Backends that cannot use the provider ignore it and use the credentials + /// in `config`, as they would without one. The default does exactly that. + #[allow(unused_variables)] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + self.build(config) + } +} + +/// Supplies fresh, backend-specific storage credentials on demand. +/// +/// A catalog that vends temporary credentials implements this trait so that +/// storage backends can re-fetch credentials as they approach expiry instead +/// of failing once the initial token's TTL runs out. +/// +/// # Debug output +/// +/// [`FileIO`](crate::io::FileIO) and storage implementations print the +/// provider in their `Debug` output, so the provider's `Debug` implementation +/// must not expose credentials or other secrets. +/// +/// # Caching +/// +/// [`load_credential`](Self::load_credential) may be called very frequently. +/// Implementations must cache internally and only re-fetch when the current +/// credential is at or near expiry; otherwise every object-store request could +/// trigger a call back to the catalog. +#[async_trait] +pub trait StorageCredentialProvider: Debug + Send + Sync { + /// Return whether this provider supplies credentials for `path`. + /// + /// Backends replace their own credential chain with the provider only for + /// paths it supports, so a provider must return `false` for the storage it + /// does not serve, such as other schemes behind a resolving storage. + fn supports_path(&self, path: &str) -> bool; + + /// Load a fresh credential for the storage location identified by `path`. + /// + /// `path` is an absolute location: usually the file being accessed, such + /// as `s3://bucket/warehouse/db/table/data/file.parquet`, but for a bulk + /// delete the location shared by a batch, which may be a storage root + /// (`s3://bucket/`) or the prefix of a previously returned credential + /// (`s3://bucket/warehouse/db/table`). Providers that vend distinct + /// credentials per location prefix use it to select the most specific + /// match, which must [cover](StorageCredential::covers) `path`. + /// + /// Backends read the credential's expiry from its config, such as + /// `s3.session-token-expires-at-ms`, and load a new one before it expires. + /// A credential without an expiry is used for as long as the backend lives. + async fn load_credential(&self, path: &str) -> Result; + + /// Return a factory that rebuilds an equivalent provider in another process. + /// + /// [`FileIO::serialize_all`](crate::io::FileIO::serialize_all) serializes this + /// factory in place of the provider. On an error, which the default always + /// returns, it serializes the `FileIO` without the provider and logs a + /// warning. + fn factory(&self) -> Result> { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "storage credential provider cannot be serialized", + )) + } +} + +/// Serializable recipe that rebuilds a [`StorageCredentialProvider`] after +/// [`FileIO`](crate::io::FileIO) deserialization. +/// +/// Factories are serialized through [`typetag`](https://docs.rs/typetag), so +/// implementations must use `#[typetag::serde]`, and the receiving binary must +/// link the concrete implementation. +#[typetag::serde(tag = "type")] +pub trait StorageCredentialProviderFactory: Debug + Send + Sync { + /// Build a provider for a `FileIO` with the given storage configuration. + fn build(&self, config: &StorageConfig) -> Result>; +} + +/// A vended storage credential: the storage properties that apply to +/// locations under a prefix, like Java's `StorageCredential` and the REST +/// catalog's `storage-credentials`. +/// +/// `config` holds backend storage properties, such as `s3.access-key-id`, +/// `s3.secret-access-key`, `s3.session-token` and +/// `s3.session-token-expires-at-ms`. Backends read their credential and its +/// expiry from it. +#[derive(Clone, PartialEq, Eq)] +pub struct StorageCredential { + /// Storage-location prefix this credential applies to, such as + /// `s3://bucket/table` or, for a whole scheme, `s3`. + prefix: String, + /// Backend storage properties holding the credential. + config: HashMap, +} + +impl StorageCredential { + /// Create a credential for locations under `prefix`. + /// + /// An empty prefix covers no location, as Java rejects it. + pub fn new(prefix: impl Into, config: HashMap) -> Self { + Self { + prefix: prefix.into(), + config, + } + } + + /// Return the storage-location prefix this credential applies to. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// Return the storage properties holding the credential. + pub fn config(&self) -> &HashMap { + &self.config + } + + /// Return whether this credential applies to `location`. + /// + /// As in Java, `location` must start with the non-empty prefix, compared + /// as plain strings: `s3` covers every `s3://`, `s3a://` and `s3n://` + /// location, and `s3://bucket/tab` covers `s3://bucket/table`. + pub fn covers(&self, location: &str) -> bool { + !self.prefix.is_empty() && location.starts_with(&self.prefix) + } +} + +impl Debug for StorageCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Config values are secrets. + f.debug_struct("StorageCredential") + .field("prefix", &self.prefix) + .field("config_keys", &self.config.keys().collect::>()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scoped(prefix: &str) -> StorageCredential { + StorageCredential::new( + prefix, + HashMap::from([("k".to_string(), "secret".to_string())]), + ) + } + + #[test] + fn credential_prefix_matches_like_java() { + let credential = scoped("s3://bucket/table"); + assert!(credential.covers("s3://bucket/table")); + assert!(credential.covers("s3://bucket/table/data/file.parquet")); + assert!(credential.covers("s3://bucket/table2/file.parquet")); + assert!(!credential.covers("s3://bucket/tab")); + assert!(!credential.covers("s3a://bucket/table/file.parquet")); + + // A scheme prefix covers every location with that scheme, including + // `s3a` and `s3n`. + assert!(scoped("s3").covers("s3a://bucket/file.parquet")); + assert!(!scoped("s3").covers("gs://bucket/file.parquet")); + assert!(!scoped("").covers("s3://bucket/file.parquet")); + } + + #[test] + fn credential_debug_omits_config_values() { + let debug = format!("{:?}", scoped("s3://bucket")); + assert!(debug.contains("s3://bucket"), "{debug}"); + assert!(!debug.contains("secret"), "{debug}"); + } } diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index d3403598e8..b26b6fab31 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -41,7 +41,7 @@ opendal-all = [ opendal-azdls = ["opendal/services-azdls"] opendal-fs = ["opendal/services-fs"] -opendal-gcs = ["opendal/services-gcs"] +opendal-gcs = ["opendal/services-gcs", "reqsign-google", "reqsign-core"] opendal-hf = ["opendal/services-hf"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] @@ -56,15 +56,18 @@ futures = { workspace = true } iceberg = { workspace = true } iceberg-property-macro = { workspace = true } opendal = { workspace = true } -reqsign-aws-v4 = { version = "3.0.0", optional = true } -reqsign-core = { version = "3.0.0", optional = true } +reqsign-aws-v4 = { version = "3.3.0", optional = true } +reqsign-core = { version = "3.3.1", optional = true } +reqsign-google = { version = "3.0.4", optional = true } serde = { workspace = true } +tracing = { workspace = true } typetag = { workspace = true } url = { workspace = true } [dev-dependencies] async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } +mockito = { workspace = true } reqwest = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index c2e310335c..a2f6eb9198 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -8,6 +8,7 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::Gcs::credential_provider: core::option::Option> pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc @@ -22,6 +23,7 @@ pub iceberg_storage_opendal::OpenDalStorage::Oss::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::S3::credential_provider: core::option::Option> pub iceberg_storage_opendal::OpenDalStorage::S3::customized_credential_load: core::option::Option impl core::clone::Clone for iceberg_storage_opendal::OpenDalStorage pub fn iceberg_storage_opendal::OpenDalStorage::clone(&self) -> iceberg_storage_opendal::OpenDalStorage @@ -58,6 +60,7 @@ impl core::fmt::Debug for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +pub fn iceberg_storage_opendal::OpenDalStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::storage::config::StorageConfig, credential_provider: core::option::Option>) -> iceberg::error::Result> impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalStorageFactory @@ -113,6 +116,7 @@ impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorageFactor pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::storage::config::StorageConfig, credential_provider: core::option::Option>) -> iceberg::error::Result> impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalResolvingStorageFactory diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index a00282cc22..76fe5e6404 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -17,17 +17,23 @@ //! Google Cloud Storage properties use std::collections::HashMap; +use std::sync::Arc; use iceberg::io::{ GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD, GCS_DISABLE_VM_METADATA, - GCS_NO_AUTH, GCS_SERVICE_HOST, GCS_TOKEN, + GCS_NO_AUTH, GCS_SERVICE_HOST, GCS_TOKEN, GCS_TOKEN_EXPIRES_AT, StorageCredentialProvider, }; use iceberg::{Error, ErrorKind, Result}; -use opendal::Operator; use opendal::services::GcsConfig; +use opendal::{Configurator, Operator}; +use reqsign_core::{Context, ProvideCredential, Result as ReqsignResult}; +use reqsign_google::{Credential as GoogleCredential, Token as GoogleToken}; use url::Url; -use crate::utils::{from_opendal_error, is_truthy}; +use crate::utils::{ + VendedCredentialSource, credential_expiry, from_opendal_error, is_truthy, + required_credential_property, +}; /// Parse iceberg properties to [`GcsConfig`]. pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result { @@ -45,7 +51,9 @@ pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result) -> Result Result { +pub(crate) fn gcs_config_build( + cfg: &GcsConfig, + credential_provider: &Option>, + path: &str, + credential_location: Option<&str>, +) -> Result { let url = Url::parse(path)?; + if !matches!(url.scheme(), "gs" | "gcs") { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, expected gs:// or gcs://"), + )); + } let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, @@ -82,5 +101,155 @@ pub(crate) fn gcs_config_build(cfg: &GcsConfig, path: &str) -> Result let mut cfg = cfg.clone(); cfg.bucket = bucket.to_string(); - Operator::from_config(cfg).map_err(from_opendal_error) + + let credential_provider = credential_provider + .as_ref() + .filter(|provider| provider.supports_path(path)); + if credential_provider.is_some() { + if cfg.skip_signature { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid GCS auth settings: anonymous access cannot be combined with refreshable credentials", + )); + } + disable_other_credential_sources(&mut cfg); + } + + let mut builder = cfg.into_builder(); + + // A catalog-supplied provider re-fetches the vended OAuth2 token as it nears expiry + if let Some(provider) = credential_provider { + builder = + builder.credential_provider(VendedGcsCredentialProvider(VendedCredentialSource::new( + Arc::clone(provider), + credential_location.unwrap_or(path).to_string(), + ))); + } + + Operator::new(builder).map_err(from_opendal_error) +} + +/// Disable every configured and ambient credential source but a vended one. +/// +/// `reqsign_google` continues to the next provider even when a provider returns +/// an error, and OpenDAL prepends custom providers to its default chain. With +/// the other sources disabled, the vended provider is effectively the sole +/// source, and refresh failures cannot silently fall back. +fn disable_other_credential_sources(cfg: &mut GcsConfig) { + cfg.token = None; + cfg.credential = None; + cfg.credential_path = None; + cfg.service_account = None; + cfg.disable_vm_metadata = true; + cfg.disable_config_load = true; +} + +/// Adapts a generic [`StorageCredentialProvider`] into a `reqsign` +/// [`ProvideCredential`], so the GCS signer can obtain and refresh vended OAuth2 +/// tokens. +#[derive(Debug)] +struct VendedGcsCredentialProvider(VendedCredentialSource); + +impl ProvideCredential for VendedGcsCredentialProvider { + type Credential = GoogleCredential; + + async fn provide_credential(&self, _ctx: &Context) -> ReqsignResult> { + let token = self + .0 + .load("GCS", |config| { + Ok(GoogleToken { + access_token: required_credential_property(config, GCS_TOKEN)?.to_string(), + expires_at: credential_expiry(config, GCS_TOKEN_EXPIRES_AT)?, + }) + }) + .await?; + Ok(Some(GoogleCredential::with_token(token))) + } +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use iceberg::io::{S3_ACCESS_KEY_ID, StorageCredential}; + use reqsign_core::time::Timestamp; + + use super::*; + + #[derive(Debug)] + struct FixedProvider(StorageCredential); + + #[async_trait] + impl StorageCredentialProvider for FixedProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + Ok(self.0.clone()) + } + } + + fn adapter(credential: StorageCredential) -> VendedGcsCredentialProvider { + VendedGcsCredentialProvider(VendedCredentialSource::new( + Arc::new(FixedProvider(credential)), + "gs://bucket/table/file.parquet".to_string(), + )) + } + + fn credential(config: &[(&str, &str)]) -> StorageCredential { + StorageCredential::new( + "gs://bucket/table", + config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ) + } + + #[tokio::test] + async fn vended_adapter_returns_expiring_oauth2_token() { + let credential = credential(&[(GCS_TOKEN, "ya29.token"), (GCS_TOKEN_EXPIRES_AT, "1500")]); + + let google = adapter(credential) + .provide_credential(&Context::new()) + .await + .unwrap() + .unwrap(); + let token = google.token.expect("a token credential"); + assert_eq!(token.access_token, "ya29.token"); + assert_eq!( + token.expires_at, + Some(Timestamp::from_millisecond(1500).unwrap()) + ); + } + + #[tokio::test] + async fn vended_adapter_rejects_other_credentials() { + assert!( + adapter(credential(&[(S3_ACCESS_KEY_ID, "AK")])) + .provide_credential(&Context::new()) + .await + .is_err() + ); + } + + #[test] + fn vended_credentials_disable_every_other_source() { + let mut cfg = gcs_config_parse(HashMap::from([ + (GCS_TOKEN.to_string(), "static-token".to_string()), + (GCS_CREDENTIALS_JSON.to_string(), "e30=".to_string()), + ])) + .unwrap(); + cfg.credential_path = Some("/path/to/credentials.json".to_string()); + cfg.service_account = Some("account@example.com".to_string()); + + disable_other_credential_sources(&mut cfg); + + assert_eq!(cfg.token, None); + assert_eq!(cfg.credential, None); + assert_eq!(cfg.credential_path, None); + assert_eq!(cfg.service_account, None); + assert!(cfg.disable_vm_metadata); + assert!(cfg.disable_config_load); + } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index f58d6f534c..3c521dbd8d 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -37,7 +37,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use iceberg_property_macro::Properties; @@ -210,10 +210,36 @@ where )) } +/// Fails serialization of a storage holding a credential provider, which would +/// otherwise silently fall back to the configured or ambient credentials. +/// [`FileIO`](iceberg::io::FileIO) serializes its provider separately. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn serialize_credential_provider( + _provider: &Option>, + _serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + Err(serde::ser::Error::custom( + "storage with a credential provider cannot be serialized; serialize its FileIO instead", + )) +} + #[typetag::serde(name = "OpenDalStorageFactory")] impl StorageFactory for OpenDalStorageFactory { - #[allow(unused_variables)] fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + #[allow(unused_variables)] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + // Only S3 and GCS consume the provider; other backends use the + // credentials in `config`. let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] @@ -229,11 +255,13 @@ impl StorageFactory for OpenDalStorageFactory { } => Ok(Arc::new(OpenDalStorage::S3 { config: s3_config_parse(config.props().clone())?.into(), customized_credential_load: customized_credential_load.clone(), + credential_provider, client_config, })), #[cfg(feature = "opendal-gcs")] OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), + credential_provider, client_config, })), #[cfg(feature = "opendal-oss")] @@ -278,6 +306,7 @@ fn default_memory_operator() -> Operator { /// /// The serialized representation is not a stable format and may change between crate versions. #[derive(Clone, Debug, Serialize, Deserialize)] +#[non_exhaustive] pub enum OpenDalStorage { /// Memory storage variant. #[cfg(feature = "opendal-memory")] @@ -300,6 +329,7 @@ pub enum OpenDalStorage { /// /// Accepts any S3-family URL (`s3://`, `s3a://`, `s3n://`); the scheme is /// derived from the path at call time. + #[non_exhaustive] #[cfg(feature = "opendal-s3")] S3 { /// S3 configuration. @@ -307,20 +337,36 @@ pub enum OpenDalStorage { /// Custom AWS credential loader. #[serde(skip)] customized_credential_load: Option, + /// Provider of refreshable vended credentials, supplied by the catalog. + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_credential_provider" + )] + credential_provider: Option>, /// Backend-independent client settings. #[serde(default)] client_config: OpenDalClientConfig, }, /// GCS storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-gcs")] Gcs { /// GCS configuration. config: Arc, + /// Provider of refreshable vended credentials, supplied by the catalog. + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_credential_provider" + )] + credential_provider: Option>, /// Backend-independent client settings. #[serde(default)] client_config: OpenDalClientConfig, }, /// OSS storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-oss")] Oss { /// OSS configuration. @@ -335,6 +381,7 @@ pub enum OpenDalStorage { /// `abfs[s]://@.dfs./` or /// `wasb[s]://@.blob./`. /// The scheme is derived from the path at call time. + #[non_exhaustive] #[cfg(feature = "opendal-azdls")] Azdls { /// Azure DLS configuration. @@ -348,6 +395,7 @@ pub enum OpenDalStorage { /// Accepts paths of the form /// `hf:////[@]/`, /// where `` must be one of `models`, `datasets`, `spaces`, or `buckets`. + #[non_exhaustive] #[cfg(feature = "opendal-hf")] Hf { /// HuggingFace Hub configuration (token + endpoint). @@ -358,6 +406,62 @@ pub enum OpenDalStorage { }, } +/// Groups bulk deletes by operator and credential scope. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct DeleteBatchKey { + storage: String, + credential: BatchCredential, +} + +impl DeleteBatchKey { + /// Open batches to flush before adding a path with this key. + /// + /// A scoped credential inside the scope of an open batch means the catalog + /// is replacing that batch's broader credential, such as a scheme-wide or + /// account-wide seed. Such batches are flushed while their credential is + /// still valid: once it expires, nothing may cover their whole scope. + fn superseded_batches<'a>( + &self, + open: impl Iterator, + ) -> Vec { + let BatchCredential::Scoped { prefix } = &self.credential else { + return Vec::new(); + }; + open.filter(|key| { + key.storage == self.storage + && key + .credential + .location() + .is_some_and(|location| location != prefix && prefix.starts_with(location)) + }) + .cloned() + .collect() + } +} + +/// The credential shared by the paths of a bulk-delete batch. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum BatchCredential { + /// Served by the backend's configured credentials. + Static, + /// A dynamic credential for a whole scheme, such as `s3`, looked up for + /// the storage root. + SchemeWide { root: String }, + /// A dynamic credential scoped to `prefix`, looked up for the prefix. + Scoped { prefix: String }, +} + +impl BatchCredential { + /// Location the batch operator looks dynamic credentials up for. + fn location(&self) -> Option<&str> { + match self { + BatchCredential::Static => None, + BatchCredential::SchemeWide { root } => Some(root), + BatchCredential::Scoped { prefix } => Some(prefix), + } + } +} + impl OpenDalStorage { /// Creates operator from path. /// @@ -371,10 +475,23 @@ impl OpenDalStorage { /// /// * An [`opendal::Operator`] instance used to operate on file. /// * Relative path to the root uri of [`opendal::Operator`]. - #[allow(unreachable_code, unused_variables)] pub(crate) fn create_operator<'a>( &self, path: &'a impl AsRef, + ) -> Result<(Operator, &'a str)> { + self.create_operator_with_credential_location(path, None) + } + + /// Creates an operator whose dynamic credentials are looked up for + /// `credential_location` instead of `path`. + /// + /// A bulk-delete batch passes its scope location, so every credential the + /// operator loads covers all paths in the batch. + #[allow(unreachable_code, unreachable_patterns, unused_variables)] + fn create_operator_with_credential_location<'a>( + &self, + path: &'a impl AsRef, + credential_location: Option<&str>, ) -> Result<(Operator, &'a str)> { let path = path.as_ref(); let (operator, relative_path): (Operator, &str) = match self { @@ -399,9 +516,16 @@ impl OpenDalStorage { OpenDalStorage::S3 { config, customized_credential_load, + credential_provider, .. } => { - let op = s3_config_build(config, customized_credential_load, path)?; + let op = s3_config_build( + config, + customized_credential_load, + credential_provider, + path, + credential_location, + )?; let op_info = op.info(); // Use the URL scheme in the path for prefix matching. This enables @@ -423,9 +547,20 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-gcs")] - OpenDalStorage::Gcs { config, .. } => { - let operator = gcs_config_build(config, path)?; - let prefix = format!("gs://{}/", operator.info().name()); + OpenDalStorage::Gcs { + config, + credential_provider, + .. + } => { + let operator = + gcs_config_build(config, credential_provider, path, credential_location)?; + let url = url::Url::parse(path).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}: {e}"), + ) + })?; + let prefix = format!("{}://{}/", url.scheme(), operator.info().name()); if path.starts_with(&prefix) { (operator, &path[prefix.len()..]) } else { @@ -517,6 +652,7 @@ impl OpenDalStorage { /// /// For most backends the URL host (bucket name) is sufficient. For HF the host /// encodes the repo type, not the repo identity, so a more specific key is used. + #[allow(unreachable_patterns)] fn batch_key_for_path(&self, path: &str) -> String { match self { #[cfg(feature = "opendal-hf")] @@ -528,12 +664,69 @@ impl OpenDalStorage { } } + /// Return the dynamic credential provider that serves `path`, if any. + fn credential_provider_for_path( + &self, + path: &str, + ) -> Option<&Arc> { + let provider: &Arc = (match self { + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + customized_credential_load: None, + credential_provider: Some(provider), + .. + } => Some(provider), + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { + credential_provider: Some(provider), + .. + } => Some(provider), + _ => None, + })?; + provider.supports_path(path).then_some(provider) + } + + /// Returns a key that keeps bulk deletes within one operator and credential + /// scope. Loading the credential is normally a cache hit and avoids rebuilding + /// an operator for every path while preventing a batch from crossing prefixes. + async fn delete_batch_key_for_path(&self, path: &str) -> Result { + let credential = match self.credential_provider_for_path(path) { + Some(provider) => { + let credential = provider.load_credential(path).await?; + if !credential.covers(path) { + return Err(Error::new( + ErrorKind::DataInvalid, + utils::uncovered_location_message(path, &credential), + )); + } + let prefix = credential.prefix(); + if prefix.contains("://") { + BatchCredential::Scoped { + prefix: prefix.to_string(), + } + } else { + // A scheme-wide prefix such as `s3` is no location to look + // up, so the batch looks its credential up for the root. + BatchCredential::SchemeWide { + root: utils::storage_root(path)?, + } + } + } + None => BatchCredential::Static, + }; + + Ok(DeleteBatchKey { + storage: self.batch_key_for_path(path), + credential, + }) + } + /// Extracts the relative path from an absolute path without building an operator. /// /// This is a lightweight alternative to [`create_operator`](Self::create_operator) for cases /// where only the relative path is needed (e.g. bulk deletes where the operator is already /// available). - #[allow(unreachable_code, unused_variables)] + #[allow(unreachable_code, unreachable_patterns, unused_variables)] pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> { match self { #[cfg(feature = "opendal-memory")] @@ -564,13 +757,19 @@ impl OpenDalStorage { #[cfg(feature = "opendal-gcs")] OpenDalStorage::Gcs { .. } => { let url = url::Url::parse(path)?; + if !matches!(url.scheme(), "gs" | "gcs") { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, expected gs:// or gcs://"), + )); + } let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, format!("Invalid gcs url: {path}, missing bucket"), ) })?; - let prefix = format!("gs://{}/", bucket); + let prefix = format!("{}://{}/", url.scheme(), bucket); if path.starts_with(&prefix) { Ok(&path[prefix.len()..]) } else { @@ -696,17 +895,26 @@ impl Storage for OpenDalStorage { } async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { - let mut deleters: HashMap = HashMap::new(); + let mut deleters: HashMap = HashMap::new(); while let Some(path) = paths.next().await { - let bucket = self.batch_key_for_path(&path); + let batch_key = self.delete_batch_key_for_path(&path).await?; + + for key in batch_key.superseded_batches(deleters.keys()) { + if let Some(mut deleter) = deleters.remove(&key) { + deleter.close().await.map_err(from_opendal_error)?; + } + } - let (relative_path, deleter) = match deleters.entry(bucket) { + let (relative_path, deleter) = match deleters.entry(batch_key) { Entry::Occupied(entry) => { (self.relativize_path(&path)?.to_string(), entry.into_mut()) } Entry::Vacant(entry) => { - let (op, rel) = self.create_operator(&path)?; + let (op, rel) = self.create_operator_with_credential_location( + &path, + entry.key().credential.location(), + )?; let rel = rel.to_string(); let deleter = op.deleter().await.map_err(from_opendal_error)?; (rel, entry.insert(deleter)) @@ -805,8 +1013,51 @@ impl FileWrite for OpenDalWriter { #[cfg(test)] mod tests { + #[allow(unused_imports)] use super::*; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[derive(Debug)] + struct AlwaysSupportedCredentialProvider; + + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[async_trait] + impl StorageCredentialProvider for AlwaysSupportedCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> Result { + let prefix = if path.contains("/table-a/") { + "s3://bucket/table-a" + } else { + "s3://bucket/table-b" + }; + Ok(s3_credential(prefix, "access-key")) + } + } + + /// An S3 credential for locations under `prefix`. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + fn s3_credential( + prefix: impl Into, + access_key: &str, + ) -> iceberg::io::StorageCredential { + iceberg::io::StorageCredential::new( + prefix, + HashMap::from([ + ( + iceberg::io::S3_ACCESS_KEY_ID.to_string(), + access_key.to_string(), + ), + ( + iceberg::io::S3_SECRET_ACCESS_KEY.to_string(), + "secret-key".to_string(), + ), + ]), + ) + } + fn client_config(value: &str) -> Result { OpenDalClientConfig::from_properties(&HashMap::from([( OPENDAL_IO_TIMEOUT_MS.to_string(), @@ -868,6 +1119,7 @@ mod tests { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, + credential_provider: None, client_config: client_config("45000").unwrap(), }; @@ -1003,6 +1255,22 @@ mod tests { assert_eq!(op.info().scheme().to_string(), "memory"); } + #[cfg(all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ))] + #[test] + fn test_factory_ignores_credentials_for_unsupported_backend() { + let storage = OpenDalStorageFactory::Memory + .build_with_credential_provider( + &StorageConfig::new(), + Some(Arc::new(AlwaysSupportedCredentialProvider)), + ) + .expect("memory must ignore a credential provider"); + + assert!(storage.new_input("memory:/key").is_ok()); + } + #[cfg(feature = "opendal-memory")] #[tokio::test] async fn test_writer_close_returns_stored_size() { @@ -1082,6 +1350,7 @@ mod tests { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, + credential_provider: None, client_config: OpenDalClientConfig::default(), }; @@ -1098,27 +1367,378 @@ mod tests { } } + /// Vends a credential for the whole `s3` scheme until it has served + /// `table-c`, then credentials scoped to each table. The scheme-wide + /// credential expires once the `table-c` batch is signed. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct ChangingScopeProvider { + scoped: std::sync::atomic::AtomicBool, + scheme_wide_expired: std::sync::atomic::AtomicBool, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for ChangingScopeProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> Result { + use std::sync::atomic::Ordering; + + if path.starts_with("s3://bucket/table-c") { + self.scoped.store(true, Ordering::SeqCst); + } + // Signing the scoped batch means the scheme-wide credential is gone. + if path == "s3://bucket/table-c" { + self.scheme_wide_expired.store(true, Ordering::SeqCst); + } + match path.strip_prefix("s3://bucket/") { + Some(rest) if self.scoped.load(Ordering::SeqCst) && !rest.is_empty() => { + let table = rest.split('/').next().unwrap_or_default(); + Ok(s3_credential(format!("s3://bucket/{table}"), "SCOPED_AK")) + } + _ if self.scheme_wide_expired.load(Ordering::SeqCst) => Err(Error::new( + ErrorKind::Unexpected, + "the scheme-wide credential expired", + )), + _ => Ok(s3_credential("s3", "SCHEME_AK")), + } + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_delete_stream_signs_each_batch_with_its_vended_credential() { + let mut server = mockito::Server::new_async().await; + let mut delete = |path: &str, access_key: &str| { + server + .mock("DELETE", path) + .match_header( + "authorization", + mockito::Matcher::Regex(format!("Credential={access_key}/")), + ) + .expect(1) + .with_status(204) + }; + // The scheme-wide batch is flushed, with its own credential, before + // the scoped credential that replaces it is used. + let scheme_wide = delete("/bucket/table-b/f.parquet", "SCHEME_AK") + .create_async() + .await; + let scoped = delete("/bucket/table-c/f.parquet", "SCOPED_AK") + .create_async() + .await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(ChangingScopeProvider::default())), + client_config: OpenDalClientConfig::default(), + }; + storage + .delete_stream( + futures::stream::iter([ + "s3://bucket/table-b/f.parquet".to_string(), + "s3://bucket/table-c/f.parquet".to_string(), + ]) + .boxed(), + ) + .await + .unwrap(); + + scheme_wide.assert_async().await; + scoped.assert_async().await; + } + + /// Vends `token` as a GCS credential, or fails when it is `None`. + #[cfg(feature = "opendal-gcs")] + #[derive(Debug)] + struct GcsTokenProvider(Option<&'static str>); + + #[cfg(feature = "opendal-gcs")] + #[async_trait] + impl StorageCredentialProvider for GcsTokenProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + let token = self + .0 + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "refresh failed"))?; + Ok(iceberg::io::StorageCredential::new( + "gs", + HashMap::from([(iceberg::io::GCS_TOKEN.to_string(), token.to_string())]), + )) + } + } + + #[cfg(feature = "opendal-gcs")] + fn gcs_storage(server: &mockito::Server, provider: GcsTokenProvider) -> OpenDalStorage { + let config = gcs_config_parse(HashMap::from([ + (iceberg::io::GCS_SERVICE_HOST.to_string(), server.url()), + ( + iceberg::io::GCS_TOKEN.to_string(), + "static-token".to_string(), + ), + ])) + .unwrap(); + OpenDalStorage::Gcs { + config: Arc::new(config), + credential_provider: Some(Arc::new(provider)), + client_config: OpenDalClientConfig::default(), + } + } + #[cfg(feature = "opendal-gcs")] + #[tokio::test] + async fn test_gcs_signs_with_the_vended_token() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", mockito::Matcher::Any) + .match_header("authorization", "Bearer vended-token") + .expect_at_least(1) + .with_status(404) + .create_async() + .await; + + let storage = gcs_storage(&server, GcsTokenProvider(Some("vended-token"))); + assert!(!storage.exists("gs://bucket/file.parquet").await.unwrap()); + mock.assert_async().await; + } + + #[cfg(feature = "opendal-gcs")] + #[tokio::test] + async fn test_gcs_does_not_fall_back_when_the_provider_fails() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + + let storage = gcs_storage(&server, GcsTokenProvider(None)); + assert!(storage.exists("gs://bucket/file.parquet").await.is_err()); + mock.assert_async().await; + } + + #[cfg(feature = "opendal-s3")] #[test] - fn test_relativize_path_gcs() { - let storage = OpenDalStorage::Gcs { - config: Arc::new(GcsConfig::default()), + fn test_storage_with_credential_provider_is_not_serializable() { + let storage = + |credential_provider: Option>| OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider, + client_config: OpenDalClientConfig::default(), + }; + + assert!(serde_json::to_string(&storage(None)).is_ok()); + let error = + serde_json::to_string(&storage(Some(Arc::new(AlwaysSupportedCredentialProvider)))) + .unwrap_err(); + assert!(error.to_string().contains("credential provider"), "{error}"); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_dynamic_credentials_batch_by_prefix() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), client_config: OpenDalClientConfig::default(), }; + let first = "s3://bucket/table-a/file.parquet"; + let same_scope = "s3://bucket/table-a/other.parquet"; + let other_scope = "s3://bucket/table-b/file.parquet"; + let first_key = storage.delete_batch_key_for_path(first).await.unwrap(); + assert_eq!(first_key.credential.location(), Some("s3://bucket/table-a")); assert_eq!( + first_key, + storage.delete_batch_key_for_path(same_scope).await.unwrap() + ); + assert_ne!( + first_key, storage - .relativize_path("gs://my-bucket/path/to/file.parquet") - .unwrap(), - "path/to/file.parquet" + .delete_batch_key_for_path(other_scope) + .await + .unwrap() + ); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_scheme_wide_dynamic_credentials_batch_by_storage_root() { + #[derive(Debug)] + struct SchemeWideProvider; + + #[async_trait] + impl StorageCredentialProvider for SchemeWideProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + Ok(s3_credential("s3", "access-key")) + } + } + + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider: Some(Arc::new(SchemeWideProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let key = storage + .delete_batch_key_for_path("s3://bucket/table-a/file.parquet") + .await + .unwrap(); + assert_eq!(key.credential, BatchCredential::SchemeWide { + root: "s3://bucket/".to_string() + }); + assert_eq!( + key, + storage + .delete_batch_key_for_path("s3://bucket/table-b/file.parquet") + .await + .unwrap() + ); + } + + #[test] + fn test_scoped_batch_flushes_broader_batches_of_same_storage() { + let key = |storage: &str, credential: BatchCredential| DeleteBatchKey { + storage: storage.to_string(), + credential, + }; + let unscoped = key("bucket", BatchCredential::SchemeWide { + root: "s3://bucket/".to_string(), + }); + let other_storage = key("other", BatchCredential::SchemeWide { + root: "s3://other/".to_string(), + }); + let scoped = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-a".to_string(), + }); + let open = [unscoped.clone(), other_storage, scoped.clone()]; + + let new_scope = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-b".to_string(), + }); + assert_eq!(new_scope.superseded_batches(open.iter()), vec![ + unscoped.clone() + ]); + + // A narrower scope also supersedes a broader scoped batch. + let nested = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-a/data".to_string(), + }); + let superseded = nested.superseded_batches(open.iter()); + assert_eq!(superseded.len(), 2); + assert!(superseded.contains(&unscoped) && superseded.contains(&scoped)); + // A batch never supersedes itself. + assert_eq!(scoped.superseded_batches(open.iter()), vec![ + unscoped.clone() + ]); + // Scheme-wide and static paths leave open batches alone. + assert!(unscoped.superseded_batches(open.iter()).is_empty()); + assert!( + key("bucket", BatchCredential::Static) + .superseded_batches(open.iter()) + .is_empty() ); } + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_custom_s3_credential_loader_ignores_dynamic_provider_for_batching() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: Some(CustomAwsCredentialLoader::new( + reqsign_aws_v4::StaticCredentialProvider::new("access-key", "secret-key"), + )), + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let key = storage + .delete_batch_key_for_path("s3://bucket/table-a/file.parquet") + .await + .unwrap(); + assert_eq!(key.credential, BatchCredential::Static); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_s3_rejects_anonymous_dynamic_credentials() { + let mut config = S3Config::default(); + config.skip_signature = true; + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let error = storage + .create_operator(&"s3://bucket/file.parquet") + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_gcs_rejects_anonymous_dynamic_credentials() { + let mut config = GcsConfig::default(); + config.skip_signature = true; + let storage = OpenDalStorage::Gcs { + config: Arc::new(config), + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let error = storage + .create_operator(&"gs://bucket/file.parquet") + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_relativize_path_gcs() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + credential_provider: None, + client_config: OpenDalClientConfig::default(), + }; + + for scheme in ["gs", "gcs"] { + let path = format!("{scheme}://my-bucket/path/to/file.parquet"); + assert_eq!( + storage.relativize_path(&path).unwrap(), + "path/to/file.parquet" + ); + let (_, relative) = storage.create_operator(&path).unwrap(); + assert_eq!(relative, "path/to/file.parquet"); + } + } + #[cfg(feature = "opendal-gcs")] #[test] fn test_relativize_path_gcs_invalid_scheme() { let storage = OpenDalStorage::Gcs { config: Arc::new(GcsConfig::default()), + credential_provider: None, client_config: OpenDalClientConfig::default(), }; @@ -1127,6 +1747,11 @@ mod tests { .relativize_path("s3://my-bucket/path/to/file.parquet") .is_err() ); + assert!( + storage + .create_operator(&"s3://my-bucket/path/to/file.parquet") + .is_err() + ); } #[cfg(feature = "opendal-oss")] diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 51e76da472..114e8c405e 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -27,7 +27,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; @@ -81,10 +81,14 @@ fn extract_scheme(path: &str) -> Result<&'static str> { } /// Build an [`OpenDalStorage`] variant for the given scheme and config properties. +#[allow(unused_variables)] fn build_storage_for_scheme( scheme: &'static str, props: &HashMap, #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] credential_provider: &Option< + Arc, + >, ) -> Result { let client_config = OpenDalClientConfig::from_properties(props)?; match scheme { @@ -94,6 +98,7 @@ fn build_storage_for_scheme( Ok(OpenDalStorage::S3 { config: Arc::new(config), customized_credential_load: customized_credential_load.clone(), + credential_provider: credential_provider.clone(), client_config, }) } @@ -102,6 +107,7 @@ fn build_storage_for_scheme( let config = crate::gcs::gcs_config_parse(props.clone())?; Ok(OpenDalStorage::Gcs { config: Arc::new(config), + credential_provider: credential_provider.clone(), client_config, }) } @@ -205,11 +211,24 @@ impl OpenDalResolvingStorageFactory { #[typetag::serde] impl StorageFactory for OpenDalResolvingStorageFactory { fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + #[allow(unused_variables)] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + // Without a compatible backend the provider is ignored, and every + // backend uses the credentials in `config`. Ok(Arc::new(OpenDalResolvingStorage { props: config.props().clone(), storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: self.customized_credential_load.clone(), + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + credential_provider, })) } } @@ -220,7 +239,7 @@ impl StorageFactory for OpenDalResolvingStorageFactory { /// Sub-storages are lazily created on first use for each scheme and cached /// for subsequent operations. Scheme aliases like `s3`/`s3a`/`s3n` map to /// the same canonical scheme, so they share a storage instance. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Serialize, Deserialize)] pub struct OpenDalResolvingStorage { /// Configuration properties shared across all backends. props: HashMap, @@ -231,6 +250,23 @@ pub struct OpenDalResolvingStorage { #[cfg(feature = "opendal-s3")] #[serde(skip)] customized_credential_load: Option, + /// Provider of refreshable vended credentials. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "crate::serialize_credential_provider" + )] + credential_provider: Option>, +} + +impl std::fmt::Debug for OpenDalResolvingStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // `props` can contain storage secrets + f.debug_struct("OpenDalResolvingStorage") + .field("property_keys", &self.props.keys().collect::>()) + .finish_non_exhaustive() + } } impl OpenDalResolvingStorage { @@ -266,6 +302,8 @@ impl OpenDalResolvingStorage { &self.props, #[cfg(feature = "opendal-s3")] &self.customized_credential_load, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + &self.credential_provider, )?; let storage = Arc::new(storage); cache.insert(scheme, storage.clone()); @@ -326,6 +364,19 @@ impl Storage for OpenDalResolvingStorage { Ok(()) } + #[cfg_attr( + not(any( + feature = "opendal-memory", + feature = "opendal-fs", + feature = "opendal-s3", + feature = "opendal-gcs", + feature = "opendal-oss", + feature = "opendal-azdls", + feature = "opendal-hf" + )), + // Without a backend, there is no storage to resolve. + allow(unreachable_code) + )] fn new_input(&self, path: &str) -> Result { Ok(InputFile::new( Arc::new(self.resolve(path)?.as_ref().clone()), @@ -333,6 +384,19 @@ impl Storage for OpenDalResolvingStorage { )) } + #[cfg_attr( + not(any( + feature = "opendal-memory", + feature = "opendal-fs", + feature = "opendal-s3", + feature = "opendal-gcs", + feature = "opendal-oss", + feature = "opendal-azdls", + feature = "opendal-hf" + )), + // Without a backend, there is no storage to resolve. + allow(unreachable_code) + )] fn new_output(&self, path: &str) -> Result { Ok(OutputFile::new( Arc::new(self.resolve(path)?.as_ref().clone()), @@ -345,9 +409,51 @@ impl Storage for OpenDalResolvingStorage { mod tests { use std::time::Duration; + #[allow(unused_imports)] use super::*; use crate::OPENDAL_IO_TIMEOUT_MS; + #[cfg(any( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ) + ))] + #[derive(Debug)] + struct AllPathsCredentialProvider; + + #[cfg(any( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ) + ))] + #[async_trait] + impl StorageCredentialProvider for AllPathsCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + unreachable!("unsupported backends must ignore the provider") + } + } + + #[cfg(not(any(feature = "opendal-s3", feature = "opendal-gcs")))] + #[test] + fn test_factory_ignores_credentials_without_compatible_backend() { + assert!( + OpenDalResolvingStorageFactory::new() + .build_with_credential_provider( + &StorageConfig::new(), + Some(Arc::new(AllPathsCredentialProvider)), + ) + .is_ok() + ); + } + #[cfg(feature = "opendal-s3")] #[derive(Debug)] struct EmptyCredentialLoader; @@ -386,6 +492,8 @@ mod tests { storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: None, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + credential_provider: None, } } @@ -438,6 +546,18 @@ mod tests { assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance"); } + #[cfg(all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ))] + #[test] + fn test_resolver_ignores_credentials_for_unsupported_backend() { + let mut storage = empty_resolving_storage(); + storage.credential_provider = Some(Arc::new(AllPathsCredentialProvider)); + + assert!(storage.resolve("memory:/key").is_ok()); + } + #[cfg(feature = "opendal-azdls")] #[test] fn test_resolve_azdls_aliases_share_instance() { diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 2997fa81d9..2fd7e1c4a1 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -22,7 +22,8 @@ use iceberg::io::{ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, - S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, + S3_SESSION_TOKEN, S3_SESSION_TOKEN_EXPIRES_AT_MS, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, + StorageCredentialProvider, }; use iceberg::{Error, ErrorKind, Result}; use opendal::services::S3Config; @@ -31,10 +32,15 @@ use opendal::{Configurator, Operator}; pub use reqsign_aws_v4::Credential as AwsCredential; /// Trait for types that can asynchronously supply [`AwsCredential`] to a [`CustomAwsCredentialLoader`]. pub use reqsign_core::ProvideCredential; -use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn}; +use reqsign_core::{ + Context, ProvideCredentialChain, ProvideCredentialDyn, Result as ReqsignResult, +}; use url::Url; -use crate::utils::{from_opendal_error, is_truthy}; +use crate::utils::{ + VendedCredentialSource, credential_expiry, from_opendal_error, is_truthy, + required_credential_property, +}; /// Parse iceberg props to s3 config. pub(crate) fn s3_config_parse(mut m: HashMap) -> Result { @@ -129,7 +135,9 @@ pub(crate) fn s3_config_parse(mut m: HashMap) -> Result, + credential_provider: &Option>, path: &str, + credential_location: Option<&str>, ) -> Result { let url = Url::parse(path)?; let bucket = url.host_str().ok_or_else(|| { @@ -139,6 +147,19 @@ pub(crate) fn s3_config_build( ) })?; + // Preserve the existing custom-loader precedence: an explicitly configured loader + // is the sole source, otherwise install the catalog provider as a replacement chain so + // refresh failures cannot fall through to broader ambient AWS credentials. + let credential_provider = credential_provider + .as_ref() + .filter(|provider| provider.supports_path(path)); + if customized_credential_load.is_none() && credential_provider.is_some() && cfg.skip_signature { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid S3 auth settings: anonymous access cannot be combined with refreshable credentials", + )); + } + let mut builder = cfg .clone() .into_builder() @@ -148,11 +169,49 @@ pub(crate) fn s3_config_build( if let Some(loader) = customized_credential_load { let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0)); builder = builder.credential_provider_chain(chain); + } else if let Some(provider) = credential_provider { + let chain = ProvideCredentialChain::new().push(VendedS3CredentialProvider( + VendedCredentialSource::new( + Arc::clone(provider), + credential_location.unwrap_or(path).to_string(), + ), + )); + builder = builder.credential_provider_chain(chain); } Operator::new(builder).map_err(from_opendal_error) } +/// Adapts a generic [`StorageCredentialProvider`] into a reqsign +/// [`ProvideCredential`], so the S3 signer can obtain and refresh vended +/// credentials. +#[derive(Debug)] +struct VendedS3CredentialProvider(VendedCredentialSource); + +impl ProvideCredential for VendedS3CredentialProvider { + type Credential = AwsCredential; + + async fn provide_credential(&self, _ctx: &Context) -> ReqsignResult> { + let credential = self + .0 + .load("S3", |config| { + Ok(AwsCredential { + access_key_id: required_credential_property(config, S3_ACCESS_KEY_ID)? + .to_string(), + secret_access_key: required_credential_property(config, S3_SECRET_ACCESS_KEY)? + .to_string(), + session_token: config + .get(S3_SESSION_TOKEN) + .filter(|token| !token.is_empty()) + .cloned(), + expires_in: credential_expiry(config, S3_SESSION_TOKEN_EXPIRES_AT_MS)?, + }) + }) + .await?; + Ok(Some(credential)) + } +} + /// Custom AWS credential loader. /// /// Wraps any [`ProvideCredential`] implementation for use with the S3 storage backend. @@ -176,17 +235,102 @@ impl std::fmt::Debug for CustomAwsCredentialLoader { impl CustomAwsCredentialLoader { /// Create a new custom AWS credential loader from any [`ProvideCredential`] implementation. pub fn new(provider: impl ProvideCredential + 'static) -> Self { - Self(Arc::new(provider) as Arc>) + Self(Arc::new(provider)) } } #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; + + use async_trait::async_trait; + use iceberg::io::{GCS_TOKEN, S3_PATH_STYLE_ACCESS, StorageCredential}; + use reqsign_core::time::Timestamp; + + use super::*; + + #[derive(Debug)] + struct FixedProvider(StorageCredential); - use iceberg::io::S3_PATH_STYLE_ACCESS; + #[async_trait] + impl StorageCredentialProvider for FixedProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + Ok(self.0.clone()) + } + } + + fn adapter(credential: StorageCredential) -> VendedS3CredentialProvider { + VendedS3CredentialProvider(VendedCredentialSource::new( + Arc::new(FixedProvider(credential)), + "s3://bucket/table/file.parquet".to_string(), + )) + } + + fn credential(prefix: &str, config: &[(&str, &str)]) -> StorageCredential { + StorageCredential::new( + prefix, + config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ) + } - use super::s3_config_parse; + #[tokio::test] + async fn vended_adapter_returns_expiring_aws_credential() { + let credential = credential("s3://bucket/table", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + (S3_SESSION_TOKEN, "TOK"), + (S3_SESSION_TOKEN_EXPIRES_AT_MS, "1500"), + ]); + + let aws = adapter(credential) + .provide_credential(&Context::new()) + .await + .unwrap() + .unwrap(); + assert_eq!(aws.access_key_id, "AK"); + assert_eq!(aws.secret_access_key, "SK"); + assert_eq!(aws.session_token.as_deref(), Some("TOK")); + assert_eq!( + aws.expires_in, + Some(Timestamp::from_millisecond(1500).unwrap()) + ); + } + + #[tokio::test] + async fn vended_adapter_rejects_incomplete_or_uncovering_credentials() { + for credential in [ + // A credential for another backend. + credential("s3", &[(GCS_TOKEN, "token")]), + // No secret key. + credential("s3", &[(S3_ACCESS_KEY_ID, "AK")]), + // An unparsable expiry. + credential("s3", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + (S3_SESSION_TOKEN_EXPIRES_AT_MS, "soon"), + ]), + // A prefix that does not cover the path. + credential("s3://bucket/other", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + ]), + ] { + assert!( + adapter(credential) + .provide_credential(&Context::new()) + .await + .is_err() + ); + } + } fn parse_with(prop: Option<&str>) -> bool { let mut props = HashMap::new(); diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index 56f8c18059..d94dad6b43 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn is_truthy(value: &str) -> bool { ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()) } @@ -27,3 +28,296 @@ pub(crate) fn from_opendal_error(e: opendal::Error) -> iceberg::Error { ) .with_source(e) } + +/// The non-empty value of `key` in a vended credential's config. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn required_credential_property<'a>( + config: &'a std::collections::HashMap, + key: &str, +) -> reqsign_core::Result<&'a str> { + config + .get(key) + .map(String::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + reqsign_core::Error::unexpected(format!("vended credential is missing {key}")) + }) +} + +/// The latest epoch millisecond `reqsign` timestamps can represent, +/// `9999-12-30T22:00:00Z`. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +const MAX_TIMESTAMP_MILLIS: i64 = 253_402_207_200_000; + +/// The epoch-millisecond expiry under `key` in a vended credential's config, +/// if present, as the `reqsign` timestamp backend credentials use. Later +/// expiries, such as `Long.MAX_VALUE` for a credential that never expires, +/// are clamped to the latest representable timestamp. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn credential_expiry( + config: &std::collections::HashMap, + key: &str, +) -> reqsign_core::Result> { + config + .get(key) + .filter(|value| !value.is_empty()) + .map(|value| { + value + .parse::() + .ok() + .and_then(|millis| { + reqsign_core::time::Timestamp::from_millisecond( + millis.min(MAX_TIMESTAMP_MILLIS), + ) + .ok() + }) + .ok_or_else(|| { + reqsign_core::Error::unexpected(format!( + "vended credential has an invalid {key}" + )) + }) + }) + .transpose() +} + +/// Validate that a provider's credential covers the location for which the +/// backend requested it. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn validate_credential_prefix( + location: &str, + credential: &iceberg::io::StorageCredential, +) -> reqsign_core::Result<()> { + if credential.covers(location) { + Ok(()) + } else { + Err(reqsign_core::Error::unexpected(uncovered_location_message( + location, credential, + ))) + } +} + +/// The error message for a vended credential that does not cover `location`. +pub(crate) fn uncovered_location_message( + location: &str, + credential: &iceberg::io::StorageCredential, +) -> String { + format!( + "vended credential prefix {:?} does not cover storage location {location:?}", + credential.prefix() + ) +} + +/// The root of the storage location containing `path`, e.g. `s3://bucket/`. +pub(crate) fn storage_root(path: &str) -> iceberg::Result { + let url = url::Url::parse(path)?; + Ok(format!("{}://{}/", url.scheme(), url.authority())) +} + +/// Loads vended credentials for one storage location on behalf of a backend's +/// `reqsign` credential provider. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) struct VendedCredentialSource { + provider: std::sync::Arc, + /// Location handed to the provider: the path an operator serves, or the + /// scope shared by a bulk-delete batch. + location: String, +} + +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +impl VendedCredentialSource { + pub(crate) fn new( + provider: std::sync::Arc, + location: String, + ) -> Self { + Self { provider, location } + } + + /// Load a credential covering the location and convert its config with + /// `extract` into the backend's credential. + /// + /// Errors are logged here: reqsign's credential chain replaces them with an + /// error that only says no credential could be loaded, and reports the + /// cause only through the `log` crate. + pub(crate) async fn load( + &self, + backend: &str, + extract: impl FnOnce(&std::collections::HashMap) -> reqsign_core::Result, + ) -> reqsign_core::Result { + self.try_load(backend, extract).await.inspect_err(|error| { + tracing::warn!( + "cannot use vended {backend} credentials for {}: {error}", + self.location + ) + }) + } + + async fn try_load( + &self, + backend: &str, + extract: impl FnOnce(&std::collections::HashMap) -> reqsign_core::Result, + ) -> reqsign_core::Result { + let credential = self + .provider + .load_credential(&self.location) + .await + .map_err(|e| { + reqsign_core::Error::unexpected(format!( + "failed to load vended {backend} credential for {}: {e}", + self.location + )) + .with_source(e) + })?; + validate_credential_prefix(&self.location, &credential)?; + extract(credential.config()) + } +} + +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +impl std::fmt::Debug for VendedCredentialSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VendedCredentialSource") + .field("location", &self.location) + .finish_non_exhaustive() + } +} + +#[cfg(all(test, any(feature = "opendal-s3", feature = "opendal-gcs")))] +mod tests { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use iceberg::io::{ + GCS_TOKEN, GCS_TOKEN_EXPIRES_AT, StorageCredential, StorageCredentialProvider, + }; + + use super::*; + + /// Returns a credential scoped to `prefix` and records requested locations. + #[derive(Debug)] + struct RecordingProvider { + prefix: &'static str, + requested: Mutex>, + } + + #[async_trait] + impl StorageCredentialProvider for RecordingProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> iceberg::Result { + self.requested.lock().unwrap().push(path.to_string()); + Ok(StorageCredential::new( + self.prefix, + HashMap::from([(GCS_TOKEN.to_string(), "token".to_string())]), + )) + } + } + + async fn load(prefix: &'static str, location: &str) -> reqsign_core::Result<()> { + let provider = Arc::new(RecordingProvider { + prefix, + requested: Mutex::new(Vec::new()), + }); + let source = VendedCredentialSource::new(provider.clone(), location.to_string()); + let result = source + .load("GCS", |config| { + required_credential_property(config, GCS_TOKEN).map(|_| ()) + }) + .await; + assert_eq!(*provider.requested.lock().unwrap(), vec![location]); + result + } + + #[tokio::test] + async fn vended_source_requires_a_covering_credential() { + let location = "gs://bucket/table/data/file.parquet"; + assert!(load("gs", location).await.is_ok()); + assert!(load("gs://bucket/table", location).await.is_ok()); + assert!(load("gs://bucket/tab", location).await.is_ok()); + assert!( + load("gs://bucket/table/data/other", location) + .await + .is_err() + ); + assert!(load("gcs://bucket", location).await.is_err()); + assert!(load("", location).await.is_err()); + } + + #[test] + fn credential_properties_are_parsed() { + let config = HashMap::from([ + (GCS_TOKEN.to_string(), "token".to_string()), + (GCS_TOKEN_EXPIRES_AT.to_string(), "1500".to_string()), + ]); + assert_eq!( + required_credential_property(&config, GCS_TOKEN).unwrap(), + "token" + ); + assert!(required_credential_property(&config, "missing").is_err()); + assert_eq!( + credential_expiry(&config, GCS_TOKEN_EXPIRES_AT).unwrap(), + Some(reqsign_core::time::Timestamp::from_millisecond(1500).unwrap()) + ); + assert_eq!(credential_expiry(&config, "missing").unwrap(), None); + // An expiry beyond year 9999, such as Java's `Long.MAX_VALUE`, is + // clamped rather than rejected. + let never = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), i64::MAX.to_string())]); + assert_eq!( + credential_expiry(&never, GCS_TOKEN_EXPIRES_AT).unwrap(), + Some(reqsign_core::time::Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS).unwrap()) + ); + assert!(reqsign_core::time::Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS + 1).is_err()); + // The value is never quoted in the error. + let invalid = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), "secret".to_string())]); + let error = credential_expiry(&invalid, GCS_TOKEN_EXPIRES_AT) + .unwrap_err() + .to_string(); + assert!(!error.contains("secret"), "{error}"); + } + + #[test] + fn storage_root_keeps_scheme_and_authority() { + assert_eq!( + storage_root("s3a://bucket/table/file.parquet").unwrap(), + "s3a://bucket/" + ); + assert_eq!( + storage_root("abfss://fs@account.dfs.core.windows.net/table/file.parquet").unwrap(), + "abfss://fs@account.dfs.core.windows.net/" + ); + assert!(storage_root("not a url").is_err()); + } + + #[derive(Debug)] + struct FailingProvider; + + #[async_trait] + impl StorageCredentialProvider for FailingProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> iceberg::Result { + Err(iceberg::Error::new( + iceberg::ErrorKind::Unexpected, + "catalog returned 503", + )) + } + } + + #[tokio::test] + async fn vended_source_errors_carry_the_cause() { + let source = VendedCredentialSource::new( + Arc::new(FailingProvider), + "gs://bucket/file.parquet".to_string(), + ); + let error = source + .load("GCS", |_| Ok(())) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("catalog returned 503"), "{error}"); + } +} From f631771198df4b98b251c391b43aadde060579cb Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 6 Oct 2026 17:29:32 +0100 Subject: [PATCH 2/8] chore: regenerate the OpenDAL public API --- crates/storage/opendal/public-api.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index a2f6eb9198..462a6897e3 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -1,15 +1,15 @@ pub mod iceberg_storage_opendal pub use iceberg_storage_opendal::AwsCredential pub use iceberg_storage_opendal::ProvideCredential -pub enum iceberg_storage_opendal::OpenDalStorage -pub iceberg_storage_opendal::OpenDalStorage::Azdls +#[non_exhaustive] pub enum iceberg_storage_opendal::OpenDalStorage +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Azdls pub iceberg_storage_opendal::OpenDalStorage::Azdls::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::Gcs +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Gcs::credential_provider: core::option::Option> -pub iceberg_storage_opendal::OpenDalStorage::Hf +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::LocalFs @@ -17,10 +17,10 @@ pub iceberg_storage_opendal::OpenDalStorage::LocalFs::client_config: iceberg_sto pub iceberg_storage_opendal::OpenDalStorage::Memory pub iceberg_storage_opendal::OpenDalStorage::Memory::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Memory::operator: opendal_core::types::operator::operator::Operator -pub iceberg_storage_opendal::OpenDalStorage::Oss +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Oss pub iceberg_storage_opendal::OpenDalStorage::Oss::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Oss::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::S3 +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::S3 pub iceberg_storage_opendal::OpenDalStorage::S3::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::S3::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::S3::credential_provider: core::option::Option> From e5e50f39960c320072b1dd852748d668ac766b11 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 6 Oct 2026 22:38:44 +0100 Subject: [PATCH 3/8] refactor(io): address review feedback on refreshable credentials - Leave the `gcs.no-auth`, `gcs://` path and resolving-storage `Debug` fixes for a separate PR, keeping `main`'s behavior here. - Note the `OpenDalStorage` break in the changelog. - Import names instead of spelling out paths. - Regenerate `Cargo.lock` from `main`, dropping unrelated downgrades. --- CHANGELOG.md | 1 + Cargo.lock | 16 ++--- crates/iceberg/src/io/file_io.rs | 27 ++++---- crates/storage/opendal/src/gcs.rs | 10 +-- crates/storage/opendal/src/lib.rs | 86 +++++++++---------------- crates/storage/opendal/src/resolving.rs | 22 +++---- crates/storage/opendal/src/utils.rs | 78 +++++++++++----------- 7 files changed, 104 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f357a03e57..870ef2e2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * `EncryptedInputFile::metadata()` is now synchronous and derives the plaintext size from key metadata without a storage stat. Remove `.await` from calls to this method. * AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract. * `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. +* `iceberg_storage_opendal::OpenDalStorage` and its variants are now `#[non_exhaustive]`, and the `S3` and `Gcs` variants have a new `credential_provider` field. Code outside the crate can no longer build a variant with a struct literal or match the enum exhaustively: build storages with `OpenDalStorageFactory` instead, and add `..` to variant patterns and a wildcard arm to `match` expressions. ## [v0.10.1] - 2026-07-28 diff --git a/Cargo.lock b/Cargo.lock index c52fb6769d..e0bd589256 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1413,7 +1413,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2013,7 +2013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4700,7 +4700,7 @@ dependencies = [ "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5282,7 +5282,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5340,7 +5340,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6212,10 +6212,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7088,7 +7088,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index e61ed8de57..bf996f3012 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -16,7 +16,7 @@ // under the License. use std::ops::Range; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Once, OnceLock}; use bytes::Bytes; use futures::{Stream, StreamExt}; @@ -101,7 +101,7 @@ mod _serde { /// Warns that a `FileIO` crossing a process boundary loses its credential /// provider. Engines may serialize a `FileIO` per task, so each cause, tracked /// by its own `warned`, is reported once. -fn warn_once_without_provider(warned: &'static std::sync::Once, reason: impl FnOnce() -> String) { +fn warn_once_without_provider(warned: &'static Once, reason: impl FnOnce() -> String) { warned.call_once(|| { tracing::warn!( "{}; its credentials will not be refreshed after deserialization", @@ -166,7 +166,7 @@ impl FileIO { .factory() .and_then(|factory| Ok(serde_json::to_value(factory)?)) .inspect_err(|error| { - static WARNED: std::sync::Once = std::sync::Once::new(); + static WARNED: Once = Once::new(); warn_once_without_provider(&WARNED, || { format!("serializing FileIO without its credential provider: {error}") }) @@ -210,7 +210,7 @@ impl FileIO { Ok(provider_factory) => provider_factory .build(&config) .inspect_err(|error| { - static WARNED: std::sync::Once = std::sync::Once::new(); + static WARNED: Once = Once::new(); warn_once_without_provider(&WARNED, || { format!( "deserializing FileIO without its credential provider, which \ @@ -220,7 +220,7 @@ impl FileIO { }) .ok(), Err(_) => { - static WARNED: std::sync::Once = std::sync::Once::new(); + static WARNED: Once = Once::new(); warn_once_without_provider(&WARNED, || { format!( "deserializing FileIO without its credential provider, whose \ @@ -566,14 +566,17 @@ impl OutputFile { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::fs::{File, create_dir_all}; use std::io::Write; use std::path::Path; use std::sync::Arc; + use async_trait::async_trait; use bytes::Bytes; use futures::AsyncReadExt; use futures::io::AllowStdIo; + use serde::{Deserialize, Serialize}; use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; @@ -586,7 +589,7 @@ mod tests { #[derive(Debug)] struct TestCredentialProvider; - #[async_trait::async_trait] + #[async_trait] impl StorageCredentialProvider for TestCredentialProvider { fn supports_path(&self, _path: &str) -> bool { true @@ -603,7 +606,7 @@ mod tests { endpoint: Option, } - #[async_trait::async_trait] + #[async_trait] impl StorageCredentialProvider for PortableCredentialProvider { fn supports_path(&self, _path: &str) -> bool { true @@ -612,7 +615,7 @@ mod tests { async fn load_credential(&self, _path: &str) -> Result { Ok(StorageCredential::new( "gs", - std::collections::HashMap::from([( + HashMap::from([( GCS_TOKEN.to_string(), self.endpoint.clone().unwrap_or_default(), )]), @@ -624,7 +627,7 @@ mod tests { } } - #[derive(Debug, serde::Serialize, serde::Deserialize)] + #[derive(Debug, Serialize, Deserialize)] struct PortableCredentialProviderFactory; #[typetag::serde] @@ -823,7 +826,7 @@ mod tests { #[derive(Debug)] struct UnserializableFactoryProvider; - #[async_trait::async_trait] + #[async_trait] impl StorageCredentialProvider for UnserializableFactoryProvider { fn supports_path(&self, _path: &str) -> bool { true @@ -838,10 +841,10 @@ mod tests { } } - #[derive(Debug, serde::Deserialize)] + #[derive(Debug, Deserialize)] struct UnserializableFactory; - impl serde::Serialize for UnserializableFactory { + impl Serialize for UnserializableFactory { fn serialize( &self, _serializer: S, diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index 76fe5e6404..8a8795e3e0 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -51,9 +51,7 @@ pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result, ) -> Result { let url = Url::parse(path)?; - if !matches!(url.scheme(), "gs" | "gcs") { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid gcs url: {path}, expected gs:// or gcs://"), - )); - } let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 3c521dbd8d..c63c314c75 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -554,13 +554,7 @@ impl OpenDalStorage { } => { let operator = gcs_config_build(config, credential_provider, path, credential_location)?; - let url = url::Url::parse(path).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid gcs url: {path}: {e}"), - ) - })?; - let prefix = format!("{}://{}/", url.scheme(), operator.info().name()); + let prefix = format!("gs://{}/", operator.info().name()); if path.starts_with(&prefix) { (operator, &path[prefix.len()..]) } else { @@ -757,19 +751,13 @@ impl OpenDalStorage { #[cfg(feature = "opendal-gcs")] OpenDalStorage::Gcs { .. } => { let url = url::Url::parse(path)?; - if !matches!(url.scheme(), "gs" | "gcs") { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid gcs url: {path}, expected gs:// or gcs://"), - )); - } let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, format!("Invalid gcs url: {path}, missing bucket"), ) })?; - let prefix = format!("{}://{}/", url.scheme(), bucket); + let prefix = format!("gs://{}/", bucket); if path.starts_with(&prefix) { Ok(&path[prefix.len()..]) } else { @@ -1013,6 +1001,14 @@ impl FileWrite for OpenDalWriter { #[cfg(test)] mod tests { + #[cfg(feature = "opendal-s3")] + use std::sync::atomic::{AtomicBool, Ordering}; + + #[cfg(feature = "opendal-gcs")] + use iceberg::io::{GCS_SERVICE_HOST, GCS_TOKEN}; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use iceberg::io::{S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, StorageCredential}; + #[allow(unused_imports)] use super::*; @@ -1027,7 +1023,7 @@ mod tests { true } - async fn load_credential(&self, path: &str) -> Result { + async fn load_credential(&self, path: &str) -> Result { let prefix = if path.contains("/table-a/") { "s3://bucket/table-a" } else { @@ -1039,21 +1035,12 @@ mod tests { /// An S3 credential for locations under `prefix`. #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] - fn s3_credential( - prefix: impl Into, - access_key: &str, - ) -> iceberg::io::StorageCredential { - iceberg::io::StorageCredential::new( + fn s3_credential(prefix: impl Into, access_key: &str) -> StorageCredential { + StorageCredential::new( prefix, HashMap::from([ - ( - iceberg::io::S3_ACCESS_KEY_ID.to_string(), - access_key.to_string(), - ), - ( - iceberg::io::S3_SECRET_ACCESS_KEY.to_string(), - "secret-key".to_string(), - ), + (S3_ACCESS_KEY_ID.to_string(), access_key.to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "secret-key".to_string()), ]), ) } @@ -1373,8 +1360,8 @@ mod tests { #[cfg(feature = "opendal-s3")] #[derive(Debug, Default)] struct ChangingScopeProvider { - scoped: std::sync::atomic::AtomicBool, - scheme_wide_expired: std::sync::atomic::AtomicBool, + scoped: AtomicBool, + scheme_wide_expired: AtomicBool, } #[cfg(feature = "opendal-s3")] @@ -1384,9 +1371,7 @@ mod tests { true } - async fn load_credential(&self, path: &str) -> Result { - use std::sync::atomic::Ordering; - + async fn load_credential(&self, path: &str) -> Result { if path.starts_with("s3://bucket/table-c") { self.scoped.store(true, Ordering::SeqCst); } @@ -1469,13 +1454,13 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result { let token = self .0 .ok_or_else(|| Error::new(ErrorKind::Unexpected, "refresh failed"))?; - Ok(iceberg::io::StorageCredential::new( + Ok(StorageCredential::new( "gs", - HashMap::from([(iceberg::io::GCS_TOKEN.to_string(), token.to_string())]), + HashMap::from([(GCS_TOKEN.to_string(), token.to_string())]), )) } } @@ -1483,11 +1468,8 @@ mod tests { #[cfg(feature = "opendal-gcs")] fn gcs_storage(server: &mockito::Server, provider: GcsTokenProvider) -> OpenDalStorage { let config = gcs_config_parse(HashMap::from([ - (iceberg::io::GCS_SERVICE_HOST.to_string(), server.url()), - ( - iceberg::io::GCS_TOKEN.to_string(), - "static-token".to_string(), - ), + (GCS_SERVICE_HOST.to_string(), server.url()), + (GCS_TOKEN.to_string(), "static-token".to_string()), ])) .unwrap(); OpenDalStorage::Gcs { @@ -1587,7 +1569,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result { Ok(s3_credential("s3", "access-key")) } } @@ -1722,15 +1704,12 @@ mod tests { client_config: OpenDalClientConfig::default(), }; - for scheme in ["gs", "gcs"] { - let path = format!("{scheme}://my-bucket/path/to/file.parquet"); - assert_eq!( - storage.relativize_path(&path).unwrap(), - "path/to/file.parquet" - ); - let (_, relative) = storage.create_operator(&path).unwrap(); - assert_eq!(relative, "path/to/file.parquet"); - } + assert_eq!( + storage + .relativize_path("gs://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); } #[cfg(feature = "opendal-gcs")] @@ -1747,11 +1726,6 @@ mod tests { .relativize_path("s3://my-bucket/path/to/file.parquet") .is_err() ); - assert!( - storage - .create_operator(&"s3://my-bucket/path/to/file.parquet") - .is_err() - ); } #[cfg(feature = "opendal-oss")] diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 114e8c405e..eeee1b4203 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -239,7 +239,7 @@ impl StorageFactory for OpenDalResolvingStorageFactory { /// Sub-storages are lazily created on first use for each scheme and cached /// for subsequent operations. Scheme aliases like `s3`/`s3a`/`s3n` map to /// the same canonical scheme, so they share a storage instance. -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct OpenDalResolvingStorage { /// Configuration properties shared across all backends. props: HashMap, @@ -260,15 +260,6 @@ pub struct OpenDalResolvingStorage { credential_provider: Option>, } -impl std::fmt::Debug for OpenDalResolvingStorage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // `props` can contain storage secrets - f.debug_struct("OpenDalResolvingStorage") - .field("property_keys", &self.props.keys().collect::>()) - .finish_non_exhaustive() - } -} - impl OpenDalResolvingStorage { /// Resolve the storage for the given path by extracting the canonical scheme and /// returning the cached or newly-created [`OpenDalStorage`]. @@ -410,6 +401,15 @@ mod tests { use std::time::Duration; #[allow(unused_imports)] + #[cfg(any( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ) + ))] + use iceberg::io::StorageCredential; + use super::*; use crate::OPENDAL_IO_TIMEOUT_MS; @@ -436,7 +436,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result { unreachable!("unsupported backends must ignore the provider") } } diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index d94dad6b43..5321137612 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -15,6 +15,20 @@ // specific language governing permissions and limitations // under the License. +use cfg_if::cfg_if; +use iceberg::io::StorageCredential; + +cfg_if! { + if #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] { + use std::collections::HashMap; + use std::sync::Arc; + + use iceberg::io::StorageCredentialProvider; + use reqsign_core::time::Timestamp; + use reqsign_core::{Error as ReqsignError, Result as ReqsignResult}; + } +} + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn is_truthy(value: &str) -> bool { ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()) @@ -32,16 +46,14 @@ pub(crate) fn from_opendal_error(e: opendal::Error) -> iceberg::Error { /// The non-empty value of `key` in a vended credential's config. #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn required_credential_property<'a>( - config: &'a std::collections::HashMap, + config: &'a HashMap, key: &str, -) -> reqsign_core::Result<&'a str> { +) -> ReqsignResult<&'a str> { config .get(key) .map(String::as_str) .filter(|value| !value.is_empty()) - .ok_or_else(|| { - reqsign_core::Error::unexpected(format!("vended credential is missing {key}")) - }) + .ok_or_else(|| ReqsignError::unexpected(format!("vended credential is missing {key}"))) } /// The latest epoch millisecond `reqsign` timestamps can represent, @@ -55,9 +67,9 @@ const MAX_TIMESTAMP_MILLIS: i64 = 253_402_207_200_000; /// are clamped to the latest representable timestamp. #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn credential_expiry( - config: &std::collections::HashMap, + config: &HashMap, key: &str, -) -> reqsign_core::Result> { +) -> ReqsignResult> { config .get(key) .filter(|value| !value.is_empty()) @@ -66,15 +78,10 @@ pub(crate) fn credential_expiry( .parse::() .ok() .and_then(|millis| { - reqsign_core::time::Timestamp::from_millisecond( - millis.min(MAX_TIMESTAMP_MILLIS), - ) - .ok() + Timestamp::from_millisecond(millis.min(MAX_TIMESTAMP_MILLIS)).ok() }) .ok_or_else(|| { - reqsign_core::Error::unexpected(format!( - "vended credential has an invalid {key}" - )) + ReqsignError::unexpected(format!("vended credential has an invalid {key}")) }) }) .transpose() @@ -85,22 +92,19 @@ pub(crate) fn credential_expiry( #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn validate_credential_prefix( location: &str, - credential: &iceberg::io::StorageCredential, -) -> reqsign_core::Result<()> { + credential: &StorageCredential, +) -> ReqsignResult<()> { if credential.covers(location) { Ok(()) } else { - Err(reqsign_core::Error::unexpected(uncovered_location_message( + Err(ReqsignError::unexpected(uncovered_location_message( location, credential, ))) } } /// The error message for a vended credential that does not cover `location`. -pub(crate) fn uncovered_location_message( - location: &str, - credential: &iceberg::io::StorageCredential, -) -> String { +pub(crate) fn uncovered_location_message(location: &str, credential: &StorageCredential) -> String { format!( "vended credential prefix {:?} does not cover storage location {location:?}", credential.prefix() @@ -117,7 +121,7 @@ pub(crate) fn storage_root(path: &str) -> iceberg::Result { /// `reqsign` credential provider. #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) struct VendedCredentialSource { - provider: std::sync::Arc, + provider: Arc, /// Location handed to the provider: the path an operator serves, or the /// scope shared by a bulk-delete batch. location: String, @@ -125,10 +129,7 @@ pub(crate) struct VendedCredentialSource { #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] impl VendedCredentialSource { - pub(crate) fn new( - provider: std::sync::Arc, - location: String, - ) -> Self { + pub(crate) fn new(provider: Arc, location: String) -> Self { Self { provider, location } } @@ -141,8 +142,8 @@ impl VendedCredentialSource { pub(crate) async fn load( &self, backend: &str, - extract: impl FnOnce(&std::collections::HashMap) -> reqsign_core::Result, - ) -> reqsign_core::Result { + extract: impl FnOnce(&HashMap) -> ReqsignResult, + ) -> ReqsignResult { self.try_load(backend, extract).await.inspect_err(|error| { tracing::warn!( "cannot use vended {backend} credentials for {}: {error}", @@ -154,14 +155,14 @@ impl VendedCredentialSource { async fn try_load( &self, backend: &str, - extract: impl FnOnce(&std::collections::HashMap) -> reqsign_core::Result, - ) -> reqsign_core::Result { + extract: impl FnOnce(&HashMap) -> ReqsignResult, + ) -> ReqsignResult { let credential = self .provider .load_credential(&self.location) .await .map_err(|e| { - reqsign_core::Error::unexpected(format!( + ReqsignError::unexpected(format!( "failed to load vended {backend} credential for {}: {e}", self.location )) @@ -183,13 +184,10 @@ impl std::fmt::Debug for VendedCredentialSource { #[cfg(all(test, any(feature = "opendal-s3", feature = "opendal-gcs")))] mod tests { - use std::collections::HashMap; - use std::sync::{Arc, Mutex}; + use std::sync::Mutex; use async_trait::async_trait; - use iceberg::io::{ - GCS_TOKEN, GCS_TOKEN_EXPIRES_AT, StorageCredential, StorageCredentialProvider, - }; + use iceberg::io::{GCS_TOKEN, GCS_TOKEN_EXPIRES_AT}; use super::*; @@ -215,7 +213,7 @@ mod tests { } } - async fn load(prefix: &'static str, location: &str) -> reqsign_core::Result<()> { + async fn load(prefix: &'static str, location: &str) -> ReqsignResult<()> { let provider = Arc::new(RecordingProvider { prefix, requested: Mutex::new(Vec::new()), @@ -258,7 +256,7 @@ mod tests { assert!(required_credential_property(&config, "missing").is_err()); assert_eq!( credential_expiry(&config, GCS_TOKEN_EXPIRES_AT).unwrap(), - Some(reqsign_core::time::Timestamp::from_millisecond(1500).unwrap()) + Some(Timestamp::from_millisecond(1500).unwrap()) ); assert_eq!(credential_expiry(&config, "missing").unwrap(), None); // An expiry beyond year 9999, such as Java's `Long.MAX_VALUE`, is @@ -266,9 +264,9 @@ mod tests { let never = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), i64::MAX.to_string())]); assert_eq!( credential_expiry(&never, GCS_TOKEN_EXPIRES_AT).unwrap(), - Some(reqsign_core::time::Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS).unwrap()) + Some(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS).unwrap()) ); - assert!(reqsign_core::time::Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS + 1).is_err()); + assert!(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS + 1).is_err()); // The value is never quoted in the error. let invalid = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), "secret".to_string())]); let error = credential_expiry(&invalid, GCS_TOKEN_EXPIRES_AT) From 7ff3d887594502ca2dd04d4b7b95ce3b8c46c6f6 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 09:53:15 +0100 Subject: [PATCH 4/8] test(io): cover credential provider propagation and refresh - Warn once when the default `build_with_credential_provider` ignores a provider. The in-memory and local filesystem factories, which need no credentials, ignore it silently. - Document how early providers should refresh, given that the OpenDAL backends reload credentials within two minutes of expiry. - Test that `FileIO` and `OpenDalResolvingStorage` pass the provider to the storages they build, and that S3 and GCS operators reload a credential as it nears expiry. - Make the bulk-delete test check that a superseded batch is flushed before the stream ends, instead of depending on map order. - Mark `OpenDalStorage::Memory` and `LocalFs` `#[non_exhaustive]`, like the other variants. - Describe the refresh properties as catalog table properties. - Keep `main`'s `reqsign-aws-v4` floor. - Prefix the new tests with `test_`, keep comments to this change, and restore the `CustomAwsCredentialLoader::new` cast. --- Cargo.lock | 32 ++-- crates/iceberg/src/io/file_io.rs | 45 ++++- crates/iceberg/src/io/storage/config/azdls.rs | 6 +- crates/iceberg/src/io/storage/config/gcs.rs | 4 +- crates/iceberg/src/io/storage/config/s3.rs | 4 +- crates/iceberg/src/io/storage/local_fs.rs | 12 +- crates/iceberg/src/io/storage/memory.rs | 12 +- crates/iceberg/src/io/storage/mod.rs | 32 +++- crates/storage/opendal/Cargo.toml | 2 +- crates/storage/opendal/public-api.txt | 4 +- crates/storage/opendal/src/gcs.rs | 6 +- crates/storage/opendal/src/lib.rs | 181 +++++++++++++++++- crates/storage/opendal/src/resolving.rs | 56 +++--- crates/storage/opendal/src/s3.rs | 6 +- crates/storage/opendal/src/utils.rs | 8 +- 15 files changed, 326 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0bd589256..443b9124fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3949,7 +3949,7 @@ dependencies = [ "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "serde", "serde_json", @@ -4029,7 +4029,7 @@ dependencies = [ "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4073,7 +4073,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -4108,7 +4108,7 @@ dependencies = [ "http 1.5.0", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -4128,7 +4128,7 @@ dependencies = [ "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.41.0", + "quick-xml", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -4636,16 +4636,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quick-xml" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quinn" version = "0.11.11" @@ -4944,9 +4934,9 @@ dependencies = [ [[package]] name = "reqsign-aws-core" -version = "3.2.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" dependencies = [ "bytes", "form_urlencoded", @@ -4954,7 +4944,7 @@ dependencies = [ "http 1.5.0", "log", "percent-encoding", - "quick-xml 0.42.0", + "quick-xml", "reqsign-core", "rust-ini", "serde", @@ -4965,14 +4955,14 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.3.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" dependencies = [ "bytes", "http 1.5.0", "log", - "quick-xml 0.42.0", + "quick-xml", "reqsign-aws-core", "reqsign-core", "serde", diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index bf996f3012..bbe5c6a0be 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -188,7 +188,7 @@ impl FileIO { /// documented by each storage factory implementation. /// /// A credential provider is rebuilt when the binary links its factory implementation, such as - /// the REST catalog's. Otherwise, or when rebuilding fails, the `FileIO` is deserialized without + /// a catalog's. Otherwise, or when rebuilding fails, the `FileIO` is deserialized without /// it and logs a warning, as [`FileIO::serialize_all`] does. pub fn deserialize_all(bytes: &[u8]) -> Result { let _serde::DeserializedFileIO { @@ -571,6 +571,7 @@ mod tests { use std::io::Write; use std::path::Path; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use bytes::Bytes; @@ -582,8 +583,9 @@ mod tests { use super::{FileIO, FileIOBuilder}; use crate::Result; use crate::io::{ - GCS_TOKEN, LocalFsStorageFactory, MemoryStorageFactory, StorageConfig, StorageCredential, - StorageCredentialProvider, StorageCredentialProviderFactory, + GCS_TOKEN, LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, + StorageCredential, StorageCredentialProvider, StorageCredentialProviderFactory, + StorageFactory, }; #[derive(Debug)] @@ -785,6 +787,43 @@ mod tests { assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + /// Memory storage that records whether it was given a credential provider. + #[derive(Debug, Default, Serialize, Deserialize)] + struct ProviderRecordingFactory { + #[serde(skip)] + received_provider: Arc, + } + + #[typetag::serde] + impl StorageFactory for ProviderRecordingFactory { + fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + self.received_provider + .store(credential_provider.is_some(), Ordering::SeqCst); + MemoryStorageFactory.build(config) + } + } + + #[tokio::test] + async fn test_file_io_passes_credential_provider_to_factory() { + let received_provider = Arc::new(AtomicBool::new(false)); + let file_io = FileIOBuilder::new(Arc::new(ProviderRecordingFactory { + received_provider: Arc::clone(&received_provider), + })) + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + file_io.exists("memory://file").await.unwrap(); + assert!(received_provider.load(Ordering::SeqCst)); + } + #[tokio::test] async fn test_file_io_ignores_credentials_for_unsupported_factory() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) diff --git a/crates/iceberg/src/io/storage/config/azdls.rs b/crates/iceberg/src/io/storage/config/azdls.rs index 7d49a354dc..833254a2af 100644 --- a/crates/iceberg/src/io/storage/config/azdls.rs +++ b/crates/iceberg/src/io/storage/config/azdls.rs @@ -36,13 +36,13 @@ pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name"; pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key"; /// The shared access signature. pub const ADLS_SAS_TOKEN: &str = "adls.sas-token"; -/// Prefix for account-specific shared access signatures vended by a REST catalog. +/// Prefix for account-specific shared access signatures vended by a catalog. pub const ADLS_SAS_TOKEN_PREFIX: &str = "adls.sas-token."; /// Prefix for the epoch-millisecond expiration of an account-specific SAS token. pub const ADLS_SAS_TOKEN_EXPIRES_AT_MS_PREFIX: &str = "adls.sas-token-expires-at-ms."; -/// Table property naming the endpoint used to refresh vended ADLS credentials. +/// Table property naming the endpoint a catalog serves to refresh vended ADLS credentials. pub const ADLS_REFRESH_CREDENTIALS_ENDPOINT: &str = "adls.refresh-credentials-endpoint"; -/// Table property controlling whether vended ADLS credentials are refreshed. +/// Table property enabling refresh of vended ADLS credentials. Defaults to `true`. pub const ADLS_REFRESH_CREDENTIALS_ENABLED: &str = "adls.refresh-credentials-enabled"; /// The tenant-id. pub const ADLS_TENANT_ID: &str = "adls.tenant-id"; diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs index 6d06ab6a34..83ad92e975 100644 --- a/crates/iceberg/src/io/storage/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -43,9 +43,9 @@ pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; pub const GCS_TOKEN: &str = "gcs.oauth2.token"; /// Epoch-millisecond timestamp at which the vended GCS OAuth2 token expires. pub const GCS_TOKEN_EXPIRES_AT: &str = "gcs.oauth2.token-expires-at"; -/// Endpoint used to fetch and refresh vended GCS OAuth2 credentials. +/// Table property naming the endpoint a catalog serves to refresh vended GCS OAuth2 credentials. pub const GCS_REFRESH_CREDENTIALS_ENDPOINT: &str = "gcs.oauth2.refresh-credentials-endpoint"; -/// Whether vended GCS OAuth2 credentials should be refreshed. Defaults to `true`. +/// Table property enabling refresh of vended GCS OAuth2 credentials. Defaults to `true`. pub const GCS_REFRESH_CREDENTIALS_ENABLED: &str = "gcs.oauth2.refresh-credentials-enabled"; /// Option to skip signing requests (e.g. for public buckets/folders). pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index b0dcc115c1..87b4654912 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -42,9 +42,9 @@ pub const S3_SESSION_TOKEN_EXPIRES_AT_MS: &str = "s3.session-token-expires-at-ms pub const S3_REGION: &str = "s3.region"; /// Region to use for the S3 client (takes precedence over [`S3_REGION`]). pub const CLIENT_REGION: &str = "client.region"; -/// Endpoint used to fetch and refresh vended AWS credentials. +/// Table property naming the endpoint a catalog serves to refresh vended AWS credentials. pub const AWS_REFRESH_CREDENTIALS_ENDPOINT: &str = "client.refresh-credentials-endpoint"; -/// Whether vended AWS credentials should be refreshed. Defaults to `true`. +/// Table property enabling refresh of vended AWS credentials. Defaults to `true`. pub const AWS_REFRESH_CREDENTIALS_ENABLED: &str = "client.refresh-credentials-enabled"; /// S3 Path Style Access. pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access"; diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index cb780e86e7..4c8427a917 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -36,7 +36,7 @@ use serde::{Deserialize, Serialize}; use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use crate::{Error, ErrorKind, Result}; @@ -318,6 +318,16 @@ impl StorageFactory for LocalFsStorageFactory { fn build(&self, _config: &StorageConfig) -> Result> { Ok(Arc::new(LocalFsStorage::new())) } + + /// Local filesystem storage needs no credentials. + #[allow(unused_variables)] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + self.build(config) + } } #[cfg(test)] diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index fefa7973e8..b9c54b7619 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -35,7 +35,7 @@ use serde::{Deserialize, Serialize}; use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use crate::{Error, ErrorKind, Result}; @@ -242,6 +242,16 @@ impl StorageFactory for MemoryStorageFactory { fn build(&self, _config: &StorageConfig) -> Result> { Ok(Arc::new(MemoryStorage::new())) } + + /// In-memory storage needs no credentials. + #[allow(unused_variables)] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + self.build(config) + } } /// File reader for in-memory storage. diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index 7de00854ac..d4aca495d3 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -23,7 +23,7 @@ mod memory; use std::collections::HashMap; use std::fmt::Debug; -use std::sync::Arc; +use std::sync::{Arc, Once}; use async_trait::async_trait; use bytes::Bytes; @@ -145,13 +145,25 @@ pub trait StorageFactory: Debug + Send + Sync { /// that the backend can call to obtain and refresh short-lived credentials. /// /// Backends that cannot use the provider ignore it and use the credentials - /// in `config`, as they would without one. The default does exactly that. - #[allow(unused_variables)] + /// in `config`, as they would without one. The default does so and logs a + /// warning once, since the storage's credentials are then not refreshed. + /// Factories that wrap another factory should forward the provider to it, + /// and factories whose storage needs no credentials can ignore it silently. fn build_with_credential_provider( &self, config: &StorageConfig, credential_provider: Option>, ) -> Result> { + if credential_provider.is_some() { + static WARNED: Once = Once::new(); + WARNED.call_once(|| { + tracing::warn!( + "{} ignores storage credential providers; its storage uses the credentials \ + in its configuration, which are not refreshed", + std::any::type_name::() + ) + }); + } self.build(config) } } @@ -174,6 +186,13 @@ pub trait StorageFactory: Debug + Send + Sync { /// Implementations must cache internally and only re-fetch when the current /// credential is at or near expiry; otherwise every object-store request could /// trigger a call back to the catalog. +/// +/// Backends may treat a credential as stale before it expires, and load +/// another on every request until they get a fresher one. The OpenDAL S3 and +/// GCS backends do so within two minutes of expiry, and reject a newly loaded +/// credential that expires within ten seconds. Implementations should +/// therefore refresh early enough that the credential they return has more +/// than two minutes left. #[async_trait] pub trait StorageCredentialProvider: Debug + Send + Sync { /// Return whether this provider supplies credentials for `path`. @@ -225,8 +244,7 @@ pub trait StorageCredentialProviderFactory: Debug + Send + Sync { } /// A vended storage credential: the storage properties that apply to -/// locations under a prefix, like Java's `StorageCredential` and the REST -/// catalog's `storage-credentials`. +/// locations under a prefix, like Java's `StorageCredential`. /// /// `config` holds backend storage properties, such as `s3.access-key-id`, /// `s3.secret-access-key`, `s3.session-token` and @@ -294,7 +312,7 @@ mod tests { } #[test] - fn credential_prefix_matches_like_java() { + fn test_credential_prefix_matches_like_java() { let credential = scoped("s3://bucket/table"); assert!(credential.covers("s3://bucket/table")); assert!(credential.covers("s3://bucket/table/data/file.parquet")); @@ -310,7 +328,7 @@ mod tests { } #[test] - fn credential_debug_omits_config_values() { + fn test_credential_debug_omits_config_values() { let debug = format!("{:?}", scoped("s3://bucket")); assert!(debug.contains("s3://bucket"), "{debug}"); assert!(!debug.contains("secret"), "{debug}"); diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index b26b6fab31..56b4a65469 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -56,7 +56,7 @@ futures = { workspace = true } iceberg = { workspace = true } iceberg-property-macro = { workspace = true } opendal = { workspace = true } -reqsign-aws-v4 = { version = "3.3.0", optional = true } +reqsign-aws-v4 = { version = "3.0.0", optional = true } reqsign-core = { version = "3.3.1", optional = true } reqsign-google = { version = "3.0.4", optional = true } serde = { workspace = true } diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 462a6897e3..704250689b 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -12,9 +12,9 @@ pub iceberg_storage_opendal::OpenDalStorage::Gcs::credential_provider: core::opt #[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::LocalFs +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::LocalFs pub iceberg_storage_opendal::OpenDalStorage::LocalFs::client_config: iceberg_storage_opendal::OpenDalClientConfig -pub iceberg_storage_opendal::OpenDalStorage::Memory +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Memory pub iceberg_storage_opendal::OpenDalStorage::Memory::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Memory::operator: opendal_core::types::operator::operator::Operator #[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Oss diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index 8a8795e3e0..27e32d0039 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -199,7 +199,7 @@ mod tests { } #[tokio::test] - async fn vended_adapter_returns_expiring_oauth2_token() { + async fn test_vended_adapter_returns_expiring_oauth2_token() { let credential = credential(&[(GCS_TOKEN, "ya29.token"), (GCS_TOKEN_EXPIRES_AT, "1500")]); let google = adapter(credential) @@ -216,7 +216,7 @@ mod tests { } #[tokio::test] - async fn vended_adapter_rejects_other_credentials() { + async fn test_vended_adapter_rejects_other_credentials() { assert!( adapter(credential(&[(S3_ACCESS_KEY_ID, "AK")])) .provide_credential(&Context::new()) @@ -226,7 +226,7 @@ mod tests { } #[test] - fn vended_credentials_disable_every_other_source() { + fn test_vended_credentials_disable_every_other_source() { let mut cfg = gcs_config_parse(HashMap::from([ (GCS_TOKEN.to_string(), "static-token".to_string()), (GCS_CREDENTIALS_JSON.to_string(), "e30=".to_string()), diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index c63c314c75..e7afe94f20 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -309,6 +309,7 @@ fn default_memory_operator() -> Operator { #[non_exhaustive] pub enum OpenDalStorage { /// Memory storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-memory")] Memory { /// Pre-built memory operator. @@ -319,6 +320,7 @@ pub enum OpenDalStorage { client_config: OpenDalClientConfig, }, /// Local filesystem storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-fs")] LocalFs { /// Backend-independent client settings. @@ -416,10 +418,10 @@ struct DeleteBatchKey { impl DeleteBatchKey { /// Open batches to flush before adding a path with this key. /// - /// A scoped credential inside the scope of an open batch means the catalog - /// is replacing that batch's broader credential, such as a scheme-wide or - /// account-wide seed. Such batches are flushed while their credential is - /// still valid: once it expires, nothing may cover their whole scope. + /// A scoped credential inside the scope of an open batch means the provider + /// is replacing that batch's broader credential, such as a scheme-wide one. + /// Such batches are flushed while their credential is still valid: once it + /// expires, nothing may cover their whole scope. fn superseded_batches<'a>( &self, open: impl Iterator, @@ -1002,10 +1004,16 @@ impl FileWrite for OpenDalWriter { #[cfg(test)] mod tests { #[cfg(feature = "opendal-s3")] - use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::atomic::AtomicBool; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use std::sync::atomic::{AtomicUsize, Ordering}; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use std::time::{SystemTime, UNIX_EPOCH}; + #[cfg(feature = "opendal-s3")] + use iceberg::io::S3_SESSION_TOKEN_EXPIRES_AT_MS; #[cfg(feature = "opendal-gcs")] - use iceberg::io::{GCS_SERVICE_HOST, GCS_TOKEN}; + use iceberg::io::{GCS_SERVICE_HOST, GCS_TOKEN, GCS_TOKEN_EXPIRES_AT}; #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] use iceberg::io::{S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, StorageCredential}; @@ -1362,6 +1370,8 @@ mod tests { struct ChangingScopeProvider { scoped: AtomicBool, scheme_wide_expired: AtomicBool, + /// Whether the scheme-wide batch, scoped to the storage root, was signed. + scheme_wide_signed: AtomicBool, } #[cfg(feature = "opendal-s3")] @@ -1372,6 +1382,9 @@ mod tests { } async fn load_credential(&self, path: &str) -> Result { + if path == "s3://bucket/" { + self.scheme_wide_signed.store(true, Ordering::SeqCst); + } if path.starts_with("s3://bucket/table-c") { self.scoped.store(true, Ordering::SeqCst); } @@ -1421,18 +1434,27 @@ mod tests { config.region = Some("us-east-1".to_string()); config.disable_config_load = true; config.disable_ec2_metadata = true; + let provider = Arc::new(ChangingScopeProvider::default()); let storage = OpenDalStorage::S3 { config: Arc::new(config), customized_credential_load: None, - credential_provider: Some(Arc::new(ChangingScopeProvider::default())), + credential_provider: Some(provider.clone()), client_config: OpenDalClientConfig::default(), }; + // Polled once the `table-c` path is batched, before the stream ends + // and every open batch is closed. + let scheme_wide_flushed_early = futures::stream::once(async move { + assert!(provider.scheme_wide_signed.load(Ordering::SeqCst)); + None + }) + .filter_map(futures::future::ready); storage .delete_stream( futures::stream::iter([ "s3://bucket/table-b/f.parquet".to_string(), "s3://bucket/table-c/f.parquet".to_string(), ]) + .chain(scheme_wide_flushed_early) .boxed(), ) .await @@ -1442,6 +1464,89 @@ mod tests { scoped.assert_async().await; } + /// A credential lifetime that covers one request, but falls within the two + /// minutes before expiry in which reqsign reloads a cached S3 or GCS + /// credential. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + const EXPIRING_LIFETIME: Duration = Duration::from_secs(60); + + /// The epoch millisecond `lifetime` from now, as vended expiries are given. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + fn epoch_millis_after(lifetime: Duration) -> String { + (SystemTime::now() + lifetime) + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .to_string() + } + + /// Vends `EXPIRING_AK`, which is about to expire, and then `FRESH_AK`. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct ExpiringS3Provider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for ExpiringS3Provider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + let (access_key, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { + 0 => ("EXPIRING_AK", EXPIRING_LIFETIME), + _ => ("FRESH_AK", Duration::from_secs(3600)), + }; + let mut config = s3_credential("s3", access_key).config().clone(); + config.insert( + S3_SESSION_TOKEN_EXPIRES_AT_MS.to_string(), + epoch_millis_after(lifetime), + ); + Ok(StorageCredential::new("s3", config)) + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_s3_operator_reloads_an_expiring_credential() { + let mut server = mockito::Server::new_async().await; + let mut stat = |access_key: &str| { + server + .mock("HEAD", "/bucket/file.parquet") + .match_header( + "authorization", + mockito::Matcher::Regex(format!("Credential={access_key}/")), + ) + .expect(1) + .with_status(200) + .with_header("content-length", "0") + }; + let expiring = stat("EXPIRING_AK").create_async().await; + let fresh = stat("FRESH_AK").create_async().await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(ExpiringS3Provider::default())), + client_config: OpenDalClientConfig::default(), + }; + let path = "s3://bucket/file.parquet"; + let (operator, relative_path) = storage.create_operator(&path).unwrap(); + for _ in 0..2 { + operator.stat(relative_path).await.unwrap(); + } + + expiring.assert_async().await; + fresh.assert_async().await; + } + /// Vends `token` as a GCS credential, or fails when it is `None`. #[cfg(feature = "opendal-gcs")] #[derive(Debug)] @@ -1466,7 +1571,10 @@ mod tests { } #[cfg(feature = "opendal-gcs")] - fn gcs_storage(server: &mockito::Server, provider: GcsTokenProvider) -> OpenDalStorage { + fn gcs_storage( + server: &mockito::Server, + provider: impl StorageCredentialProvider + 'static, + ) -> OpenDalStorage { let config = gcs_config_parse(HashMap::from([ (GCS_SERVICE_HOST.to_string(), server.url()), (GCS_TOKEN.to_string(), "static-token".to_string()), @@ -1496,6 +1604,63 @@ mod tests { mock.assert_async().await; } + /// Vends `expiring-token`, which is about to expire, and then `fresh-token`. + #[cfg(feature = "opendal-gcs")] + #[derive(Debug, Default)] + struct ExpiringGcsProvider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-gcs")] + #[async_trait] + impl StorageCredentialProvider for ExpiringGcsProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + let (token, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { + 0 => ("expiring-token", EXPIRING_LIFETIME), + _ => ("fresh-token", Duration::from_secs(3600)), + }; + Ok(StorageCredential::new( + "gs", + HashMap::from([ + (GCS_TOKEN.to_string(), token.to_string()), + ( + GCS_TOKEN_EXPIRES_AT.to_string(), + epoch_millis_after(lifetime), + ), + ]), + )) + } + } + + #[cfg(feature = "opendal-gcs")] + #[tokio::test] + async fn test_gcs_operator_reloads_an_expiring_token() { + let mut server = mockito::Server::new_async().await; + let mut stat = |token: &str| { + server + .mock("GET", mockito::Matcher::Any) + .match_header("authorization", format!("Bearer {token}").as_str()) + .expect(1) + .with_status(404) + }; + let expiring = stat("expiring-token").create_async().await; + let fresh = stat("fresh-token").create_async().await; + + let storage = gcs_storage(&server, ExpiringGcsProvider::default()); + let path = "gs://bucket/file.parquet"; + let (operator, relative_path) = storage.create_operator(&path).unwrap(); + for _ in 0..2 { + assert!(!operator.exists(relative_path).await.unwrap()); + } + + expiring.assert_async().await; + fresh.assert_async().await; + } + #[cfg(feature = "opendal-gcs")] #[tokio::test] async fn test_gcs_does_not_fall_back_when_the_provider_fails() { diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index eeee1b4203..e8e538ef27 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -400,36 +400,15 @@ impl Storage for OpenDalResolvingStorage { mod tests { use std::time::Duration; - #[allow(unused_imports)] - #[cfg(any( - not(any(feature = "opendal-s3", feature = "opendal-gcs")), - all( - feature = "opendal-memory", - any(feature = "opendal-s3", feature = "opendal-gcs") - ) - ))] use iceberg::io::StorageCredential; + #[allow(unused_imports)] use super::*; use crate::OPENDAL_IO_TIMEOUT_MS; - #[cfg(any( - not(any(feature = "opendal-s3", feature = "opendal-gcs")), - all( - feature = "opendal-memory", - any(feature = "opendal-s3", feature = "opendal-gcs") - ) - ))] #[derive(Debug)] struct AllPathsCredentialProvider; - #[cfg(any( - not(any(feature = "opendal-s3", feature = "opendal-gcs")), - all( - feature = "opendal-memory", - any(feature = "opendal-s3", feature = "opendal-gcs") - ) - ))] #[async_trait] impl StorageCredentialProvider for AllPathsCredentialProvider { fn supports_path(&self, _path: &str) -> bool { @@ -437,7 +416,7 @@ mod tests { } async fn load_credential(&self, _path: &str) -> Result { - unreachable!("unsupported backends must ignore the provider") + unreachable!("resolving a storage never loads credentials") } } @@ -558,6 +537,37 @@ mod tests { assert!(storage.resolve("memory:/key").is_ok()); } + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[test] + #[allow(unreachable_patterns)] + fn test_resolve_passes_the_credential_provider_to_s3_and_gcs() { + let mut storage = empty_resolving_storage(); + storage.credential_provider = Some(Arc::new(AllPathsCredentialProvider)); + + let paths: &[&str] = &[ + #[cfg(feature = "opendal-s3")] + "s3://bucket/key", + #[cfg(feature = "opendal-gcs")] + "gs://bucket/key", + ]; + for path in paths { + let has_provider = match storage.resolve(path).unwrap().as_ref() { + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + credential_provider, + .. + } => credential_provider.is_some(), + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { + credential_provider, + .. + } => credential_provider.is_some(), + _ => false, + }; + assert!(has_provider, "{path}"); + } + } + #[cfg(feature = "opendal-azdls")] #[test] fn test_resolve_azdls_aliases_share_instance() { diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 2fd7e1c4a1..fee7bd896f 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -235,7 +235,7 @@ impl std::fmt::Debug for CustomAwsCredentialLoader { impl CustomAwsCredentialLoader { /// Create a new custom AWS credential loader from any [`ProvideCredential`] implementation. pub fn new(provider: impl ProvideCredential + 'static) -> Self { - Self(Arc::new(provider)) + Self(Arc::new(provider) as Arc>) } } @@ -282,7 +282,7 @@ mod tests { } #[tokio::test] - async fn vended_adapter_returns_expiring_aws_credential() { + async fn test_vended_adapter_returns_expiring_aws_credential() { let credential = credential("s3://bucket/table", &[ (S3_ACCESS_KEY_ID, "AK"), (S3_SECRET_ACCESS_KEY, "SK"), @@ -305,7 +305,7 @@ mod tests { } #[tokio::test] - async fn vended_adapter_rejects_incomplete_or_uncovering_credentials() { + async fn test_vended_adapter_rejects_incomplete_or_uncovering_credentials() { for credential in [ // A credential for another backend. credential("s3", &[(GCS_TOKEN, "token")]), diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index 5321137612..83802b1b6e 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -229,7 +229,7 @@ mod tests { } #[tokio::test] - async fn vended_source_requires_a_covering_credential() { + async fn test_vended_source_requires_a_covering_credential() { let location = "gs://bucket/table/data/file.parquet"; assert!(load("gs", location).await.is_ok()); assert!(load("gs://bucket/table", location).await.is_ok()); @@ -244,7 +244,7 @@ mod tests { } #[test] - fn credential_properties_are_parsed() { + fn test_credential_properties_are_parsed() { let config = HashMap::from([ (GCS_TOKEN.to_string(), "token".to_string()), (GCS_TOKEN_EXPIRES_AT.to_string(), "1500".to_string()), @@ -276,7 +276,7 @@ mod tests { } #[test] - fn storage_root_keeps_scheme_and_authority() { + fn test_storage_root_keeps_scheme_and_authority() { assert_eq!( storage_root("s3a://bucket/table/file.parquet").unwrap(), "s3a://bucket/" @@ -306,7 +306,7 @@ mod tests { } #[tokio::test] - async fn vended_source_errors_carry_the_cause() { + async fn test_vended_source_errors_carry_the_cause() { let source = VendedCredentialSource::new( Arc::new(FailingProvider), "gs://bucket/file.parquet".to_string(), From f27617569d61af15cf3341fd07436d0dbb92acfc Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 11:50:55 +0100 Subject: [PATCH 5/8] test(io): cover the default credential provider fallback - Test the default `build_with_credential_provider` through a factory that implements only `build`, and say its warning is logged once per process. - Drop the GCS vended-token test, which the reload test covers. - Say that a failed load fails the request, and that a provider may come from any `FileIO`, not only a catalog. - Use "vended" rather than "dynamic" for provider credentials. - Note why the timestamp test checks reqsign's bound. --- crates/iceberg/src/io/file_io.rs | 17 ++++++++- crates/iceberg/src/io/storage/mod.rs | 12 ++++-- crates/storage/opendal/src/gcs.rs | 2 +- crates/storage/opendal/src/lib.rs | 57 +++++++++------------------- crates/storage/opendal/src/s3.rs | 2 +- crates/storage/opendal/src/utils.rs | 1 + 6 files changed, 43 insertions(+), 48 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index bbe5c6a0be..fd804a6638 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -394,7 +394,8 @@ impl FileIOBuilder { /// Attach a provider of refreshable, backend-specific credentials. /// /// Storage factories that cannot use the provider ignore it and use the - /// credentials in the configuration. + /// credentials in the configuration; the default implementation logs a + /// warning when it does. pub fn with_credential_provider( mut self, provider: Arc, @@ -824,9 +825,21 @@ mod tests { assert!(received_provider.load(Ordering::SeqCst)); } + /// Memory storage from a factory that implements only `build`, so it uses + /// the default `build_with_credential_provider`. + #[derive(Debug, Serialize, Deserialize)] + struct BuildOnlyFactory; + + #[typetag::serde] + impl StorageFactory for BuildOnlyFactory { + fn build(&self, config: &StorageConfig) -> Result> { + MemoryStorageFactory.build(config) + } + } + #[tokio::test] async fn test_file_io_ignores_credentials_for_unsupported_factory() { - let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + let file_io = FileIOBuilder::new(Arc::new(BuildOnlyFactory)) .with_credential_provider(Arc::new(TestCredentialProvider)) .build(); diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index d4aca495d3..b1ed03a41b 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -146,9 +146,10 @@ pub trait StorageFactory: Debug + Send + Sync { /// /// Backends that cannot use the provider ignore it and use the credentials /// in `config`, as they would without one. The default does so and logs a - /// warning once, since the storage's credentials are then not refreshed. - /// Factories that wrap another factory should forward the provider to it, - /// and factories whose storage needs no credentials can ignore it silently. + /// warning once per process, since the storage's credentials are then not + /// refreshed. Factories that wrap another factory should forward the + /// provider to it, and factories whose storage needs no credentials can + /// ignore it silently. fn build_with_credential_provider( &self, config: &StorageConfig, @@ -215,6 +216,9 @@ pub trait StorageCredentialProvider: Debug + Send + Sync { /// Backends read the credential's expiry from its config, such as /// `s3.session-token-expires-at-ms`, and load a new one before it expires. /// A credential without an expiry is used for as long as the backend lives. + /// + /// An error, or a credential that does not cover `path`, fails the request: + /// backends do not fall back to other credentials. async fn load_credential(&self, path: &str) -> Result; /// Return a factory that rebuilds an equivalent provider in another process. @@ -262,7 +266,7 @@ pub struct StorageCredential { impl StorageCredential { /// Create a credential for locations under `prefix`. /// - /// An empty prefix covers no location, as Java rejects it. + /// An empty prefix covers no location; Java rejects one outright. pub fn new(prefix: impl Into, config: HashMap) -> Self { Self { prefix: prefix.into(), diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index 27e32d0039..93c768d906 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -109,7 +109,7 @@ pub(crate) fn gcs_config_build( let mut builder = cfg.into_builder(); - // A catalog-supplied provider re-fetches the vended OAuth2 token as it nears expiry + // The provider re-fetches the vended OAuth2 token as it nears expiry. if let Some(provider) = credential_provider { builder = builder.credential_provider(VendedGcsCredentialProvider(VendedCredentialSource::new( diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index e7afe94f20..5cef024e05 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -339,7 +339,7 @@ pub enum OpenDalStorage { /// Custom AWS credential loader. #[serde(skip)] customized_credential_load: Option, - /// Provider of refreshable vended credentials, supplied by the catalog. + /// Provider of refreshable vended credentials, from the `FileIO`. #[serde( skip_deserializing, skip_serializing_if = "Option::is_none", @@ -356,7 +356,7 @@ pub enum OpenDalStorage { Gcs { /// GCS configuration. config: Arc, - /// Provider of refreshable vended credentials, supplied by the catalog. + /// Provider of refreshable vended credentials, from the `FileIO`. #[serde( skip_deserializing, skip_serializing_if = "Option::is_none", @@ -446,15 +446,15 @@ impl DeleteBatchKey { enum BatchCredential { /// Served by the backend's configured credentials. Static, - /// A dynamic credential for a whole scheme, such as `s3`, looked up for + /// A vended credential for a whole scheme, such as `s3`, looked up for /// the storage root. SchemeWide { root: String }, - /// A dynamic credential scoped to `prefix`, looked up for the prefix. + /// A vended credential scoped to `prefix`, looked up for the prefix. Scoped { prefix: String }, } impl BatchCredential { - /// Location the batch operator looks dynamic credentials up for. + /// Location the batch operator looks vended credentials up for. fn location(&self) -> Option<&str> { match self { BatchCredential::Static => None, @@ -484,7 +484,7 @@ impl OpenDalStorage { self.create_operator_with_credential_location(path, None) } - /// Creates an operator whose dynamic credentials are looked up for + /// Creates an operator whose vended credentials are looked up for /// `credential_location` instead of `path`. /// /// A bulk-delete batch passes its scope location, so every credential the @@ -660,7 +660,7 @@ impl OpenDalStorage { } } - /// Return the dynamic credential provider that serves `path`, if any. + /// Return the credential provider that serves `path`, if any. fn credential_provider_for_path( &self, path: &str, @@ -1547,26 +1547,20 @@ mod tests { fresh.assert_async().await; } - /// Vends `token` as a GCS credential, or fails when it is `None`. + /// Fails every credential load. #[cfg(feature = "opendal-gcs")] #[derive(Debug)] - struct GcsTokenProvider(Option<&'static str>); + struct FailingCredentialProvider; #[cfg(feature = "opendal-gcs")] #[async_trait] - impl StorageCredentialProvider for GcsTokenProvider { + impl StorageCredentialProvider for FailingCredentialProvider { fn supports_path(&self, _path: &str) -> bool { true } async fn load_credential(&self, _path: &str) -> Result { - let token = self - .0 - .ok_or_else(|| Error::new(ErrorKind::Unexpected, "refresh failed"))?; - Ok(StorageCredential::new( - "gs", - HashMap::from([(GCS_TOKEN.to_string(), token.to_string())]), - )) + Err(Error::new(ErrorKind::Unexpected, "refresh failed")) } } @@ -1587,23 +1581,6 @@ mod tests { } } - #[cfg(feature = "opendal-gcs")] - #[tokio::test] - async fn test_gcs_signs_with_the_vended_token() { - let mut server = mockito::Server::new_async().await; - let mock = server - .mock("GET", mockito::Matcher::Any) - .match_header("authorization", "Bearer vended-token") - .expect_at_least(1) - .with_status(404) - .create_async() - .await; - - let storage = gcs_storage(&server, GcsTokenProvider(Some("vended-token"))); - assert!(!storage.exists("gs://bucket/file.parquet").await.unwrap()); - mock.assert_async().await; - } - /// Vends `expiring-token`, which is about to expire, and then `fresh-token`. #[cfg(feature = "opendal-gcs")] #[derive(Debug, Default)] @@ -1671,7 +1648,7 @@ mod tests { .create_async() .await; - let storage = gcs_storage(&server, GcsTokenProvider(None)); + let storage = gcs_storage(&server, FailingCredentialProvider); assert!(storage.exists("gs://bucket/file.parquet").await.is_err()); mock.assert_async().await; } @@ -1696,7 +1673,7 @@ mod tests { #[cfg(feature = "opendal-s3")] #[tokio::test] - async fn test_dynamic_credentials_batch_by_prefix() { + async fn test_vended_credentials_batch_by_prefix() { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, @@ -1724,7 +1701,7 @@ mod tests { #[cfg(feature = "opendal-s3")] #[tokio::test] - async fn test_scheme_wide_dynamic_credentials_batch_by_storage_root() { + async fn test_scheme_wide_vended_credentials_batch_by_storage_root() { #[derive(Debug)] struct SchemeWideProvider; @@ -1808,7 +1785,7 @@ mod tests { #[cfg(feature = "opendal-s3")] #[tokio::test] - async fn test_custom_s3_credential_loader_ignores_dynamic_provider_for_batching() { + async fn test_custom_s3_credential_loader_ignores_credential_provider_for_batching() { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: Some(CustomAwsCredentialLoader::new( @@ -1827,7 +1804,7 @@ mod tests { #[cfg(feature = "opendal-s3")] #[test] - fn test_s3_rejects_anonymous_dynamic_credentials() { + fn test_s3_rejects_anonymous_vended_credentials() { let mut config = S3Config::default(); config.skip_signature = true; let storage = OpenDalStorage::S3 { @@ -1845,7 +1822,7 @@ mod tests { #[cfg(feature = "opendal-gcs")] #[test] - fn test_gcs_rejects_anonymous_dynamic_credentials() { + fn test_gcs_rejects_anonymous_vended_credentials() { let mut config = GcsConfig::default(); config.skip_signature = true; let storage = OpenDalStorage::Gcs { diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index fee7bd896f..a629409ca4 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -148,7 +148,7 @@ pub(crate) fn s3_config_build( })?; // Preserve the existing custom-loader precedence: an explicitly configured loader - // is the sole source, otherwise install the catalog provider as a replacement chain so + // is the sole source, otherwise install the credential provider as a replacement chain so // refresh failures cannot fall through to broader ambient AWS credentials. let credential_provider = credential_provider .as_ref() diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index 83802b1b6e..71394192bc 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -266,6 +266,7 @@ mod tests { credential_expiry(&never, GCS_TOKEN_EXPIRES_AT).unwrap(), Some(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS).unwrap()) ); + // Pins the constant to the latest timestamp reqsign accepts. assert!(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS + 1).is_err()); // The value is never quoted in the error. let invalid = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), "secret".to_string())]); From 8455a06dd4e25b74861d9c953546dbcf28da6d6e Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 20:01:44 +0100 Subject: [PATCH 6/8] refactor(io): mark unused credential provider parameters precisely - Name the ignored provider `_credential_provider` in the in-memory and local filesystem factories, as `build` does with `_config`. - Allow an unused provider in the OpenDAL factories only when neither S3 nor GCS support is enabled. --- crates/iceberg/public-api.txt | 8 ++++---- crates/iceberg/src/io/storage/local_fs.rs | 3 +-- crates/iceberg/src/io/storage/memory.rs | 3 +-- crates/storage/opendal/src/lib.rs | 5 ++++- crates/storage/opendal/src/resolving.rs | 5 ++++- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index ed1641a7a7..96c7b9b9c2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -906,7 +906,7 @@ impl core::fmt::Debug for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> -pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::LocalFsStorageFactory @@ -945,7 +945,7 @@ impl core::fmt::Debug for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> -pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::MemoryStorageFactory @@ -1180,10 +1180,10 @@ pub fn iceberg::io::StorageFactory::build(&self, config: &iceberg::io::StorageCo pub fn iceberg::io::StorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> -pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> -pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> pub mod iceberg::memory pub struct iceberg::memory::MemoryCatalog impl core::fmt::Debug for iceberg::memory::MemoryCatalog diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index 4c8427a917..f65d8230e0 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -320,11 +320,10 @@ impl StorageFactory for LocalFsStorageFactory { } /// Local filesystem storage needs no credentials. - #[allow(unused_variables)] fn build_with_credential_provider( &self, config: &StorageConfig, - credential_provider: Option>, + _credential_provider: Option>, ) -> Result> { self.build(config) } diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index b9c54b7619..7a151fb3c4 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -244,11 +244,10 @@ impl StorageFactory for MemoryStorageFactory { } /// In-memory storage needs no credentials. - #[allow(unused_variables)] fn build_with_credential_provider( &self, config: &StorageConfig, - credential_provider: Option>, + _credential_provider: Option>, ) -> Result> { self.build(config) } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 5cef024e05..eb75e45ae8 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -232,7 +232,10 @@ impl StorageFactory for OpenDalStorageFactory { self.build_with_credential_provider(config, None) } - #[allow(unused_variables)] + #[cfg_attr( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + allow(unused_variables) + )] fn build_with_credential_provider( &self, config: &StorageConfig, diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index e8e538ef27..4468365cb0 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -214,7 +214,10 @@ impl StorageFactory for OpenDalResolvingStorageFactory { self.build_with_credential_provider(config, None) } - #[allow(unused_variables)] + #[cfg_attr( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + allow(unused_variables) + )] fn build_with_credential_provider( &self, config: &StorageConfig, From 07744045a601a6acd01f5ec45fa961316b819dce Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Mon, 26 Oct 2026 10:47:00 +0000 Subject: [PATCH 7/8] fix(io): address review on vended credential selection and fallback - Add `StorageCredential::select`, which picks the covering credential with the longest prefix, as Java does. - Warn once per factory type, not once per process, when the default `build_with_credential_provider` ignores a provider. - Sort config keys in `StorageCredential`'s `Debug` output. - Drop the refresh-endpoint and ADLS SAS constants until the parts that read them. - Test that S3 does not fall back to static keys when the provider fails, and count provider loads in a bulk delete. --- crates/iceberg/public-api.txt | 9 +- crates/iceberg/src/io/storage/config/azdls.rs | 8 -- crates/iceberg/src/io/storage/config/gcs.rs | 4 - crates/iceberg/src/io/storage/config/s3.rs | 4 - crates/iceberg/src/io/storage/mod.rs | 73 ++++++++++--- crates/storage/opendal/src/lib.rs | 103 +++++++++++++++++- 6 files changed, 159 insertions(+), 42 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 96c7b9b9c2..82a7dfea68 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1064,6 +1064,7 @@ pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash: pub fn iceberg::io::StorageCredential::covers(&self, location: &str) -> bool pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> Self pub fn iceberg::io::StorageCredential::prefix(&self) -> &str +pub fn iceberg::io::StorageCredential::select<'a>(credentials: impl core::iter::traits::collect::IntoIterator, location: &str) -> core::option::Option<&'a Self> impl core::clone::Clone for iceberg::io::StorageCredential pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential impl core::cmp::Eq for iceberg::io::StorageCredential @@ -1078,14 +1079,8 @@ pub const iceberg::io::ADLS_AUTHORITY_HOST: &str pub const iceberg::io::ADLS_CLIENT_ID: &str pub const iceberg::io::ADLS_CLIENT_SECRET: &str pub const iceberg::io::ADLS_CONNECTION_STRING: &str -pub const iceberg::io::ADLS_REFRESH_CREDENTIALS_ENABLED: &str -pub const iceberg::io::ADLS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::ADLS_SAS_TOKEN: &str -pub const iceberg::io::ADLS_SAS_TOKEN_EXPIRES_AT_MS_PREFIX: &str -pub const iceberg::io::ADLS_SAS_TOKEN_PREFIX: &str pub const iceberg::io::ADLS_TENANT_ID: &str -pub const iceberg::io::AWS_REFRESH_CREDENTIALS_ENABLED: &str -pub const iceberg::io::AWS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::CLIENT_REGION: &str pub const iceberg::io::GCS_ALLOW_ANONYMOUS: &str pub const iceberg::io::GCS_CREDENTIALS_JSON: &str @@ -1093,8 +1088,6 @@ pub const iceberg::io::GCS_DISABLE_CONFIG_LOAD: &str pub const iceberg::io::GCS_DISABLE_VM_METADATA: &str pub const iceberg::io::GCS_NO_AUTH: &str pub const iceberg::io::GCS_PROJECT_ID: &str -pub const iceberg::io::GCS_REFRESH_CREDENTIALS_ENABLED: &str -pub const iceberg::io::GCS_REFRESH_CREDENTIALS_ENDPOINT: &str pub const iceberg::io::GCS_SERVICE_HOST: &str pub const iceberg::io::GCS_TOKEN: &str pub const iceberg::io::GCS_TOKEN_EXPIRES_AT: &str diff --git a/crates/iceberg/src/io/storage/config/azdls.rs b/crates/iceberg/src/io/storage/config/azdls.rs index 833254a2af..a9541e7791 100644 --- a/crates/iceberg/src/io/storage/config/azdls.rs +++ b/crates/iceberg/src/io/storage/config/azdls.rs @@ -36,14 +36,6 @@ pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name"; pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key"; /// The shared access signature. pub const ADLS_SAS_TOKEN: &str = "adls.sas-token"; -/// Prefix for account-specific shared access signatures vended by a catalog. -pub const ADLS_SAS_TOKEN_PREFIX: &str = "adls.sas-token."; -/// Prefix for the epoch-millisecond expiration of an account-specific SAS token. -pub const ADLS_SAS_TOKEN_EXPIRES_AT_MS_PREFIX: &str = "adls.sas-token-expires-at-ms."; -/// Table property naming the endpoint a catalog serves to refresh vended ADLS credentials. -pub const ADLS_REFRESH_CREDENTIALS_ENDPOINT: &str = "adls.refresh-credentials-endpoint"; -/// Table property enabling refresh of vended ADLS credentials. Defaults to `true`. -pub const ADLS_REFRESH_CREDENTIALS_ENABLED: &str = "adls.refresh-credentials-enabled"; /// The tenant-id. pub const ADLS_TENANT_ID: &str = "adls.tenant-id"; /// The client-id. diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs index 83ad92e975..a21faaaa1d 100644 --- a/crates/iceberg/src/io/storage/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -43,10 +43,6 @@ pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; pub const GCS_TOKEN: &str = "gcs.oauth2.token"; /// Epoch-millisecond timestamp at which the vended GCS OAuth2 token expires. pub const GCS_TOKEN_EXPIRES_AT: &str = "gcs.oauth2.token-expires-at"; -/// Table property naming the endpoint a catalog serves to refresh vended GCS OAuth2 credentials. -pub const GCS_REFRESH_CREDENTIALS_ENDPOINT: &str = "gcs.oauth2.refresh-credentials-endpoint"; -/// Table property enabling refresh of vended GCS OAuth2 credentials. Defaults to `true`. -pub const GCS_REFRESH_CREDENTIALS_ENABLED: &str = "gcs.oauth2.refresh-credentials-enabled"; /// Option to skip signing requests (e.g. for public buckets/folders). pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; /// Option to skip loading the credential from GCE metadata server. diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index 87b4654912..71ca401d7f 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -42,10 +42,6 @@ pub const S3_SESSION_TOKEN_EXPIRES_AT_MS: &str = "s3.session-token-expires-at-ms pub const S3_REGION: &str = "s3.region"; /// Region to use for the S3 client (takes precedence over [`S3_REGION`]). pub const CLIENT_REGION: &str = "client.region"; -/// Table property naming the endpoint a catalog serves to refresh vended AWS credentials. -pub const AWS_REFRESH_CREDENTIALS_ENDPOINT: &str = "client.refresh-credentials-endpoint"; -/// Table property enabling refresh of vended AWS credentials. Defaults to `true`. -pub const AWS_REFRESH_CREDENTIALS_ENABLED: &str = "client.refresh-credentials-enabled"; /// S3 Path Style Access. pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access"; /// S3 Server Side Encryption Type. diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index b1ed03a41b..d6abcb55c0 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -21,9 +21,9 @@ mod config; mod local_fs; mod memory; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::fmt::Debug; -use std::sync::{Arc, Once}; +use std::sync::{Arc, Mutex, PoisonError}; use async_trait::async_trait; use bytes::Bytes; @@ -146,8 +146,8 @@ pub trait StorageFactory: Debug + Send + Sync { /// /// Backends that cannot use the provider ignore it and use the credentials /// in `config`, as they would without one. The default does so and logs a - /// warning once per process, since the storage's credentials are then not - /// refreshed. Factories that wrap another factory should forward the + /// warning once per factory type, since the storage's credentials are then + /// not refreshed. Factories that wrap another factory should forward the /// provider to it, and factories whose storage needs no credentials can /// ignore it silently. fn build_with_credential_provider( @@ -156,14 +156,19 @@ pub trait StorageFactory: Debug + Send + Sync { credential_provider: Option>, ) -> Result> { if credential_provider.is_some() { - static WARNED: Once = Once::new(); - WARNED.call_once(|| { + // A static in a provided method is shared by every implementor. + static WARNED: Mutex> = Mutex::new(BTreeSet::new()); + let factory = std::any::type_name::(); + if WARNED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(factory) + { tracing::warn!( - "{} ignores storage credential providers; its storage uses the credentials \ - in its configuration, which are not refreshed", - std::any::type_name::() - ) - }); + "{factory} ignores storage credential providers; its storage uses the \ + credentials in its configuration, which are not refreshed" + ); + } } self.build(config) } @@ -211,7 +216,8 @@ pub trait StorageCredentialProvider: Debug + Send + Sync { /// (`s3://bucket/`) or the prefix of a previously returned credential /// (`s3://bucket/warehouse/db/table`). Providers that vend distinct /// credentials per location prefix use it to select the most specific - /// match, which must [cover](StorageCredential::covers) `path`. + /// match with [`StorageCredential::select`], which must + /// [cover](StorageCredential::covers) `path`. /// /// Backends read the credential's expiry from its config, such as /// `s3.session-token-expires-at-ms`, and load a new one before it expires. @@ -292,14 +298,26 @@ impl StorageCredential { pub fn covers(&self, location: &str) -> bool { !self.prefix.is_empty() && location.starts_with(&self.prefix) } + + /// Return the credential with the longest prefix that covers `location`. + pub fn select<'a>( + credentials: impl IntoIterator, + location: &str, + ) -> Option<&'a Self> { + credentials + .into_iter() + .filter(|credential| credential.covers(location)) + .max_by_key(|credential| credential.prefix.len()) + } } impl Debug for StorageCredential { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Config values are secrets. + // Config values are secrets. Keys are sorted for stable output. + let config_keys: BTreeSet<_> = self.config.keys().collect(); f.debug_struct("StorageCredential") .field("prefix", &self.prefix) - .field("config_keys", &self.config.keys().collect::>()) + .field("config_keys", &config_keys) .finish() } } @@ -331,10 +349,37 @@ mod tests { assert!(!scoped("").covers("s3://bucket/file.parquet")); } + #[test] + fn test_select_prefers_the_longest_covering_prefix() { + let credentials = [ + scoped("s3"), + scoped("s3://bucket/table"), + scoped("s3://bucket"), + scoped("s3://other"), + ]; + let select = |location| { + StorageCredential::select(&credentials, location).map(StorageCredential::prefix) + }; + + assert_eq!(select("s3://bucket/table/file"), Some("s3://bucket/table")); + assert_eq!(select("s3://bucket/other/file"), Some("s3://bucket")); + assert_eq!(select("s3a://bucket/table/file"), Some("s3")); + assert_eq!(select("gs://bucket/table/file"), None); + } + #[test] fn test_credential_debug_omits_config_values() { let debug = format!("{:?}", scoped("s3://bucket")); assert!(debug.contains("s3://bucket"), "{debug}"); assert!(!debug.contains("secret"), "{debug}"); + + let credential = StorageCredential::new( + "s3", + HashMap::from([ + ("b".to_string(), String::new()), + ("a".to_string(), String::new()), + ]), + ); + assert!(format!("{credential:?}").contains(r#"{"a", "b"}"#)); } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index eb75e45ae8..3c8388c87d 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1013,12 +1013,14 @@ mod tests { #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] use std::time::{SystemTime, UNIX_EPOCH}; - #[cfg(feature = "opendal-s3")] - use iceberg::io::S3_SESSION_TOKEN_EXPIRES_AT_MS; #[cfg(feature = "opendal-gcs")] use iceberg::io::{GCS_SERVICE_HOST, GCS_TOKEN, GCS_TOKEN_EXPIRES_AT}; #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] use iceberg::io::{S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, StorageCredential}; + #[cfg(feature = "opendal-s3")] + use iceberg::io::{ + S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SESSION_TOKEN_EXPIRES_AT_MS, + }; #[allow(unused_imports)] use super::*; @@ -1551,11 +1553,11 @@ mod tests { } /// Fails every credential load. - #[cfg(feature = "opendal-gcs")] + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] #[derive(Debug)] struct FailingCredentialProvider; - #[cfg(feature = "opendal-gcs")] + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] #[async_trait] impl StorageCredentialProvider for FailingCredentialProvider { fn supports_path(&self, _path: &str) -> bool { @@ -1674,6 +1676,99 @@ mod tests { assert!(error.to_string().contains("credential provider"), "{error}"); } + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_s3_does_not_fall_back_when_the_provider_fails() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("HEAD", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + // REST catalogs often return static keys next to vended credentials. + let config = s3_config_parse(HashMap::from([ + (S3_ENDPOINT.to_string(), server.url()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_ACCESS_KEY_ID.to_string(), "STATIC_AK".to_string()), + ( + S3_SECRET_ACCESS_KEY.to_string(), + "static-secret".to_string(), + ), + ])) + .unwrap(); + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(FailingCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + assert!(storage.exists("s3://bucket/file.parquet").await.is_err()); + mock.assert_async().await; + } + + /// Vends one credential scoped to `s3://bucket/table` and counts loads. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct CountingS3Provider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for CountingS3Provider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result { + self.loads.fetch_add(1, Ordering::SeqCst); + Ok(s3_credential("s3://bucket/table", "SCOPED_AK")) + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_delete_stream_loads_a_credential_per_path_and_batch() { + let mut server = mockito::Server::new_async().await; + let delete = server + .mock("POST", mockito::Matcher::Any) + .match_header( + "authorization", + mockito::Matcher::Regex("Credential=SCOPED_AK/".to_string()), + ) + .expect(1) + .with_status(200) + .with_body(r#""#) + .create_async() + .await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let provider = Arc::new(CountingS3Provider::default()); + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(provider.clone()), + client_config: OpenDalClientConfig::default(), + }; + let paths: Vec = (0..3) + .map(|i| format!("s3://bucket/table/f{i}.parquet")) + .collect(); + storage + .delete_stream(futures::stream::iter(paths.clone()).boxed()) + .await + .unwrap(); + + delete.assert_async().await; + // One load to choose each path's batch, and one to sign the batch. + assert_eq!(provider.loads.load(Ordering::SeqCst), paths.len() + 1); + } + #[cfg(feature = "opendal-s3")] #[tokio::test] async fn test_vended_credentials_batch_by_prefix() { From 7586b8d11e9971f2cf5b3b5055d0732169f2ee48 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Mon, 26 Oct 2026 11:49:30 +0000 Subject: [PATCH 8/8] feat(io)!: return shared credentials from storage credential providers - `StorageCredentialProvider::load_credential` returns an `Arc`, so frequent calls share the cached credential instead of copying it. - Log failed credential loads in the OpenDAL backends at debug level, since they recur on every request; providers log their own failures. - Add `FileIO::has_credential_provider`, so callers can tell whether a deserialized `FileIO` kept its provider. --- crates/iceberg/public-api.txt | 3 ++- crates/iceberg/src/io/file_io.rs | 25 +++++++++++++------ crates/iceberg/src/io/storage/mod.rs | 10 +++++--- crates/storage/opendal/src/gcs.rs | 4 +-- crates/storage/opendal/src/lib.rs | 33 ++++++++++++++----------- crates/storage/opendal/src/resolving.rs | 2 +- crates/storage/opendal/src/s3.rs | 4 +-- crates/storage/opendal/src/utils.rs | 14 ++++++----- 8 files changed, 57 insertions(+), 38 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index a9f5a9765a..9216e69ed6 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -810,6 +810,7 @@ pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert: pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> pub fn iceberg::io::FileIO::deserialize_all(bytes: &[u8]) -> iceberg::Result pub async fn iceberg::io::FileIO::exists(&self, path: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::io::FileIO::has_credential_provider(&self) -> bool pub fn iceberg::io::FileIO::new_input(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_output(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_with_fs() -> Self @@ -1191,7 +1192,7 @@ pub fn iceberg::io::MemoryStorage::write<'life0, 'life1, 'async_trait>(&'life0 s pub fn iceberg::io::MemoryStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub trait iceberg::io::StorageCredentialProvider: core::fmt::Debug + core::marker::Send + core::marker::Sync pub fn iceberg::io::StorageCredentialProvider::factory(&self) -> iceberg::Result> -pub fn iceberg::io::StorageCredentialProvider::load_credential<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::StorageCredentialProvider::load_credential<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg::io::StorageCredentialProvider::supports_path(&self, path: &str) -> bool pub trait iceberg::io::StorageCredentialProviderFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize pub fn iceberg::io::StorageCredentialProviderFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index fd804a6638..f1d73ea6bf 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -239,6 +239,14 @@ impl FileIO { }) } + /// Returns whether this `FileIO` has a credential provider to refresh its storage credentials. + /// + /// A deserialized `FileIO` has none when its provider could not be serialized or rebuilt; see + /// [`FileIO::serialize_all`] and [`FileIO::deserialize_all`]. + pub fn has_credential_provider(&self) -> bool { + self.credential_provider.is_some() + } + /// Returns a copy of this `FileIO` without its credential provider. /// /// The copy uses only the credentials in its storage configuration, which are not refreshed. @@ -598,7 +606,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { unreachable!("unsupported factories must ignore the provider") } } @@ -615,14 +623,14 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { - Ok(StorageCredential::new( + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(StorageCredential::new( "gs", HashMap::from([( GCS_TOKEN.to_string(), self.endpoint.clone().unwrap_or_default(), )]), - )) + ))) } fn factory(&self) -> Result> { @@ -860,7 +868,7 @@ mod tests { .build(); let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); - assert!(deserialized.credential_provider.is_none()); + assert!(!deserialized.has_credential_provider()); assert_eq!( deserialized.config().get("s3.access-key-id"), Some(&"static-key".to_string()) @@ -884,7 +892,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { unreachable!("serialization tests never load credentials") } @@ -919,7 +927,7 @@ mod tests { .build(); let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); - assert!(deserialized.credential_provider.is_none()); + assert!(!deserialized.has_credential_provider()); } #[test] @@ -935,7 +943,7 @@ mod tests { let deserialized = FileIO::deserialize_all(&serde_json::to_vec(&serialized).unwrap()).unwrap(); - assert!(deserialized.credential_provider.is_none()); + assert!(!deserialized.has_credential_provider()); assert_eq!( deserialized.config().get("s3.access-key-id"), Some(&"static-key".to_string()) @@ -950,6 +958,7 @@ mod tests { .build(); let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(deserialized.has_credential_provider()); let credential = deserialized .credential_provider .unwrap() diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index d6abcb55c0..4ddefc3c7f 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -191,7 +191,8 @@ pub trait StorageFactory: Debug + Send + Sync { /// [`load_credential`](Self::load_credential) may be called very frequently. /// Implementations must cache internally and only re-fetch when the current /// credential is at or near expiry; otherwise every object-store request could -/// trigger a call back to the catalog. +/// trigger a call back to the catalog. Returning the cached [`Arc`] keeps each +/// call from copying the credential. /// /// Backends may treat a credential as stale before it expires, and load /// another on every request until they get a fresher one. The OpenDAL S3 and @@ -224,8 +225,11 @@ pub trait StorageCredentialProvider: Debug + Send + Sync { /// A credential without an expiry is used for as long as the backend lives. /// /// An error, or a credential that does not cover `path`, fails the request: - /// backends do not fall back to other credentials. - async fn load_credential(&self, path: &str) -> Result; + /// backends do not fall back to other credentials. Backends log such + /// failures only at debug level, since they recur on every request while a + /// provider is failing, so implementations should log their own, such as + /// once per failed refresh. + async fn load_credential(&self, path: &str) -> Result>; /// Return a factory that rebuilds an equivalent provider in another process. /// diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index 93c768d906..73d241af28 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -176,8 +176,8 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { - Ok(self.0.clone()) + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(self.0.clone())) } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index a0c9d9d653..ff67f9ef04 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1076,13 +1076,13 @@ mod tests { true } - async fn load_credential(&self, path: &str) -> Result { + async fn load_credential(&self, path: &str) -> Result> { let prefix = if path.contains("/table-a/") { "s3://bucket/table-a" } else { "s3://bucket/table-b" }; - Ok(s3_credential(prefix, "access-key")) + Ok(Arc::new(s3_credential(prefix, "access-key"))) } } @@ -1428,7 +1428,7 @@ mod tests { true } - async fn load_credential(&self, path: &str) -> Result { + async fn load_credential(&self, path: &str) -> Result> { if path == "s3://bucket/" { self.scheme_wide_signed.store(true, Ordering::SeqCst); } @@ -1442,13 +1442,16 @@ mod tests { match path.strip_prefix("s3://bucket/") { Some(rest) if self.scoped.load(Ordering::SeqCst) && !rest.is_empty() => { let table = rest.split('/').next().unwrap_or_default(); - Ok(s3_credential(format!("s3://bucket/{table}"), "SCOPED_AK")) + Ok(Arc::new(s3_credential( + format!("s3://bucket/{table}"), + "SCOPED_AK", + ))) } _ if self.scheme_wide_expired.load(Ordering::SeqCst) => Err(Error::new( ErrorKind::Unexpected, "the scheme-wide credential expired", )), - _ => Ok(s3_credential("s3", "SCHEME_AK")), + _ => Ok(Arc::new(s3_credential("s3", "SCHEME_AK"))), } } } @@ -1541,7 +1544,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { let (access_key, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { 0 => ("EXPIRING_AK", EXPIRING_LIFETIME), _ => ("FRESH_AK", Duration::from_secs(3600)), @@ -1551,7 +1554,7 @@ mod tests { S3_SESSION_TOKEN_EXPIRES_AT_MS.to_string(), epoch_millis_after(lifetime), ); - Ok(StorageCredential::new("s3", config)) + Ok(Arc::new(StorageCredential::new("s3", config))) } } @@ -1606,7 +1609,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { Err(Error::new(ErrorKind::Unexpected, "refresh failed")) } } @@ -1642,12 +1645,12 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { let (token, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { 0 => ("expiring-token", EXPIRING_LIFETIME), _ => ("fresh-token", Duration::from_secs(3600)), }; - Ok(StorageCredential::new( + Ok(Arc::new(StorageCredential::new( "gs", HashMap::from([ (GCS_TOKEN.to_string(), token.to_string()), @@ -1656,7 +1659,7 @@ mod tests { epoch_millis_after(lifetime), ), ]), - )) + ))) } } @@ -1764,9 +1767,9 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { self.loads.fetch_add(1, Ordering::SeqCst); - Ok(s3_credential("s3://bucket/table", "SCOPED_AK")) + Ok(Arc::new(s3_credential("s3://bucket/table", "SCOPED_AK"))) } } @@ -1851,8 +1854,8 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { - Ok(s3_credential("s3", "access-key")) + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(s3_credential("s3", "access-key"))) } } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index d67e8885b5..250ef50de7 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -428,7 +428,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { + async fn load_credential(&self, _path: &str) -> Result> { unreachable!("resolving a storage never loads credentials") } } diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index a629409ca4..cef90535b8 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -259,8 +259,8 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> Result { - Ok(self.0.clone()) + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(self.0.clone())) } } diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index 71394192bc..318d483108 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -138,14 +138,16 @@ impl VendedCredentialSource { /// /// Errors are logged here: reqsign's credential chain replaces them with an /// error that only says no credential could be loaded, and reports the - /// cause only through the `log` crate. + /// cause only through the `log` crate. They are logged at debug level, as + /// a failing provider fails every request; providers log their own + /// failures. pub(crate) async fn load( &self, backend: &str, extract: impl FnOnce(&HashMap) -> ReqsignResult, ) -> ReqsignResult { self.try_load(backend, extract).await.inspect_err(|error| { - tracing::warn!( + tracing::debug!( "cannot use vended {backend} credentials for {}: {error}", self.location ) @@ -204,12 +206,12 @@ mod tests { true } - async fn load_credential(&self, path: &str) -> iceberg::Result { + async fn load_credential(&self, path: &str) -> iceberg::Result> { self.requested.lock().unwrap().push(path.to_string()); - Ok(StorageCredential::new( + Ok(Arc::new(StorageCredential::new( self.prefix, HashMap::from([(GCS_TOKEN.to_string(), "token".to_string())]), - )) + ))) } } @@ -298,7 +300,7 @@ mod tests { true } - async fn load_credential(&self, _path: &str) -> iceberg::Result { + async fn load_credential(&self, _path: &str) -> iceberg::Result> { Err(iceberg::Error::new( iceberg::ErrorKind::Unexpected, "catalog returned 503",