diff --git a/CHANGELOG.md b/CHANGELOG.md index 728b057e77..993588a9d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract. * `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. * `Transaction::commit` now returns `ErrorKind::PreconditionFailed` for read-only tables (`Table::readonly()`, e.g. from `StaticTable::into_table` or `TableBuilder::readonly(true)`), including empty transactions, which previously returned the table unchanged. Java treats an empty transaction on a static table as a no-op. +* `iceberg_storage_opendal::OpenDalStorage` and its variants are now `#[non_exhaustive]`, and the `S3` and `Gcs` variants have a new `credential_provider` field. Code outside the crate can no longer build a variant with a struct literal or match the enum exhaustively: build storages with `OpenDalStorageFactory` instead, and add `..` to variant patterns and a wildcard arm to `match` expressions. ## [v0.10.1] - 2026-07-28 diff --git a/Cargo.lock b/Cargo.lock index 2b057687b7..70ba7aaced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -530,6 +530,12 @@ dependencies = [ "syn", ] +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + [[package]] name = "atoi" version = "2.0.0" @@ -3093,9 +3099,11 @@ dependencies = [ "iceberg", "iceberg-property-macro", "iceberg_test_utils", + "mockito", "opendal", "reqsign-aws-v4", "reqsign-core", + "reqsign-google", "reqwest 0.12.28", "serde", "serde_json", @@ -5252,14 +5260,14 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.3.0" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" dependencies = [ "anyhow", + "asyncband", "base64 0.23.0", "bytes", - "futures", "hex", "hmac 0.13.0", "http 1.5.0", diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 5f87d98810..9216e69ed6 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -810,11 +810,13 @@ pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert: pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> pub fn iceberg::io::FileIO::deserialize_all(bytes: &[u8]) -> iceberg::Result pub async fn iceberg::io::FileIO::exists(&self, path: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::io::FileIO::has_credential_provider(&self) -> bool pub fn iceberg::io::FileIO::new_input(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_output(&self, path: impl core::convert::AsRef) -> iceberg::Result pub fn iceberg::io::FileIO::new_with_fs() -> Self pub fn iceberg::io::FileIO::new_with_memory() -> Self pub fn iceberg::io::FileIO::serialize_all(&self) -> iceberg::Result> +pub fn iceberg::io::FileIO::without_credential_provider(&self) -> Self impl core::clone::Clone for iceberg::io::FileIO pub fn iceberg::io::FileIO::clone(&self) -> iceberg::io::FileIO impl core::fmt::Debug for iceberg::io::FileIO @@ -824,6 +826,7 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_credential_provider(self, provider: alloc::sync::Arc) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder @@ -927,6 +930,7 @@ impl core::fmt::Debug for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::LocalFsStorageFactory @@ -965,6 +969,7 @@ impl core::fmt::Debug for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl serde_core::ser::Serialize for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::MemoryStorageFactory @@ -1077,6 +1082,21 @@ impl serde_core::ser::Serialize for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::StorageCredential +impl iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageCredential::covers(&self, location: &str) -> bool +pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::io::StorageCredential::prefix(&self) -> &str +pub fn iceberg::io::StorageCredential::select<'a>(credentials: impl core::iter::traits::collect::IntoIterator, location: &str) -> core::option::Option<&'a Self> +impl core::clone::Clone for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential +impl core::cmp::Eq for iceberg::io::StorageCredential +impl core::cmp::PartialEq for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::eq(&self, other: &iceberg::io::StorageCredential) -> bool +impl core::fmt::Debug for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential pub const iceberg::io::ADLS_ACCOUNT_KEY: &str pub const iceberg::io::ADLS_ACCOUNT_NAME: &str pub const iceberg::io::ADLS_AUTHORITY_HOST: &str @@ -1094,6 +1114,7 @@ pub const iceberg::io::GCS_NO_AUTH: &str pub const iceberg::io::GCS_PROJECT_ID: &str pub const iceberg::io::GCS_SERVICE_HOST: &str pub const iceberg::io::GCS_TOKEN: &str +pub const iceberg::io::GCS_TOKEN_EXPIRES_AT: &str pub const iceberg::io::GCS_USER_PROJECT: &str pub const iceberg::io::HDFS_HADOOP_CONF_PREFIX: &str pub const iceberg::io::HDFS_HOST: &str @@ -1117,6 +1138,7 @@ pub const iceberg::io::S3_PATH_STYLE_ACCESS: &str pub const iceberg::io::S3_REGION: &str pub const iceberg::io::S3_SECRET_ACCESS_KEY: &str pub const iceberg::io::S3_SESSION_TOKEN: &str +pub const iceberg::io::S3_SESSION_TOKEN_EXPIRES_AT_MS: &str pub const iceberg::io::S3_SSE_KEY: &str pub const iceberg::io::S3_SSE_MD5: &str pub const iceberg::io::S3_SSE_TYPE: &str @@ -1168,12 +1190,21 @@ pub fn iceberg::io::MemoryStorage::read<'life0, 'life1, 'async_trait>(&'life0 se pub fn iceberg::io::MemoryStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg::io::MemoryStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg::io::MemoryStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub trait iceberg::io::StorageCredentialProvider: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg::io::StorageCredentialProvider::factory(&self) -> iceberg::Result> +pub fn iceberg::io::StorageCredentialProvider::load_credential<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::StorageCredentialProvider::supports_path(&self, path: &str) -> bool +pub trait iceberg::io::StorageCredentialProviderFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize +pub fn iceberg::io::StorageCredentialProviderFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> pub trait iceberg::io::StorageFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize pub fn iceberg::io::StorageFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::StorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::LocalFsStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub fn iceberg::io::MemoryStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::StorageConfig, _credential_provider: core::option::Option>) -> iceberg::Result> pub mod iceberg::memory pub struct iceberg::memory::MemoryCatalog impl core::fmt::Debug for iceberg::memory::MemoryCatalog diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 42eec4db56..f1d73ea6bf 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -16,13 +16,14 @@ // under the License. use std::ops::Range; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Once, OnceLock}; use bytes::Bytes; use futures::{Stream, StreamExt}; use super::storage::{ - LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, + LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageCredentialProvider, + StorageCredentialProviderFactory, StorageFactory, }; use crate::Result; @@ -65,6 +66,8 @@ pub struct FileIO { config: StorageConfig, /// Factory for creating storage instances factory: Arc, + /// Optional provider of refreshable, backend-specific credentials + credential_provider: Option>, /// Cached storage instance (lazily initialized) storage: Arc>>, } @@ -80,15 +83,33 @@ mod _serde { pub(super) struct SerializableFileIO<'a> { pub(super) config: &'a StorageConfig, pub(super) factory: &'a Arc, + #[serde(skip_serializing_if = "Option::is_none")] + pub(super) credential_provider: Option, } #[derive(Deserialize)] pub(super) struct DeserializedFileIO { pub(super) config: StorageConfig, pub(super) factory: Arc, + /// Kept as JSON, so that a provider factory the receiving binary does + /// not link can be dropped instead of failing the whole `FileIO`. + #[serde(default)] + pub(super) credential_provider: Option, } } +/// Warns that a `FileIO` crossing a process boundary loses its credential +/// provider. Engines may serialize a `FileIO` per task, so each cause, tracked +/// by its own `warned`, is reported once. +fn warn_once_without_provider(warned: &'static Once, reason: impl FnOnce() -> String) { + warned.call_once(|| { + tracing::warn!( + "{}; its credentials will not be refreshed after deserialization", + reason() + ) + }); +} + impl FileIO { /// Create a new FileIO backed by in-memory storage. /// @@ -97,6 +118,7 @@ impl FileIO { Self { config: StorageConfig::new(), factory: Arc::new(MemoryStorageFactory), + credential_provider: None, storage: Arc::new(OnceLock::new()), } } @@ -108,6 +130,7 @@ impl FileIO { Self { config: StorageConfig::new(), factory: Arc::new(LocalFsStorageFactory), + credential_provider: None, storage: Arc::new(OnceLock::new()), } } @@ -123,14 +146,38 @@ impl FileIO { /// /// All storage configuration properties are included in the serialized representation. These /// properties may contain credentials or other sensitive values, so the returned bytes must be - /// protected in transit and at rest by the application embedding this crate. + /// protected in transit and at rest by the application embedding this crate. A serialized + /// credential provider may likewise carry catalog authentication and vended credentials. /// /// Storage factories are serialized through [`typetag`](https://docs.rs/typetag). Third-party /// factories must use `#[typetag::serde]` on their [`StorageFactory`] implementation. + /// + /// A credential provider is serialized as the + /// [`StorageCredentialProviderFactory`] returned by + /// [`StorageCredentialProvider::factory`], and rebuilt on deserialization. A provider that + /// cannot be rebuilt in another process is left out with a warning, as with + /// [`FileIO::without_credential_provider`]: the deserialized `FileIO` uses the credentials in + /// its configuration, which are not refreshed. Storage factories that hold their own + /// credential sources, such as a custom AWS credential loader, still fail to serialize, as + /// leaving them out would leave no credentials at all. pub fn serialize_all(&self) -> Result> { + let credential_provider = self.credential_provider.as_ref().and_then(|provider| { + provider + .factory() + .and_then(|factory| Ok(serde_json::to_value(factory)?)) + .inspect_err(|error| { + static WARNED: Once = Once::new(); + warn_once_without_provider(&WARNED, || { + format!("serializing FileIO without its credential provider: {error}") + }) + }) + .ok() + }); + Ok(serde_json::to_vec(&_serde::SerializableFileIO { config: &self.config, factory: &self.factory, + credential_provider, })?) } @@ -139,15 +186,81 @@ impl FileIO { /// The receiving binary must use a compatible crate version and link the concrete factory /// implementation so it is registered with `typetag`. Backend-specific requirements are /// documented by each storage factory implementation. + /// + /// A credential provider is rebuilt when the binary links its factory implementation, such as + /// a catalog's. Otherwise, or when rebuilding fails, the `FileIO` is deserialized without + /// it and logs a warning, as [`FileIO::serialize_all`] does. pub fn deserialize_all(bytes: &[u8]) -> Result { - let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; + let _serde::DeserializedFileIO { + config, + factory, + credential_provider, + } = serde_json::from_slice(bytes)?; + let credential_provider = credential_provider.and_then(|provider_factory| { + // The factory may hold credentials, and serde errors quote the + // offending value, so only its type is reported. + let factory_type = provider_factory + .get("type") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown") + .to_owned(); + match serde_json::from_value::>( + provider_factory, + ) { + Ok(provider_factory) => provider_factory + .build(&config) + .inspect_err(|error| { + static WARNED: Once = Once::new(); + warn_once_without_provider(&WARNED, || { + format!( + "deserializing FileIO without its credential provider, which \ + could not be rebuilt: {error}" + ) + }) + }) + .ok(), + Err(_) => { + static WARNED: Once = Once::new(); + warn_once_without_provider(&WARNED, || { + format!( + "deserializing FileIO without its credential provider, whose \ + factory {factory_type} is not linked or does not match this version" + ) + }); + None + } + } + }); Ok(Self { config, factory, + credential_provider, storage: Arc::new(OnceLock::new()), }) } + /// Returns whether this `FileIO` has a credential provider to refresh its storage credentials. + /// + /// A deserialized `FileIO` has none when its provider could not be serialized or rebuilt; see + /// [`FileIO::serialize_all`] and [`FileIO::deserialize_all`]. + pub fn has_credential_provider(&self) -> bool { + self.credential_provider.is_some() + } + + /// Returns a copy of this `FileIO` without its credential provider. + /// + /// The copy uses only the credentials in its storage configuration, which are not refreshed. + /// Use this to serialize a `FileIO` without its credential provider, even when the provider + /// could be rebuilt in another process. + pub fn without_credential_provider(&self) -> Self { + Self { + config: self.config.clone(), + factory: Arc::clone(&self.factory), + credential_provider: None, + storage: Arc::new(OnceLock::new()), + } + } + /// Get the storage configuration. pub fn config(&self) -> &StorageConfig { &self.config @@ -163,8 +276,11 @@ impl FileIO { return Ok(storage.clone()); } - // Build the storage - let storage = self.factory.build(&self.config)?; + // Build the storage, passing any credential provider so backends that + // support refreshable credentials can wire it into their operators. + let storage = self + .factory + .build_with_credential_provider(&self.config, self.credential_provider.clone())?; // Try to set it (another thread might have set it first) let _ = self.storage.set(storage.clone()); @@ -247,6 +363,8 @@ pub struct FileIOBuilder { factory: Arc, /// Storage configuration config: StorageConfig, + /// Optional provider of refreshable, backend-specific credentials + credential_provider: Option>, } impl FileIOBuilder { @@ -255,6 +373,7 @@ impl FileIOBuilder { Self { factory, config: StorageConfig::new(), + credential_provider: None, } } @@ -280,11 +399,25 @@ impl FileIOBuilder { &self.config } + /// Attach a provider of refreshable, backend-specific credentials. + /// + /// Storage factories that cannot use the provider ignore it and use the + /// credentials in the configuration; the default implementation logs a + /// warning when it does. + pub fn with_credential_provider( + mut self, + provider: Arc, + ) -> Self { + self.credential_provider = Some(provider); + self + } + /// Builds [`FileIO`]. pub fn build(self) -> FileIO { FileIO { config: self.config, factory: self.factory, + credential_provider: self.credential_provider, storage: Arc::new(OnceLock::new()), } } @@ -442,18 +575,80 @@ impl OutputFile { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::fs::{File, create_dir_all}; use std::io::Write; use std::path::Path; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use async_trait::async_trait; use bytes::Bytes; use futures::AsyncReadExt; use futures::io::AllowStdIo; + use serde::{Deserialize, Serialize}; use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; - use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; + use crate::Result; + use crate::io::{ + GCS_TOKEN, LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, + StorageCredential, StorageCredentialProvider, StorageCredentialProviderFactory, + StorageFactory, + }; + + #[derive(Debug)] + struct TestCredentialProvider; + + #[async_trait] + impl StorageCredentialProvider for TestCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + unreachable!("unsupported factories must ignore the provider") + } + } + + /// A provider rebuilt from the `FileIO` configuration, like a catalog provider. + #[derive(Debug)] + struct PortableCredentialProvider { + endpoint: Option, + } + + #[async_trait] + impl StorageCredentialProvider for PortableCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(StorageCredential::new( + "gs", + HashMap::from([( + GCS_TOKEN.to_string(), + self.endpoint.clone().unwrap_or_default(), + )]), + ))) + } + + fn factory(&self) -> Result> { + Ok(Arc::new(PortableCredentialProviderFactory)) + } + } + + #[derive(Debug, Serialize, Deserialize)] + struct PortableCredentialProviderFactory; + + #[typetag::serde] + impl StorageCredentialProviderFactory for PortableCredentialProviderFactory { + fn build(&self, config: &StorageConfig) -> Result> { + Ok(Arc::new(PortableCredentialProvider { + endpoint: config.get("endpoint").cloned(), + })) + } + } fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -601,6 +796,182 @@ mod tests { assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + /// Memory storage that records whether it was given a credential provider. + #[derive(Debug, Default, Serialize, Deserialize)] + struct ProviderRecordingFactory { + #[serde(skip)] + received_provider: Arc, + } + + #[typetag::serde] + impl StorageFactory for ProviderRecordingFactory { + fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + self.received_provider + .store(credential_provider.is_some(), Ordering::SeqCst); + MemoryStorageFactory.build(config) + } + } + + #[tokio::test] + async fn test_file_io_passes_credential_provider_to_factory() { + let received_provider = Arc::new(AtomicBool::new(false)); + let file_io = FileIOBuilder::new(Arc::new(ProviderRecordingFactory { + received_provider: Arc::clone(&received_provider), + })) + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + file_io.exists("memory://file").await.unwrap(); + assert!(received_provider.load(Ordering::SeqCst)); + } + + /// Memory storage from a factory that implements only `build`, so it uses + /// the default `build_with_credential_provider`. + #[derive(Debug, Serialize, Deserialize)] + struct BuildOnlyFactory; + + #[typetag::serde] + impl StorageFactory for BuildOnlyFactory { + fn build(&self, config: &StorageConfig) -> Result> { + MemoryStorageFactory.build(config) + } + } + + #[tokio::test] + async fn test_file_io_ignores_credentials_for_unsupported_factory() { + let file_io = FileIOBuilder::new(Arc::new(BuildOnlyFactory)) + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + file_io + .new_output("memory://file") + .unwrap() + .write("data".into()) + .await + .unwrap(); + assert!(file_io.exists("memory://file").await.unwrap()); + } + + #[test] + fn test_file_io_drops_unserializable_credential_provider() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("s3.access-key-id", "static-key") + .with_credential_provider(Arc::new(TestCredentialProvider)) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(!deserialized.has_credential_provider()); + assert_eq!( + deserialized.config().get("s3.access-key-id"), + Some(&"static-key".to_string()) + ); + assert_eq!( + file_io.serialize_all().unwrap(), + file_io + .without_credential_provider() + .serialize_all() + .unwrap() + ); + } + + /// A provider whose factory cannot be serialized. + #[derive(Debug)] + struct UnserializableFactoryProvider; + + #[async_trait] + impl StorageCredentialProvider for UnserializableFactoryProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + unreachable!("serialization tests never load credentials") + } + + fn factory(&self) -> Result> { + Ok(Arc::new(UnserializableFactory)) + } + } + + #[derive(Debug, Deserialize)] + struct UnserializableFactory; + + impl Serialize for UnserializableFactory { + fn serialize( + &self, + _serializer: S, + ) -> std::result::Result { + Err(serde::ser::Error::custom("cannot serialize")) + } + } + + #[typetag::serde] + impl StorageCredentialProviderFactory for UnserializableFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + unreachable!("never serialized") + } + } + + #[test] + fn test_file_io_drops_provider_whose_factory_fails_to_serialize() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_credential_provider(Arc::new(UnserializableFactoryProvider)) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(!deserialized.has_credential_provider()); + } + + #[test] + fn test_file_io_deserializes_without_unknown_credential_provider() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("s3.access-key-id", "static-key") + .with_credential_provider(Arc::new(PortableCredentialProvider { endpoint: None })) + .build(); + // A receiving binary that does not link the provider's factory. + let mut serialized: serde_json::Value = + serde_json::from_slice(&file_io.serialize_all().unwrap()).unwrap(); + serialized["credential_provider"]["type"] = "UnlinkedProviderFactory".into(); + + let deserialized = + FileIO::deserialize_all(&serde_json::to_vec(&serialized).unwrap()).unwrap(); + assert!(!deserialized.has_credential_provider()); + assert_eq!( + deserialized.config().get("s3.access-key-id"), + Some(&"static-key".to_string()) + ); + } + + #[tokio::test] + async fn test_file_io_rebuilds_credential_provider_after_serialization() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("endpoint", "https://catalog/credentials") + .with_credential_provider(Arc::new(PortableCredentialProvider { endpoint: None })) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + assert!(deserialized.has_credential_provider()); + let credential = deserialized + .credential_provider + .unwrap() + .load_credential("gs://bucket/file") + .await + .unwrap(); + // The provider is rebuilt from the deserialized configuration. + assert_eq!( + credential.config().get(GCS_TOKEN).map(String::as_str), + Some("https://catalog/credentials") + ); + } + #[tokio::test] async fn test_memory_file_io_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs index 99867e5dfc..a21faaaa1d 100644 --- a/crates/iceberg/src/io/storage/config/gcs.rs +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -41,6 +41,8 @@ pub const GCS_NO_AUTH: &str = "gcs.no-auth"; pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; /// Google Cloud Storage token. pub const GCS_TOKEN: &str = "gcs.oauth2.token"; +/// Epoch-millisecond timestamp at which the vended GCS OAuth2 token expires. +pub const GCS_TOKEN_EXPIRES_AT: &str = "gcs.oauth2.token-expires-at"; /// Option to skip signing requests (e.g. for public buckets/folders). pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; /// Option to skip loading the credential from GCE metadata server. diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs index ae3bc4ba3b..71ca401d7f 100644 --- a/crates/iceberg/src/io/storage/config/s3.rs +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -36,6 +36,8 @@ pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id"; pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key"; /// S3 session token (required when using temporary credentials). pub const S3_SESSION_TOKEN: &str = "s3.session-token"; +/// Epoch-millisecond timestamp at which the vended S3 session token expires. +pub const S3_SESSION_TOKEN_EXPIRES_AT_MS: &str = "s3.session-token-expires-at-ms"; /// S3 region. pub const S3_REGION: &str = "s3.region"; /// Region to use for the S3 client (takes precedence over [`S3_REGION`]). diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index cb780e86e7..f65d8230e0 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -36,7 +36,7 @@ use serde::{Deserialize, Serialize}; use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use crate::{Error, ErrorKind, Result}; @@ -318,6 +318,15 @@ impl StorageFactory for LocalFsStorageFactory { fn build(&self, _config: &StorageConfig) -> Result> { Ok(Arc::new(LocalFsStorage::new())) } + + /// Local filesystem storage needs no credentials. + fn build_with_credential_provider( + &self, + config: &StorageConfig, + _credential_provider: Option>, + ) -> Result> { + self.build(config) + } } #[cfg(test)] diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index fefa7973e8..7a151fb3c4 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -35,7 +35,7 @@ use serde::{Deserialize, Serialize}; use crate::error::invalid_data; use crate::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use crate::{Error, ErrorKind, Result}; @@ -242,6 +242,15 @@ impl StorageFactory for MemoryStorageFactory { fn build(&self, _config: &StorageConfig) -> Result> { Ok(Arc::new(MemoryStorage::new())) } + + /// In-memory storage needs no credentials. + fn build_with_credential_provider( + &self, + config: &StorageConfig, + _credential_provider: Option>, + ) -> Result> { + self.build(config) + } } /// File reader for in-memory storage. diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index 5276c7771f..4ddefc3c7f 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -21,8 +21,9 @@ mod config; mod local_fs; mod memory; +use std::collections::{BTreeSet, HashMap}; use std::fmt::Debug; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use async_trait::async_trait; use bytes::Bytes; @@ -32,7 +33,7 @@ pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; pub use memory::{MemoryStorage, MemoryStorageFactory}; use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; -use crate::Result; +use crate::{Error, ErrorKind, Result}; /// Trait for storage operations in Iceberg. /// @@ -139,4 +140,250 @@ pub trait StorageFactory: Debug + Send + Sync { /// A `Result` containing an `Arc` on success, or an error /// if the storage could not be created. fn build(&self, config: &StorageConfig) -> Result>; + + /// Build a new Storage instance, optionally supplying a credential provider + /// that the backend can call to obtain and refresh short-lived credentials. + /// + /// Backends that cannot use the provider ignore it and use the credentials + /// in `config`, as they would without one. The default does so and logs a + /// warning once per factory type, since the storage's credentials are then + /// not refreshed. Factories that wrap another factory should forward the + /// provider to it, and factories whose storage needs no credentials can + /// ignore it silently. + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + if credential_provider.is_some() { + // A static in a provided method is shared by every implementor. + static WARNED: Mutex> = Mutex::new(BTreeSet::new()); + let factory = std::any::type_name::(); + if WARNED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(factory) + { + tracing::warn!( + "{factory} ignores storage credential providers; its storage uses the \ + credentials in its configuration, which are not refreshed" + ); + } + } + self.build(config) + } +} + +/// Supplies fresh, backend-specific storage credentials on demand. +/// +/// A catalog that vends temporary credentials implements this trait so that +/// storage backends can re-fetch credentials as they approach expiry instead +/// of failing once the initial token's TTL runs out. +/// +/// # Debug output +/// +/// [`FileIO`](crate::io::FileIO) and storage implementations print the +/// provider in their `Debug` output, so the provider's `Debug` implementation +/// must not expose credentials or other secrets. +/// +/// # Caching +/// +/// [`load_credential`](Self::load_credential) may be called very frequently. +/// Implementations must cache internally and only re-fetch when the current +/// credential is at or near expiry; otherwise every object-store request could +/// trigger a call back to the catalog. Returning the cached [`Arc`] keeps each +/// call from copying the credential. +/// +/// Backends may treat a credential as stale before it expires, and load +/// another on every request until they get a fresher one. The OpenDAL S3 and +/// GCS backends do so within two minutes of expiry, and reject a newly loaded +/// credential that expires within ten seconds. Implementations should +/// therefore refresh early enough that the credential they return has more +/// than two minutes left. +#[async_trait] +pub trait StorageCredentialProvider: Debug + Send + Sync { + /// Return whether this provider supplies credentials for `path`. + /// + /// Backends replace their own credential chain with the provider only for + /// paths it supports, so a provider must return `false` for the storage it + /// does not serve, such as other schemes behind a resolving storage. + fn supports_path(&self, path: &str) -> bool; + + /// Load a fresh credential for the storage location identified by `path`. + /// + /// `path` is an absolute location: usually the file being accessed, such + /// as `s3://bucket/warehouse/db/table/data/file.parquet`, but for a bulk + /// delete the location shared by a batch, which may be a storage root + /// (`s3://bucket/`) or the prefix of a previously returned credential + /// (`s3://bucket/warehouse/db/table`). Providers that vend distinct + /// credentials per location prefix use it to select the most specific + /// match with [`StorageCredential::select`], which must + /// [cover](StorageCredential::covers) `path`. + /// + /// Backends read the credential's expiry from its config, such as + /// `s3.session-token-expires-at-ms`, and load a new one before it expires. + /// A credential without an expiry is used for as long as the backend lives. + /// + /// An error, or a credential that does not cover `path`, fails the request: + /// backends do not fall back to other credentials. Backends log such + /// failures only at debug level, since they recur on every request while a + /// provider is failing, so implementations should log their own, such as + /// once per failed refresh. + async fn load_credential(&self, path: &str) -> Result>; + + /// Return a factory that rebuilds an equivalent provider in another process. + /// + /// [`FileIO::serialize_all`](crate::io::FileIO::serialize_all) serializes this + /// factory in place of the provider. On an error, which the default always + /// returns, it serializes the `FileIO` without the provider and logs a + /// warning. + fn factory(&self) -> Result> { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "storage credential provider cannot be serialized", + )) + } +} + +/// Serializable recipe that rebuilds a [`StorageCredentialProvider`] after +/// [`FileIO`](crate::io::FileIO) deserialization. +/// +/// Factories are serialized through [`typetag`](https://docs.rs/typetag), so +/// implementations must use `#[typetag::serde]`, and the receiving binary must +/// link the concrete implementation. +#[typetag::serde(tag = "type")] +pub trait StorageCredentialProviderFactory: Debug + Send + Sync { + /// Build a provider for a `FileIO` with the given storage configuration. + fn build(&self, config: &StorageConfig) -> Result>; +} + +/// A vended storage credential: the storage properties that apply to +/// locations under a prefix, like Java's `StorageCredential`. +/// +/// `config` holds backend storage properties, such as `s3.access-key-id`, +/// `s3.secret-access-key`, `s3.session-token` and +/// `s3.session-token-expires-at-ms`. Backends read their credential and its +/// expiry from it. +#[derive(Clone, PartialEq, Eq)] +pub struct StorageCredential { + /// Storage-location prefix this credential applies to, such as + /// `s3://bucket/table` or, for a whole scheme, `s3`. + prefix: String, + /// Backend storage properties holding the credential. + config: HashMap, +} + +impl StorageCredential { + /// Create a credential for locations under `prefix`. + /// + /// An empty prefix covers no location; Java rejects one outright. + pub fn new(prefix: impl Into, config: HashMap) -> Self { + Self { + prefix: prefix.into(), + config, + } + } + + /// Return the storage-location prefix this credential applies to. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// Return the storage properties holding the credential. + pub fn config(&self) -> &HashMap { + &self.config + } + + /// Return whether this credential applies to `location`. + /// + /// As in Java, `location` must start with the non-empty prefix, compared + /// as plain strings: `s3` covers every `s3://`, `s3a://` and `s3n://` + /// location, and `s3://bucket/tab` covers `s3://bucket/table`. + pub fn covers(&self, location: &str) -> bool { + !self.prefix.is_empty() && location.starts_with(&self.prefix) + } + + /// Return the credential with the longest prefix that covers `location`. + pub fn select<'a>( + credentials: impl IntoIterator, + location: &str, + ) -> Option<&'a Self> { + credentials + .into_iter() + .filter(|credential| credential.covers(location)) + .max_by_key(|credential| credential.prefix.len()) + } +} + +impl Debug for StorageCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Config values are secrets. Keys are sorted for stable output. + let config_keys: BTreeSet<_> = self.config.keys().collect(); + f.debug_struct("StorageCredential") + .field("prefix", &self.prefix) + .field("config_keys", &config_keys) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scoped(prefix: &str) -> StorageCredential { + StorageCredential::new( + prefix, + HashMap::from([("k".to_string(), "secret".to_string())]), + ) + } + + #[test] + fn test_credential_prefix_matches_like_java() { + let credential = scoped("s3://bucket/table"); + assert!(credential.covers("s3://bucket/table")); + assert!(credential.covers("s3://bucket/table/data/file.parquet")); + assert!(credential.covers("s3://bucket/table2/file.parquet")); + assert!(!credential.covers("s3://bucket/tab")); + assert!(!credential.covers("s3a://bucket/table/file.parquet")); + + // A scheme prefix covers every location with that scheme, including + // `s3a` and `s3n`. + assert!(scoped("s3").covers("s3a://bucket/file.parquet")); + assert!(!scoped("s3").covers("gs://bucket/file.parquet")); + assert!(!scoped("").covers("s3://bucket/file.parquet")); + } + + #[test] + fn test_select_prefers_the_longest_covering_prefix() { + let credentials = [ + scoped("s3"), + scoped("s3://bucket/table"), + scoped("s3://bucket"), + scoped("s3://other"), + ]; + let select = |location| { + StorageCredential::select(&credentials, location).map(StorageCredential::prefix) + }; + + assert_eq!(select("s3://bucket/table/file"), Some("s3://bucket/table")); + assert_eq!(select("s3://bucket/other/file"), Some("s3://bucket")); + assert_eq!(select("s3a://bucket/table/file"), Some("s3")); + assert_eq!(select("gs://bucket/table/file"), None); + } + + #[test] + fn test_credential_debug_omits_config_values() { + let debug = format!("{:?}", scoped("s3://bucket")); + assert!(debug.contains("s3://bucket"), "{debug}"); + assert!(!debug.contains("secret"), "{debug}"); + + let credential = StorageCredential::new( + "s3", + HashMap::from([ + ("b".to_string(), String::new()), + ("a".to_string(), String::new()), + ]), + ); + assert!(format!("{credential:?}").contains(r#"{"a", "b"}"#)); + } } diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 83541e89c5..ec2d1b01a4 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -41,7 +41,12 @@ opendal-all = [ opendal-azdls = ["opendal/services-azdls"] opendal-fs = ["opendal/services-fs"] -opendal-gcs = ["opendal/services-gcs"] +opendal-gcs = [ + "opendal/services-gcs", + "reqsign-google", + "reqsign-core", + "dep:tracing", +] opendal-hdfs-native = [ "opendal/services-hdfs-native", "dep:tokio", @@ -50,7 +55,12 @@ opendal-hdfs-native = [ opendal-hf = ["opendal/services-hf"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] -opendal-s3 = ["opendal/services-s3", "reqsign-aws-v4", "reqsign-core"] +opendal-s3 = [ + "opendal/services-s3", + "reqsign-aws-v4", + "reqsign-core", + "dep:tracing", +] [dependencies] anyhow = { workspace = true } @@ -62,7 +72,8 @@ iceberg = { workspace = true } iceberg-property-macro = { workspace = true } opendal = { workspace = true } reqsign-aws-v4 = { version = "3.0.0", optional = true } -reqsign-core = { version = "3.0.0", optional = true } +reqsign-core = { version = "3.3.1", optional = true } +reqsign-google = { version = "3.0.4", optional = true } serde = { workspace = true } tokio = { workspace = true, optional = true } tracing = { workspace = true, optional = true } @@ -72,6 +83,7 @@ url = { workspace = true } [dev-dependencies] async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } +mockito = { workspace = true } reqwest = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 797d90d1e8..d729bb8b6d 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -1,28 +1,30 @@ pub mod iceberg_storage_opendal pub use iceberg_storage_opendal::AwsCredential pub use iceberg_storage_opendal::ProvideCredential -pub enum iceberg_storage_opendal::OpenDalStorage -pub iceberg_storage_opendal::OpenDalStorage::Azdls +#[non_exhaustive] pub enum iceberg_storage_opendal::OpenDalStorage +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Azdls pub iceberg_storage_opendal::OpenDalStorage::Azdls::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::Gcs +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative(iceberg_storage_opendal::HdfsNativeStorage) -pub iceberg_storage_opendal::OpenDalStorage::Hf +pub iceberg_storage_opendal::OpenDalStorage::Gcs::credential_provider: core::option::Option> +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::HdfsNative(iceberg_storage_opendal::HdfsNativeStorage) +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::LocalFs +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::LocalFs pub iceberg_storage_opendal::OpenDalStorage::LocalFs::client_config: iceberg_storage_opendal::OpenDalClientConfig -pub iceberg_storage_opendal::OpenDalStorage::Memory +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Memory pub iceberg_storage_opendal::OpenDalStorage::Memory::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Memory::operator: opendal_core::types::operator::operator::Operator -pub iceberg_storage_opendal::OpenDalStorage::Oss +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::Oss pub iceberg_storage_opendal::OpenDalStorage::Oss::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Oss::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::S3 +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::S3 pub iceberg_storage_opendal::OpenDalStorage::S3::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::S3::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::S3::credential_provider: core::option::Option> pub iceberg_storage_opendal::OpenDalStorage::S3::customized_credential_load: core::option::Option impl core::clone::Clone for iceberg_storage_opendal::OpenDalStorage pub fn iceberg_storage_opendal::OpenDalStorage::clone(&self) -> iceberg_storage_opendal::OpenDalStorage @@ -60,6 +62,7 @@ impl core::fmt::Debug for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +pub fn iceberg_storage_opendal::OpenDalStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::storage::config::StorageConfig, credential_provider: core::option::Option>) -> iceberg::error::Result> impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalStorageFactory pub fn iceberg_storage_opendal::OpenDalStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalStorageFactory @@ -124,6 +127,7 @@ impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorageFactor pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::build_with_credential_provider(&self, config: &iceberg::io::storage::config::StorageConfig, credential_provider: core::option::Option>) -> iceberg::error::Result> impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalResolvingStorageFactory diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index a00282cc22..73d241af28 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -17,17 +17,23 @@ //! Google Cloud Storage properties use std::collections::HashMap; +use std::sync::Arc; use iceberg::io::{ GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD, GCS_DISABLE_VM_METADATA, - GCS_NO_AUTH, GCS_SERVICE_HOST, GCS_TOKEN, + GCS_NO_AUTH, GCS_SERVICE_HOST, GCS_TOKEN, GCS_TOKEN_EXPIRES_AT, StorageCredentialProvider, }; use iceberg::{Error, ErrorKind, Result}; -use opendal::Operator; use opendal::services::GcsConfig; +use opendal::{Configurator, Operator}; +use reqsign_core::{Context, ProvideCredential, Result as ReqsignResult}; +use reqsign_google::{Credential as GoogleCredential, Token as GoogleToken}; use url::Url; -use crate::utils::{from_opendal_error, is_truthy}; +use crate::utils::{ + VendedCredentialSource, credential_expiry, from_opendal_error, is_truthy, + required_credential_property, +}; /// Parse iceberg properties to [`GcsConfig`]. pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result { @@ -71,7 +77,12 @@ pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result Result { +pub(crate) fn gcs_config_build( + cfg: &GcsConfig, + credential_provider: &Option>, + path: &str, + credential_location: Option<&str>, +) -> Result { let url = Url::parse(path)?; let bucket = url.host_str().ok_or_else(|| { Error::new( @@ -82,5 +93,155 @@ pub(crate) fn gcs_config_build(cfg: &GcsConfig, path: &str) -> Result let mut cfg = cfg.clone(); cfg.bucket = bucket.to_string(); - Operator::from_config(cfg).map_err(from_opendal_error) + + let credential_provider = credential_provider + .as_ref() + .filter(|provider| provider.supports_path(path)); + if credential_provider.is_some() { + if cfg.skip_signature { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid GCS auth settings: anonymous access cannot be combined with refreshable credentials", + )); + } + disable_other_credential_sources(&mut cfg); + } + + let mut builder = cfg.into_builder(); + + // The provider re-fetches the vended OAuth2 token as it nears expiry. + if let Some(provider) = credential_provider { + builder = + builder.credential_provider(VendedGcsCredentialProvider(VendedCredentialSource::new( + Arc::clone(provider), + credential_location.unwrap_or(path).to_string(), + ))); + } + + Operator::new(builder).map_err(from_opendal_error) +} + +/// Disable every configured and ambient credential source but a vended one. +/// +/// `reqsign_google` continues to the next provider even when a provider returns +/// an error, and OpenDAL prepends custom providers to its default chain. With +/// the other sources disabled, the vended provider is effectively the sole +/// source, and refresh failures cannot silently fall back. +fn disable_other_credential_sources(cfg: &mut GcsConfig) { + cfg.token = None; + cfg.credential = None; + cfg.credential_path = None; + cfg.service_account = None; + cfg.disable_vm_metadata = true; + cfg.disable_config_load = true; +} + +/// Adapts a generic [`StorageCredentialProvider`] into a `reqsign` +/// [`ProvideCredential`], so the GCS signer can obtain and refresh vended OAuth2 +/// tokens. +#[derive(Debug)] +struct VendedGcsCredentialProvider(VendedCredentialSource); + +impl ProvideCredential for VendedGcsCredentialProvider { + type Credential = GoogleCredential; + + async fn provide_credential(&self, _ctx: &Context) -> ReqsignResult> { + let token = self + .0 + .load("GCS", |config| { + Ok(GoogleToken { + access_token: required_credential_property(config, GCS_TOKEN)?.to_string(), + expires_at: credential_expiry(config, GCS_TOKEN_EXPIRES_AT)?, + }) + }) + .await?; + Ok(Some(GoogleCredential::with_token(token))) + } +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use iceberg::io::{S3_ACCESS_KEY_ID, StorageCredential}; + use reqsign_core::time::Timestamp; + + use super::*; + + #[derive(Debug)] + struct FixedProvider(StorageCredential); + + #[async_trait] + impl StorageCredentialProvider for FixedProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(self.0.clone())) + } + } + + fn adapter(credential: StorageCredential) -> VendedGcsCredentialProvider { + VendedGcsCredentialProvider(VendedCredentialSource::new( + Arc::new(FixedProvider(credential)), + "gs://bucket/table/file.parquet".to_string(), + )) + } + + fn credential(config: &[(&str, &str)]) -> StorageCredential { + StorageCredential::new( + "gs://bucket/table", + config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ) + } + + #[tokio::test] + async fn test_vended_adapter_returns_expiring_oauth2_token() { + let credential = credential(&[(GCS_TOKEN, "ya29.token"), (GCS_TOKEN_EXPIRES_AT, "1500")]); + + let google = adapter(credential) + .provide_credential(&Context::new()) + .await + .unwrap() + .unwrap(); + let token = google.token.expect("a token credential"); + assert_eq!(token.access_token, "ya29.token"); + assert_eq!( + token.expires_at, + Some(Timestamp::from_millisecond(1500).unwrap()) + ); + } + + #[tokio::test] + async fn test_vended_adapter_rejects_other_credentials() { + assert!( + adapter(credential(&[(S3_ACCESS_KEY_ID, "AK")])) + .provide_credential(&Context::new()) + .await + .is_err() + ); + } + + #[test] + fn test_vended_credentials_disable_every_other_source() { + let mut cfg = gcs_config_parse(HashMap::from([ + (GCS_TOKEN.to_string(), "static-token".to_string()), + (GCS_CREDENTIALS_JSON.to_string(), "e30=".to_string()), + ])) + .unwrap(); + cfg.credential_path = Some("/path/to/credentials.json".to_string()); + cfg.service_account = Some("account@example.com".to_string()); + + disable_other_credential_sources(&mut cfg); + + assert_eq!(cfg.token, None); + assert_eq!(cfg.credential, None); + assert_eq!(cfg.credential_path, None); + assert_eq!(cfg.service_account, None); + assert!(cfg.disable_vm_metadata); + assert!(cfg.disable_config_load); + } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 693f4c18b3..ff67f9ef04 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -37,7 +37,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use iceberg_property_macro::Properties; @@ -221,10 +221,39 @@ where )) } +/// Fails serialization of a storage holding a credential provider, which would +/// otherwise silently fall back to the configured or ambient credentials. +/// [`FileIO`](iceberg::io::FileIO) serializes its provider separately. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn serialize_credential_provider( + _provider: &Option>, + _serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + Err(serde::ser::Error::custom( + "storage with a credential provider cannot be serialized; serialize its FileIO instead", + )) +} + #[typetag::serde(name = "OpenDalStorageFactory")] impl StorageFactory for OpenDalStorageFactory { - #[allow(unused_variables)] fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + #[cfg_attr( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + allow(unused_variables) + )] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + // Only S3 and GCS consume the provider; other backends use the + // credentials in `config`. let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] @@ -240,11 +269,13 @@ impl StorageFactory for OpenDalStorageFactory { } => Ok(Arc::new(OpenDalStorage::S3 { config: s3_config_parse(config.props().clone())?.into(), customized_credential_load: customized_credential_load.clone(), + credential_provider, client_config, })), #[cfg(feature = "opendal-gcs")] OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), + credential_provider, client_config, })), #[cfg(feature = "opendal-hdfs-native")] @@ -297,8 +328,10 @@ fn default_memory_operator() -> Operator { /// /// The serialized representation is not a stable format and may change between crate versions. #[derive(Clone, Debug, Serialize, Deserialize)] +#[non_exhaustive] pub enum OpenDalStorage { /// Memory storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-memory")] Memory { /// Pre-built memory operator. @@ -309,6 +342,7 @@ pub enum OpenDalStorage { client_config: OpenDalClientConfig, }, /// Local filesystem storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-fs")] LocalFs { /// Backend-independent client settings. @@ -319,6 +353,7 @@ pub enum OpenDalStorage { /// /// Accepts any S3-family URL (`s3://`, `s3a://`, `s3n://`); the scheme is /// derived from the path at call time. + #[non_exhaustive] #[cfg(feature = "opendal-s3")] S3 { /// S3 configuration. @@ -326,23 +361,40 @@ pub enum OpenDalStorage { /// Custom AWS credential loader. #[serde(skip)] customized_credential_load: Option, + /// Provider of refreshable vended credentials, from the `FileIO`. + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_credential_provider" + )] + credential_provider: Option>, /// Backend-independent client settings. #[serde(default)] client_config: OpenDalClientConfig, }, /// GCS storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-gcs")] Gcs { /// GCS configuration. config: Arc, + /// Provider of refreshable vended credentials, from the `FileIO`. + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_credential_provider" + )] + credential_provider: Option>, /// Backend-independent client settings. #[serde(default)] client_config: OpenDalClientConfig, }, /// HDFS storage variant; see [`HdfsNativeStorage`]. + #[non_exhaustive] #[cfg(feature = "opendal-hdfs-native")] HdfsNative(HdfsNativeStorage), /// OSS storage variant. + #[non_exhaustive] #[cfg(feature = "opendal-oss")] Oss { /// OSS configuration. @@ -357,6 +409,7 @@ pub enum OpenDalStorage { /// `abfs[s]://@.dfs./` or /// `wasb[s]://@.blob./`. /// The scheme is derived from the path at call time. + #[non_exhaustive] #[cfg(feature = "opendal-azdls")] Azdls { /// Azure DLS configuration. @@ -370,6 +423,7 @@ pub enum OpenDalStorage { /// Accepts paths of the form /// `hf:////[@]/`, /// where `` must be one of `models`, `datasets`, `spaces`, or `buckets`. + #[non_exhaustive] #[cfg(feature = "opendal-hf")] Hf { /// HuggingFace Hub configuration (token + endpoint). @@ -380,6 +434,62 @@ pub enum OpenDalStorage { }, } +/// Groups bulk deletes by operator and credential scope. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct DeleteBatchKey { + storage: String, + credential: BatchCredential, +} + +impl DeleteBatchKey { + /// Open batches to flush before adding a path with this key. + /// + /// A scoped credential inside the scope of an open batch means the provider + /// is replacing that batch's broader credential, such as a scheme-wide one. + /// Such batches are flushed while their credential is still valid: once it + /// expires, nothing may cover their whole scope. + fn superseded_batches<'a>( + &self, + open: impl Iterator, + ) -> Vec { + let BatchCredential::Scoped { prefix } = &self.credential else { + return Vec::new(); + }; + open.filter(|key| { + key.storage == self.storage + && key + .credential + .location() + .is_some_and(|location| location != prefix && prefix.starts_with(location)) + }) + .cloned() + .collect() + } +} + +/// The credential shared by the paths of a bulk-delete batch. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum BatchCredential { + /// Served by the backend's configured credentials. + Static, + /// A vended credential for a whole scheme, such as `s3`, looked up for + /// the storage root. + SchemeWide { root: String }, + /// A vended credential scoped to `prefix`, looked up for the prefix. + Scoped { prefix: String }, +} + +impl BatchCredential { + /// Location the batch operator looks vended credentials up for. + fn location(&self) -> Option<&str> { + match self { + BatchCredential::Static => None, + BatchCredential::SchemeWide { root } => Some(root), + BatchCredential::Scoped { prefix } => Some(prefix), + } + } +} + impl OpenDalStorage { /// Creates operator from path. /// @@ -393,10 +503,23 @@ impl OpenDalStorage { /// /// * An [`opendal::Operator`] instance used to operate on file. /// * Relative path to the root uri of [`opendal::Operator`]. - #[allow(unreachable_code, unused_variables)] pub(crate) fn create_operator<'a>( &self, path: &'a impl AsRef, + ) -> Result<(Operator, &'a str)> { + self.create_operator_with_credential_location(path, None) + } + + /// Creates an operator whose vended credentials are looked up for + /// `credential_location` instead of `path`. + /// + /// A bulk-delete batch passes its scope location, so every credential the + /// operator loads covers all paths in the batch. + #[allow(unreachable_code, unreachable_patterns, unused_variables)] + fn create_operator_with_credential_location<'a>( + &self, + path: &'a impl AsRef, + credential_location: Option<&str>, ) -> Result<(Operator, &'a str)> { let path = path.as_ref(); let (operator, relative_path): (Operator, &str) = match self { @@ -421,9 +544,16 @@ impl OpenDalStorage { OpenDalStorage::S3 { config, customized_credential_load, + credential_provider, .. } => { - let op = s3_config_build(config, customized_credential_load, path)?; + let op = s3_config_build( + config, + customized_credential_load, + credential_provider, + path, + credential_location, + )?; let op_info = op.info(); // Use the URL scheme in the path for prefix matching. This enables @@ -445,8 +575,13 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-gcs")] - OpenDalStorage::Gcs { config, .. } => { - let operator = gcs_config_build(config, path)?; + OpenDalStorage::Gcs { + config, + credential_provider, + .. + } => { + let operator = + gcs_config_build(config, credential_provider, path, credential_location)?; let prefix = format!("gs://{}/", operator.info().name()); if path.starts_with(&prefix) { (operator, &path[prefix.len()..]) @@ -547,6 +682,7 @@ impl OpenDalStorage { /// /// For most backends the URL host (bucket name) is sufficient. For HF the host /// encodes the repo type, not the repo identity, so a more specific key is used. + #[allow(unreachable_patterns)] fn batch_key_for_path(&self, path: &str) -> String { match self { #[cfg(feature = "opendal-hf")] @@ -562,12 +698,69 @@ impl OpenDalStorage { } } + /// Return the credential provider that serves `path`, if any. + fn credential_provider_for_path( + &self, + path: &str, + ) -> Option<&Arc> { + let provider: &Arc = (match self { + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + customized_credential_load: None, + credential_provider: Some(provider), + .. + } => Some(provider), + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { + credential_provider: Some(provider), + .. + } => Some(provider), + _ => None, + })?; + provider.supports_path(path).then_some(provider) + } + + /// Returns a key that keeps bulk deletes within one operator and credential + /// scope. Loading the credential is normally a cache hit and avoids rebuilding + /// an operator for every path while preventing a batch from crossing prefixes. + async fn delete_batch_key_for_path(&self, path: &str) -> Result { + let credential = match self.credential_provider_for_path(path) { + Some(provider) => { + let credential = provider.load_credential(path).await?; + if !credential.covers(path) { + return Err(Error::new( + ErrorKind::DataInvalid, + utils::uncovered_location_message(path, &credential), + )); + } + let prefix = credential.prefix(); + if prefix.contains("://") { + BatchCredential::Scoped { + prefix: prefix.to_string(), + } + } else { + // A scheme-wide prefix such as `s3` is no location to look + // up, so the batch looks its credential up for the root. + BatchCredential::SchemeWide { + root: utils::storage_root(path)?, + } + } + } + None => BatchCredential::Static, + }; + + Ok(DeleteBatchKey { + storage: self.batch_key_for_path(path), + credential, + }) + } + /// Extracts the relative path from an absolute path without building an operator. /// /// This is a lightweight alternative to [`create_operator`](Self::create_operator) for cases /// where only the relative path is needed (e.g. bulk deletes where the operator is already /// available). - #[allow(unreachable_code, unused_variables)] + #[allow(unreachable_code, unreachable_patterns, unused_variables)] pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> { match self { #[cfg(feature = "opendal-memory")] @@ -735,17 +928,26 @@ impl Storage for OpenDalStorage { } async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { - let mut deleters: HashMap = HashMap::new(); + let mut deleters: HashMap = HashMap::new(); while let Some(path) = paths.next().await { - let bucket = self.batch_key_for_path(&path); + let batch_key = self.delete_batch_key_for_path(&path).await?; - let (relative_path, deleter) = match deleters.entry(bucket) { + for key in batch_key.superseded_batches(deleters.keys()) { + if let Some(mut deleter) = deleters.remove(&key) { + deleter.close().await.map_err(from_opendal_error)?; + } + } + + let (relative_path, deleter) = match deleters.entry(batch_key) { Entry::Occupied(entry) => { (self.relativize_path(&path)?.to_string(), entry.into_mut()) } Entry::Vacant(entry) => { - let (op, rel) = self.create_operator(&path)?; + let (op, rel) = self.create_operator_with_credential_location( + &path, + entry.key().credential.location(), + )?; let rel = rel.to_string(); let deleter = op.deleter().await.map_err(from_opendal_error)?; (rel, entry.insert(deleter)) @@ -844,8 +1046,58 @@ impl FileWrite for OpenDalWriter { #[cfg(test)] mod tests { + #[cfg(feature = "opendal-s3")] + use std::sync::atomic::AtomicBool; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use std::sync::atomic::{AtomicUsize, Ordering}; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use std::time::{SystemTime, UNIX_EPOCH}; + + #[cfg(feature = "opendal-gcs")] + use iceberg::io::{GCS_SERVICE_HOST, GCS_TOKEN, GCS_TOKEN_EXPIRES_AT}; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + use iceberg::io::{S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, StorageCredential}; + #[cfg(feature = "opendal-s3")] + use iceberg::io::{ + S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SESSION_TOKEN_EXPIRES_AT_MS, + }; + + #[allow(unused_imports)] use super::*; + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[derive(Debug)] + struct AlwaysSupportedCredentialProvider; + + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[async_trait] + impl StorageCredentialProvider for AlwaysSupportedCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> Result> { + let prefix = if path.contains("/table-a/") { + "s3://bucket/table-a" + } else { + "s3://bucket/table-b" + }; + Ok(Arc::new(s3_credential(prefix, "access-key"))) + } + } + + /// An S3 credential for locations under `prefix`. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + fn s3_credential(prefix: impl Into, access_key: &str) -> StorageCredential { + StorageCredential::new( + prefix, + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), access_key.to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "secret-key".to_string()), + ]), + ) + } + fn client_config(value: &str) -> Result { OpenDalClientConfig::from_properties(&HashMap::from([( OPENDAL_IO_TIMEOUT_MS.to_string(), @@ -907,6 +1159,7 @@ mod tests { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, + credential_provider: None, client_config: client_config("45000").unwrap(), }; @@ -1044,6 +1297,22 @@ mod tests { assert_eq!(op.info().scheme().to_string(), "memory"); } + #[cfg(all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ))] + #[test] + fn test_factory_ignores_credentials_for_unsupported_backend() { + let storage = OpenDalStorageFactory::Memory + .build_with_credential_provider( + &StorageConfig::new(), + Some(Arc::new(AlwaysSupportedCredentialProvider)), + ) + .expect("memory must ignore a credential provider"); + + assert!(storage.new_input("memory:/key").is_ok()); + } + #[cfg(feature = "opendal-memory")] #[tokio::test] async fn test_writer_close_returns_stored_size() { @@ -1123,6 +1392,7 @@ mod tests { let storage = OpenDalStorage::S3 { config: Arc::new(S3Config::default()), customized_credential_load: None, + credential_provider: None, client_config: OpenDalClientConfig::default(), }; @@ -1139,11 +1409,583 @@ mod tests { } } + /// Vends a credential for the whole `s3` scheme until it has served + /// `table-c`, then credentials scoped to each table. The scheme-wide + /// credential expires once the `table-c` batch is signed. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct ChangingScopeProvider { + scoped: AtomicBool, + scheme_wide_expired: AtomicBool, + /// Whether the scheme-wide batch, scoped to the storage root, was signed. + scheme_wide_signed: AtomicBool, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for ChangingScopeProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> Result> { + if path == "s3://bucket/" { + self.scheme_wide_signed.store(true, Ordering::SeqCst); + } + if path.starts_with("s3://bucket/table-c") { + self.scoped.store(true, Ordering::SeqCst); + } + // Signing the scoped batch means the scheme-wide credential is gone. + if path == "s3://bucket/table-c" { + self.scheme_wide_expired.store(true, Ordering::SeqCst); + } + match path.strip_prefix("s3://bucket/") { + Some(rest) if self.scoped.load(Ordering::SeqCst) && !rest.is_empty() => { + let table = rest.split('/').next().unwrap_or_default(); + Ok(Arc::new(s3_credential( + format!("s3://bucket/{table}"), + "SCOPED_AK", + ))) + } + _ if self.scheme_wide_expired.load(Ordering::SeqCst) => Err(Error::new( + ErrorKind::Unexpected, + "the scheme-wide credential expired", + )), + _ => Ok(Arc::new(s3_credential("s3", "SCHEME_AK"))), + } + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_delete_stream_signs_each_batch_with_its_vended_credential() { + let mut server = mockito::Server::new_async().await; + let mut delete = |path: &str, access_key: &str| { + server + .mock("DELETE", path) + .match_header( + "authorization", + mockito::Matcher::Regex(format!("Credential={access_key}/")), + ) + .expect(1) + .with_status(204) + }; + // The scheme-wide batch is flushed, with its own credential, before + // the scoped credential that replaces it is used. + let scheme_wide = delete("/bucket/table-b/f.parquet", "SCHEME_AK") + .create_async() + .await; + let scoped = delete("/bucket/table-c/f.parquet", "SCOPED_AK") + .create_async() + .await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let provider = Arc::new(ChangingScopeProvider::default()); + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(provider.clone()), + client_config: OpenDalClientConfig::default(), + }; + // Polled once the `table-c` path is batched, before the stream ends + // and every open batch is closed. + let scheme_wide_flushed_early = futures::stream::once(async move { + assert!(provider.scheme_wide_signed.load(Ordering::SeqCst)); + None + }) + .filter_map(futures::future::ready); + storage + .delete_stream( + futures::stream::iter([ + "s3://bucket/table-b/f.parquet".to_string(), + "s3://bucket/table-c/f.parquet".to_string(), + ]) + .chain(scheme_wide_flushed_early) + .boxed(), + ) + .await + .unwrap(); + + scheme_wide.assert_async().await; + scoped.assert_async().await; + } + + /// A credential lifetime that covers one request, but falls within the two + /// minutes before expiry in which reqsign reloads a cached S3 or GCS + /// credential. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + const EXPIRING_LIFETIME: Duration = Duration::from_secs(60); + + /// The epoch millisecond `lifetime` from now, as vended expiries are given. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + fn epoch_millis_after(lifetime: Duration) -> String { + (SystemTime::now() + lifetime) + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .to_string() + } + + /// Vends `EXPIRING_AK`, which is about to expire, and then `FRESH_AK`. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct ExpiringS3Provider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for ExpiringS3Provider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + let (access_key, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { + 0 => ("EXPIRING_AK", EXPIRING_LIFETIME), + _ => ("FRESH_AK", Duration::from_secs(3600)), + }; + let mut config = s3_credential("s3", access_key).config().clone(); + config.insert( + S3_SESSION_TOKEN_EXPIRES_AT_MS.to_string(), + epoch_millis_after(lifetime), + ); + Ok(Arc::new(StorageCredential::new("s3", config))) + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_s3_operator_reloads_an_expiring_credential() { + let mut server = mockito::Server::new_async().await; + let mut stat = |access_key: &str| { + server + .mock("HEAD", "/bucket/file.parquet") + .match_header( + "authorization", + mockito::Matcher::Regex(format!("Credential={access_key}/")), + ) + .expect(1) + .with_status(200) + .with_header("content-length", "0") + }; + let expiring = stat("EXPIRING_AK").create_async().await; + let fresh = stat("FRESH_AK").create_async().await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(ExpiringS3Provider::default())), + client_config: OpenDalClientConfig::default(), + }; + let path = "s3://bucket/file.parquet"; + let (operator, relative_path) = storage.create_operator(&path).unwrap(); + for _ in 0..2 { + operator.stat(relative_path).await.unwrap(); + } + + expiring.assert_async().await; + fresh.assert_async().await; + } + + /// Fails every credential load. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[derive(Debug)] + struct FailingCredentialProvider; + + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[async_trait] + impl StorageCredentialProvider for FailingCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + Err(Error::new(ErrorKind::Unexpected, "refresh failed")) + } + } + + #[cfg(feature = "opendal-gcs")] + fn gcs_storage( + server: &mockito::Server, + provider: impl StorageCredentialProvider + 'static, + ) -> OpenDalStorage { + let config = gcs_config_parse(HashMap::from([ + (GCS_SERVICE_HOST.to_string(), server.url()), + (GCS_TOKEN.to_string(), "static-token".to_string()), + ])) + .unwrap(); + OpenDalStorage::Gcs { + config: Arc::new(config), + credential_provider: Some(Arc::new(provider)), + client_config: OpenDalClientConfig::default(), + } + } + + /// Vends `expiring-token`, which is about to expire, and then `fresh-token`. + #[cfg(feature = "opendal-gcs")] + #[derive(Debug, Default)] + struct ExpiringGcsProvider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-gcs")] + #[async_trait] + impl StorageCredentialProvider for ExpiringGcsProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + let (token, lifetime) = match self.loads.fetch_add(1, Ordering::SeqCst) { + 0 => ("expiring-token", EXPIRING_LIFETIME), + _ => ("fresh-token", Duration::from_secs(3600)), + }; + Ok(Arc::new(StorageCredential::new( + "gs", + HashMap::from([ + (GCS_TOKEN.to_string(), token.to_string()), + ( + GCS_TOKEN_EXPIRES_AT.to_string(), + epoch_millis_after(lifetime), + ), + ]), + ))) + } + } + + #[cfg(feature = "opendal-gcs")] + #[tokio::test] + async fn test_gcs_operator_reloads_an_expiring_token() { + let mut server = mockito::Server::new_async().await; + let mut stat = |token: &str| { + server + .mock("GET", mockito::Matcher::Any) + .match_header("authorization", format!("Bearer {token}").as_str()) + .expect(1) + .with_status(404) + }; + let expiring = stat("expiring-token").create_async().await; + let fresh = stat("fresh-token").create_async().await; + + let storage = gcs_storage(&server, ExpiringGcsProvider::default()); + let path = "gs://bucket/file.parquet"; + let (operator, relative_path) = storage.create_operator(&path).unwrap(); + for _ in 0..2 { + assert!(!operator.exists(relative_path).await.unwrap()); + } + + expiring.assert_async().await; + fresh.assert_async().await; + } + + #[cfg(feature = "opendal-gcs")] + #[tokio::test] + async fn test_gcs_does_not_fall_back_when_the_provider_fails() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + + let storage = gcs_storage(&server, FailingCredentialProvider); + assert!(storage.exists("gs://bucket/file.parquet").await.is_err()); + mock.assert_async().await; + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_storage_with_credential_provider_is_not_serializable() { + let storage = + |credential_provider: Option>| OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider, + client_config: OpenDalClientConfig::default(), + }; + + assert!(serde_json::to_string(&storage(None)).is_ok()); + let error = + serde_json::to_string(&storage(Some(Arc::new(AlwaysSupportedCredentialProvider)))) + .unwrap_err(); + assert!(error.to_string().contains("credential provider"), "{error}"); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_s3_does_not_fall_back_when_the_provider_fails() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("HEAD", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + // REST catalogs often return static keys next to vended credentials. + let config = s3_config_parse(HashMap::from([ + (S3_ENDPOINT.to_string(), server.url()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_ACCESS_KEY_ID.to_string(), "STATIC_AK".to_string()), + ( + S3_SECRET_ACCESS_KEY.to_string(), + "static-secret".to_string(), + ), + ])) + .unwrap(); + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(FailingCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + assert!(storage.exists("s3://bucket/file.parquet").await.is_err()); + mock.assert_async().await; + } + + /// Vends one credential scoped to `s3://bucket/table` and counts loads. + #[cfg(feature = "opendal-s3")] + #[derive(Debug, Default)] + struct CountingS3Provider { + loads: AtomicUsize, + } + + #[cfg(feature = "opendal-s3")] + #[async_trait] + impl StorageCredentialProvider for CountingS3Provider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + self.loads.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(s3_credential("s3://bucket/table", "SCOPED_AK"))) + } + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_delete_stream_loads_a_credential_per_path_and_batch() { + let mut server = mockito::Server::new_async().await; + let delete = server + .mock("POST", mockito::Matcher::Any) + .match_header( + "authorization", + mockito::Matcher::Regex("Credential=SCOPED_AK/".to_string()), + ) + .expect(1) + .with_status(200) + .with_body(r#""#) + .create_async() + .await; + + let mut config = S3Config::default(); + config.endpoint = Some(server.url()); + config.region = Some("us-east-1".to_string()); + config.disable_config_load = true; + config.disable_ec2_metadata = true; + let provider = Arc::new(CountingS3Provider::default()); + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(provider.clone()), + client_config: OpenDalClientConfig::default(), + }; + let paths: Vec = (0..3) + .map(|i| format!("s3://bucket/table/f{i}.parquet")) + .collect(); + storage + .delete_stream(futures::stream::iter(paths.clone()).boxed()) + .await + .unwrap(); + + delete.assert_async().await; + // One load to choose each path's batch, and one to sign the batch. + assert_eq!(provider.loads.load(Ordering::SeqCst), paths.len() + 1); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_vended_credentials_batch_by_prefix() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + let first = "s3://bucket/table-a/file.parquet"; + let same_scope = "s3://bucket/table-a/other.parquet"; + let other_scope = "s3://bucket/table-b/file.parquet"; + + let first_key = storage.delete_batch_key_for_path(first).await.unwrap(); + assert_eq!(first_key.credential.location(), Some("s3://bucket/table-a")); + assert_eq!( + first_key, + storage.delete_batch_key_for_path(same_scope).await.unwrap() + ); + assert_ne!( + first_key, + storage + .delete_batch_key_for_path(other_scope) + .await + .unwrap() + ); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_scheme_wide_vended_credentials_batch_by_storage_root() { + #[derive(Debug)] + struct SchemeWideProvider; + + #[async_trait] + impl StorageCredentialProvider for SchemeWideProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(s3_credential("s3", "access-key"))) + } + } + + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + credential_provider: Some(Arc::new(SchemeWideProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let key = storage + .delete_batch_key_for_path("s3://bucket/table-a/file.parquet") + .await + .unwrap(); + assert_eq!(key.credential, BatchCredential::SchemeWide { + root: "s3://bucket/".to_string() + }); + assert_eq!( + key, + storage + .delete_batch_key_for_path("s3://bucket/table-b/file.parquet") + .await + .unwrap() + ); + } + + #[test] + fn test_scoped_batch_flushes_broader_batches_of_same_storage() { + let key = |storage: &str, credential: BatchCredential| DeleteBatchKey { + storage: storage.to_string(), + credential, + }; + let unscoped = key("bucket", BatchCredential::SchemeWide { + root: "s3://bucket/".to_string(), + }); + let other_storage = key("other", BatchCredential::SchemeWide { + root: "s3://other/".to_string(), + }); + let scoped = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-a".to_string(), + }); + let open = [unscoped.clone(), other_storage, scoped.clone()]; + + let new_scope = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-b".to_string(), + }); + assert_eq!(new_scope.superseded_batches(open.iter()), vec![ + unscoped.clone() + ]); + + // A narrower scope also supersedes a broader scoped batch. + let nested = key("bucket", BatchCredential::Scoped { + prefix: "s3://bucket/table-a/data".to_string(), + }); + let superseded = nested.superseded_batches(open.iter()); + assert_eq!(superseded.len(), 2); + assert!(superseded.contains(&unscoped) && superseded.contains(&scoped)); + // A batch never supersedes itself. + assert_eq!(scoped.superseded_batches(open.iter()), vec![ + unscoped.clone() + ]); + // Scheme-wide and static paths leave open batches alone. + assert!(unscoped.superseded_batches(open.iter()).is_empty()); + assert!( + key("bucket", BatchCredential::Static) + .superseded_batches(open.iter()) + .is_empty() + ); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_custom_s3_credential_loader_ignores_credential_provider_for_batching() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: Some(CustomAwsCredentialLoader::new( + reqsign_aws_v4::StaticCredentialProvider::new("access-key", "secret-key"), + )), + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let key = storage + .delete_batch_key_for_path("s3://bucket/table-a/file.parquet") + .await + .unwrap(); + assert_eq!(key.credential, BatchCredential::Static); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_s3_rejects_anonymous_vended_credentials() { + let mut config = S3Config::default(); + config.skip_signature = true; + let storage = OpenDalStorage::S3 { + config: Arc::new(config), + customized_credential_load: None, + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let error = storage + .create_operator(&"s3://bucket/file.parquet") + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_gcs_rejects_anonymous_vended_credentials() { + let mut config = GcsConfig::default(); + config.skip_signature = true; + let storage = OpenDalStorage::Gcs { + config: Arc::new(config), + credential_provider: Some(Arc::new(AlwaysSupportedCredentialProvider)), + client_config: OpenDalClientConfig::default(), + }; + + let error = storage + .create_operator(&"gs://bucket/file.parquet") + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + #[cfg(feature = "opendal-gcs")] #[test] fn test_relativize_path_gcs() { let storage = OpenDalStorage::Gcs { config: Arc::new(GcsConfig::default()), + credential_provider: None, client_config: OpenDalClientConfig::default(), }; @@ -1160,6 +2002,7 @@ mod tests { fn test_relativize_path_gcs_invalid_scheme() { let storage = OpenDalStorage::Gcs { config: Arc::new(GcsConfig::default()), + credential_provider: None, client_config: OpenDalClientConfig::default(), }; diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index d7a977e7ac..250ef50de7 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -27,7 +27,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredentialProvider, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; @@ -83,10 +83,14 @@ fn extract_scheme(path: &str) -> Result<&'static str> { } /// Build an [`OpenDalStorage`] variant for the given scheme and config properties. +#[allow(unused_variables)] fn build_storage_for_scheme( scheme: &'static str, props: &HashMap, #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] credential_provider: &Option< + Arc, + >, ) -> Result { let client_config = OpenDalClientConfig::from_properties(props)?; match scheme { @@ -96,6 +100,7 @@ fn build_storage_for_scheme( Ok(OpenDalStorage::S3 { config: Arc::new(config), customized_credential_load: customized_credential_load.clone(), + credential_provider: credential_provider.clone(), client_config, }) } @@ -104,6 +109,7 @@ fn build_storage_for_scheme( let config = crate::gcs::gcs_config_parse(props.clone())?; Ok(OpenDalStorage::Gcs { config: Arc::new(config), + credential_provider: credential_provider.clone(), client_config, }) } @@ -215,11 +221,27 @@ impl OpenDalResolvingStorageFactory { #[typetag::serde] impl StorageFactory for OpenDalResolvingStorageFactory { fn build(&self, config: &StorageConfig) -> Result> { + self.build_with_credential_provider(config, None) + } + + #[cfg_attr( + not(any(feature = "opendal-s3", feature = "opendal-gcs")), + allow(unused_variables) + )] + fn build_with_credential_provider( + &self, + config: &StorageConfig, + credential_provider: Option>, + ) -> Result> { + // Without a compatible backend the provider is ignored, and every + // backend uses the credentials in `config`. Ok(Arc::new(OpenDalResolvingStorage { props: config.props().clone(), storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: self.customized_credential_load.clone(), + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + credential_provider, })) } } @@ -241,6 +263,14 @@ pub struct OpenDalResolvingStorage { #[cfg(feature = "opendal-s3")] #[serde(skip)] customized_credential_load: Option, + /// Provider of refreshable vended credentials. + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "crate::serialize_credential_provider" + )] + credential_provider: Option>, } impl OpenDalResolvingStorage { @@ -276,6 +306,8 @@ impl OpenDalResolvingStorage { &self.props, #[cfg(feature = "opendal-s3")] &self.customized_credential_load, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + &self.credential_provider, )?; let storage = Arc::new(storage); cache.insert(scheme, storage.clone()); @@ -336,6 +368,19 @@ impl Storage for OpenDalResolvingStorage { Ok(()) } + #[cfg_attr( + not(any( + feature = "opendal-memory", + feature = "opendal-fs", + feature = "opendal-s3", + feature = "opendal-gcs", + feature = "opendal-oss", + feature = "opendal-azdls", + feature = "opendal-hf" + )), + // Without a backend, there is no storage to resolve. + allow(unreachable_code) + )] fn new_input(&self, path: &str) -> Result { Ok(InputFile::new( Arc::new(self.resolve(path)?.as_ref().clone()), @@ -343,6 +388,19 @@ impl Storage for OpenDalResolvingStorage { )) } + #[cfg_attr( + not(any( + feature = "opendal-memory", + feature = "opendal-fs", + feature = "opendal-s3", + feature = "opendal-gcs", + feature = "opendal-oss", + feature = "opendal-azdls", + feature = "opendal-hf" + )), + // Without a backend, there is no storage to resolve. + allow(unreachable_code) + )] fn new_output(&self, path: &str) -> Result { Ok(OutputFile::new( Arc::new(self.resolve(path)?.as_ref().clone()), @@ -355,9 +413,39 @@ impl Storage for OpenDalResolvingStorage { mod tests { use std::time::Duration; + use iceberg::io::StorageCredential; + + #[allow(unused_imports)] use super::*; use crate::OPENDAL_IO_TIMEOUT_MS; + #[derive(Debug)] + struct AllPathsCredentialProvider; + + #[async_trait] + impl StorageCredentialProvider for AllPathsCredentialProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + unreachable!("resolving a storage never loads credentials") + } + } + + #[cfg(not(any(feature = "opendal-s3", feature = "opendal-gcs")))] + #[test] + fn test_factory_ignores_credentials_without_compatible_backend() { + assert!( + OpenDalResolvingStorageFactory::new() + .build_with_credential_provider( + &StorageConfig::new(), + Some(Arc::new(AllPathsCredentialProvider)), + ) + .is_ok() + ); + } + #[cfg(feature = "opendal-s3")] #[derive(Debug)] struct EmptyCredentialLoader; @@ -396,6 +484,8 @@ mod tests { storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: None, + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + credential_provider: None, } } @@ -448,6 +538,49 @@ mod tests { assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance"); } + #[cfg(all( + feature = "opendal-memory", + any(feature = "opendal-s3", feature = "opendal-gcs") + ))] + #[test] + fn test_resolver_ignores_credentials_for_unsupported_backend() { + let mut storage = empty_resolving_storage(); + storage.credential_provider = Some(Arc::new(AllPathsCredentialProvider)); + + assert!(storage.resolve("memory:/key").is_ok()); + } + + #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] + #[test] + #[allow(unreachable_patterns)] + fn test_resolve_passes_the_credential_provider_to_s3_and_gcs() { + let mut storage = empty_resolving_storage(); + storage.credential_provider = Some(Arc::new(AllPathsCredentialProvider)); + + let paths: &[&str] = &[ + #[cfg(feature = "opendal-s3")] + "s3://bucket/key", + #[cfg(feature = "opendal-gcs")] + "gs://bucket/key", + ]; + for path in paths { + let has_provider = match storage.resolve(path).unwrap().as_ref() { + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + credential_provider, + .. + } => credential_provider.is_some(), + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { + credential_provider, + .. + } => credential_provider.is_some(), + _ => false, + }; + assert!(has_provider, "{path}"); + } + } + #[cfg(feature = "opendal-hdfs-native")] #[test] fn test_resolve_hdfs_native_returns_hdfs_variant() { diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 2997fa81d9..cef90535b8 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -22,7 +22,8 @@ use iceberg::io::{ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, - S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, + S3_SESSION_TOKEN, S3_SESSION_TOKEN_EXPIRES_AT_MS, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, + StorageCredentialProvider, }; use iceberg::{Error, ErrorKind, Result}; use opendal::services::S3Config; @@ -31,10 +32,15 @@ use opendal::{Configurator, Operator}; pub use reqsign_aws_v4::Credential as AwsCredential; /// Trait for types that can asynchronously supply [`AwsCredential`] to a [`CustomAwsCredentialLoader`]. pub use reqsign_core::ProvideCredential; -use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn}; +use reqsign_core::{ + Context, ProvideCredentialChain, ProvideCredentialDyn, Result as ReqsignResult, +}; use url::Url; -use crate::utils::{from_opendal_error, is_truthy}; +use crate::utils::{ + VendedCredentialSource, credential_expiry, from_opendal_error, is_truthy, + required_credential_property, +}; /// Parse iceberg props to s3 config. pub(crate) fn s3_config_parse(mut m: HashMap) -> Result { @@ -129,7 +135,9 @@ pub(crate) fn s3_config_parse(mut m: HashMap) -> Result, + credential_provider: &Option>, path: &str, + credential_location: Option<&str>, ) -> Result { let url = Url::parse(path)?; let bucket = url.host_str().ok_or_else(|| { @@ -139,6 +147,19 @@ pub(crate) fn s3_config_build( ) })?; + // Preserve the existing custom-loader precedence: an explicitly configured loader + // is the sole source, otherwise install the credential provider as a replacement chain so + // refresh failures cannot fall through to broader ambient AWS credentials. + let credential_provider = credential_provider + .as_ref() + .filter(|provider| provider.supports_path(path)); + if customized_credential_load.is_none() && credential_provider.is_some() && cfg.skip_signature { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid S3 auth settings: anonymous access cannot be combined with refreshable credentials", + )); + } + let mut builder = cfg .clone() .into_builder() @@ -148,11 +169,49 @@ pub(crate) fn s3_config_build( if let Some(loader) = customized_credential_load { let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0)); builder = builder.credential_provider_chain(chain); + } else if let Some(provider) = credential_provider { + let chain = ProvideCredentialChain::new().push(VendedS3CredentialProvider( + VendedCredentialSource::new( + Arc::clone(provider), + credential_location.unwrap_or(path).to_string(), + ), + )); + builder = builder.credential_provider_chain(chain); } Operator::new(builder).map_err(from_opendal_error) } +/// Adapts a generic [`StorageCredentialProvider`] into a reqsign +/// [`ProvideCredential`], so the S3 signer can obtain and refresh vended +/// credentials. +#[derive(Debug)] +struct VendedS3CredentialProvider(VendedCredentialSource); + +impl ProvideCredential for VendedS3CredentialProvider { + type Credential = AwsCredential; + + async fn provide_credential(&self, _ctx: &Context) -> ReqsignResult> { + let credential = self + .0 + .load("S3", |config| { + Ok(AwsCredential { + access_key_id: required_credential_property(config, S3_ACCESS_KEY_ID)? + .to_string(), + secret_access_key: required_credential_property(config, S3_SECRET_ACCESS_KEY)? + .to_string(), + session_token: config + .get(S3_SESSION_TOKEN) + .filter(|token| !token.is_empty()) + .cloned(), + expires_in: credential_expiry(config, S3_SESSION_TOKEN_EXPIRES_AT_MS)?, + }) + }) + .await?; + Ok(Some(credential)) + } +} + /// Custom AWS credential loader. /// /// Wraps any [`ProvideCredential`] implementation for use with the S3 storage backend. @@ -183,10 +242,95 @@ impl CustomAwsCredentialLoader { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; + + use async_trait::async_trait; + use iceberg::io::{GCS_TOKEN, S3_PATH_STYLE_ACCESS, StorageCredential}; + use reqsign_core::time::Timestamp; + + use super::*; + + #[derive(Debug)] + struct FixedProvider(StorageCredential); - use iceberg::io::S3_PATH_STYLE_ACCESS; + #[async_trait] + impl StorageCredentialProvider for FixedProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> Result> { + Ok(Arc::new(self.0.clone())) + } + } + + fn adapter(credential: StorageCredential) -> VendedS3CredentialProvider { + VendedS3CredentialProvider(VendedCredentialSource::new( + Arc::new(FixedProvider(credential)), + "s3://bucket/table/file.parquet".to_string(), + )) + } + + fn credential(prefix: &str, config: &[(&str, &str)]) -> StorageCredential { + StorageCredential::new( + prefix, + config + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + ) + } - use super::s3_config_parse; + #[tokio::test] + async fn test_vended_adapter_returns_expiring_aws_credential() { + let credential = credential("s3://bucket/table", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + (S3_SESSION_TOKEN, "TOK"), + (S3_SESSION_TOKEN_EXPIRES_AT_MS, "1500"), + ]); + + let aws = adapter(credential) + .provide_credential(&Context::new()) + .await + .unwrap() + .unwrap(); + assert_eq!(aws.access_key_id, "AK"); + assert_eq!(aws.secret_access_key, "SK"); + assert_eq!(aws.session_token.as_deref(), Some("TOK")); + assert_eq!( + aws.expires_in, + Some(Timestamp::from_millisecond(1500).unwrap()) + ); + } + + #[tokio::test] + async fn test_vended_adapter_rejects_incomplete_or_uncovering_credentials() { + for credential in [ + // A credential for another backend. + credential("s3", &[(GCS_TOKEN, "token")]), + // No secret key. + credential("s3", &[(S3_ACCESS_KEY_ID, "AK")]), + // An unparsable expiry. + credential("s3", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + (S3_SESSION_TOKEN_EXPIRES_AT_MS, "soon"), + ]), + // A prefix that does not cover the path. + credential("s3://bucket/other", &[ + (S3_ACCESS_KEY_ID, "AK"), + (S3_SECRET_ACCESS_KEY, "SK"), + ]), + ] { + assert!( + adapter(credential) + .provide_credential(&Context::new()) + .await + .is_err() + ); + } + } fn parse_with(prop: Option<&str>) -> bool { let mut props = HashMap::new(); diff --git a/crates/storage/opendal/src/utils.rs b/crates/storage/opendal/src/utils.rs index 56f8c18059..318d483108 100644 --- a/crates/storage/opendal/src/utils.rs +++ b/crates/storage/opendal/src/utils.rs @@ -15,6 +15,21 @@ // specific language governing permissions and limitations // under the License. +use cfg_if::cfg_if; +use iceberg::io::StorageCredential; + +cfg_if! { + if #[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] { + use std::collections::HashMap; + use std::sync::Arc; + + use iceberg::io::StorageCredentialProvider; + use reqsign_core::time::Timestamp; + use reqsign_core::{Error as ReqsignError, Result as ReqsignResult}; + } +} + +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] pub(crate) fn is_truthy(value: &str) -> bool { ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()) } @@ -27,3 +42,283 @@ pub(crate) fn from_opendal_error(e: opendal::Error) -> iceberg::Error { ) .with_source(e) } + +/// The non-empty value of `key` in a vended credential's config. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn required_credential_property<'a>( + config: &'a HashMap, + key: &str, +) -> ReqsignResult<&'a str> { + config + .get(key) + .map(String::as_str) + .filter(|value| !value.is_empty()) + .ok_or_else(|| ReqsignError::unexpected(format!("vended credential is missing {key}"))) +} + +/// The latest epoch millisecond `reqsign` timestamps can represent, +/// `9999-12-30T22:00:00Z`. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +const MAX_TIMESTAMP_MILLIS: i64 = 253_402_207_200_000; + +/// The epoch-millisecond expiry under `key` in a vended credential's config, +/// if present, as the `reqsign` timestamp backend credentials use. Later +/// expiries, such as `Long.MAX_VALUE` for a credential that never expires, +/// are clamped to the latest representable timestamp. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn credential_expiry( + config: &HashMap, + key: &str, +) -> ReqsignResult> { + config + .get(key) + .filter(|value| !value.is_empty()) + .map(|value| { + value + .parse::() + .ok() + .and_then(|millis| { + Timestamp::from_millisecond(millis.min(MAX_TIMESTAMP_MILLIS)).ok() + }) + .ok_or_else(|| { + ReqsignError::unexpected(format!("vended credential has an invalid {key}")) + }) + }) + .transpose() +} + +/// Validate that a provider's credential covers the location for which the +/// backend requested it. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) fn validate_credential_prefix( + location: &str, + credential: &StorageCredential, +) -> ReqsignResult<()> { + if credential.covers(location) { + Ok(()) + } else { + Err(ReqsignError::unexpected(uncovered_location_message( + location, credential, + ))) + } +} + +/// The error message for a vended credential that does not cover `location`. +pub(crate) fn uncovered_location_message(location: &str, credential: &StorageCredential) -> String { + format!( + "vended credential prefix {:?} does not cover storage location {location:?}", + credential.prefix() + ) +} + +/// The root of the storage location containing `path`, e.g. `s3://bucket/`. +pub(crate) fn storage_root(path: &str) -> iceberg::Result { + let url = url::Url::parse(path)?; + Ok(format!("{}://{}/", url.scheme(), url.authority())) +} + +/// Loads vended credentials for one storage location on behalf of a backend's +/// `reqsign` credential provider. +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +pub(crate) struct VendedCredentialSource { + provider: Arc, + /// Location handed to the provider: the path an operator serves, or the + /// scope shared by a bulk-delete batch. + location: String, +} + +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +impl VendedCredentialSource { + pub(crate) fn new(provider: Arc, location: String) -> Self { + Self { provider, location } + } + + /// Load a credential covering the location and convert its config with + /// `extract` into the backend's credential. + /// + /// Errors are logged here: reqsign's credential chain replaces them with an + /// error that only says no credential could be loaded, and reports the + /// cause only through the `log` crate. They are logged at debug level, as + /// a failing provider fails every request; providers log their own + /// failures. + pub(crate) async fn load( + &self, + backend: &str, + extract: impl FnOnce(&HashMap) -> ReqsignResult, + ) -> ReqsignResult { + self.try_load(backend, extract).await.inspect_err(|error| { + tracing::debug!( + "cannot use vended {backend} credentials for {}: {error}", + self.location + ) + }) + } + + async fn try_load( + &self, + backend: &str, + extract: impl FnOnce(&HashMap) -> ReqsignResult, + ) -> ReqsignResult { + let credential = self + .provider + .load_credential(&self.location) + .await + .map_err(|e| { + ReqsignError::unexpected(format!( + "failed to load vended {backend} credential for {}: {e}", + self.location + )) + .with_source(e) + })?; + validate_credential_prefix(&self.location, &credential)?; + extract(credential.config()) + } +} + +#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +impl std::fmt::Debug for VendedCredentialSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VendedCredentialSource") + .field("location", &self.location) + .finish_non_exhaustive() + } +} + +#[cfg(all(test, any(feature = "opendal-s3", feature = "opendal-gcs")))] +mod tests { + use std::sync::Mutex; + + use async_trait::async_trait; + use iceberg::io::{GCS_TOKEN, GCS_TOKEN_EXPIRES_AT}; + + use super::*; + + /// Returns a credential scoped to `prefix` and records requested locations. + #[derive(Debug)] + struct RecordingProvider { + prefix: &'static str, + requested: Mutex>, + } + + #[async_trait] + impl StorageCredentialProvider for RecordingProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, path: &str) -> iceberg::Result> { + self.requested.lock().unwrap().push(path.to_string()); + Ok(Arc::new(StorageCredential::new( + self.prefix, + HashMap::from([(GCS_TOKEN.to_string(), "token".to_string())]), + ))) + } + } + + async fn load(prefix: &'static str, location: &str) -> ReqsignResult<()> { + let provider = Arc::new(RecordingProvider { + prefix, + requested: Mutex::new(Vec::new()), + }); + let source = VendedCredentialSource::new(provider.clone(), location.to_string()); + let result = source + .load("GCS", |config| { + required_credential_property(config, GCS_TOKEN).map(|_| ()) + }) + .await; + assert_eq!(*provider.requested.lock().unwrap(), vec![location]); + result + } + + #[tokio::test] + async fn test_vended_source_requires_a_covering_credential() { + let location = "gs://bucket/table/data/file.parquet"; + assert!(load("gs", location).await.is_ok()); + assert!(load("gs://bucket/table", location).await.is_ok()); + assert!(load("gs://bucket/tab", location).await.is_ok()); + assert!( + load("gs://bucket/table/data/other", location) + .await + .is_err() + ); + assert!(load("gcs://bucket", location).await.is_err()); + assert!(load("", location).await.is_err()); + } + + #[test] + fn test_credential_properties_are_parsed() { + let config = HashMap::from([ + (GCS_TOKEN.to_string(), "token".to_string()), + (GCS_TOKEN_EXPIRES_AT.to_string(), "1500".to_string()), + ]); + assert_eq!( + required_credential_property(&config, GCS_TOKEN).unwrap(), + "token" + ); + assert!(required_credential_property(&config, "missing").is_err()); + assert_eq!( + credential_expiry(&config, GCS_TOKEN_EXPIRES_AT).unwrap(), + Some(Timestamp::from_millisecond(1500).unwrap()) + ); + assert_eq!(credential_expiry(&config, "missing").unwrap(), None); + // An expiry beyond year 9999, such as Java's `Long.MAX_VALUE`, is + // clamped rather than rejected. + let never = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), i64::MAX.to_string())]); + assert_eq!( + credential_expiry(&never, GCS_TOKEN_EXPIRES_AT).unwrap(), + Some(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS).unwrap()) + ); + // Pins the constant to the latest timestamp reqsign accepts. + assert!(Timestamp::from_millisecond(MAX_TIMESTAMP_MILLIS + 1).is_err()); + // The value is never quoted in the error. + let invalid = HashMap::from([(GCS_TOKEN_EXPIRES_AT.to_string(), "secret".to_string())]); + let error = credential_expiry(&invalid, GCS_TOKEN_EXPIRES_AT) + .unwrap_err() + .to_string(); + assert!(!error.contains("secret"), "{error}"); + } + + #[test] + fn test_storage_root_keeps_scheme_and_authority() { + assert_eq!( + storage_root("s3a://bucket/table/file.parquet").unwrap(), + "s3a://bucket/" + ); + assert_eq!( + storage_root("abfss://fs@account.dfs.core.windows.net/table/file.parquet").unwrap(), + "abfss://fs@account.dfs.core.windows.net/" + ); + assert!(storage_root("not a url").is_err()); + } + + #[derive(Debug)] + struct FailingProvider; + + #[async_trait] + impl StorageCredentialProvider for FailingProvider { + fn supports_path(&self, _path: &str) -> bool { + true + } + + async fn load_credential(&self, _path: &str) -> iceberg::Result> { + Err(iceberg::Error::new( + iceberg::ErrorKind::Unexpected, + "catalog returned 503", + )) + } + } + + #[tokio::test] + async fn test_vended_source_errors_carry_the_cause() { + let source = VendedCredentialSource::new( + Arc::new(FailingProvider), + "gs://bucket/file.parquet".to_string(), + ); + let error = source + .load("GCS", |_| Ok(())) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("catalog returned 503"), "{error}"); + } +}