From 9519a87a5a8a1b69a6c231531a49ad6c02e64d16 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Mon, 15 Jun 2026 18:28:21 +0800 Subject: [PATCH 01/27] feat(rest): use vended storage credentials from LoadTable response --- crates/catalog/rest/src/catalog.rs | 45 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index f8e7c4179b..80a6c53178 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -55,7 +55,7 @@ use crate::response::HttpResponse; use crate::types::{ CatalogConfig, CommitTableRequest, CommitTableResponse, CreateNamespaceRequest, CreateTableRequest, ListNamespaceResponse, ListTablesResponse, LoadTableResult, - NamespaceResponse, RegisterTableRequest, RenameTableRequest, + NamespaceResponse, RegisterTableRequest, RenameTableRequest, StorageCredential, }; /// REST catalog URI @@ -1295,6 +1295,7 @@ impl SessionCatalog for RestSessionCatalog { client .http_client .request(Method::POST, client.config.tables_endpoint(namespace)) + .header("X-Iceberg-Access-Delegation", "vended-credentials") .json(&CreateTableRequest { name: creation.name, location: creation.location, @@ -1335,11 +1336,7 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `create_table` response!", ))?; - let config = response - .config - .into_iter() - .chain(self.user_config.props.clone()) - .collect(); + let config = table_file_io_config(&response, &self.user_config.props); let file_io = self .load_file_io(Some(metadata_location), Some(config)) @@ -1380,7 +1377,9 @@ impl SessionCatalog for RestSessionCatalog { let mut request_builder = client .http_client - .request(Method::GET, client.config.table_endpoint(table_ident)); + .request(Method::GET, client.config.table_endpoint(table_ident)) + // Opt in to vended storage credentials. + .header("X-Iceberg-Access-Delegation", "vended-credentials"); if let Some((etag, _)) = &cached { request_builder = request_builder.header(header::IF_NONE_MATCH, etag); } @@ -1432,11 +1431,7 @@ impl SessionCatalog for RestSessionCatalog { .keys() .any(|key| is_short_lived_credential(key)); - let config = response - .config - .into_iter() - .chain(self.user_config.props.clone()) - .collect(); + let config = table_file_io_config(&response, &self.user_config.props); let file_io = self .load_file_io(response.metadata_location.as_deref(), Some(config)) @@ -1669,9 +1664,12 @@ impl SessionCatalog for RestSessionCatalog { } }; + // Reload for a credentialed FileIO (commit response carries no credentials). let file_io = self - .load_file_io(Some(&response.metadata_location), None) - .await?; + .load_table(context, commit.identifier()) + .await? + .file_io() + .clone(); let mut table_builder = Table::builder() .identifier(commit.identifier().clone()) @@ -1874,6 +1872,23 @@ impl RestSessionCatalogBuilder { } } +/// FileIO props: server `config`, then vended `storage_credentials` (longest prefix wins), then user props. +fn table_file_io_config( + response: &LoadTableResult, + user_props: &HashMap, +) -> HashMap { + let mut config: HashMap = response.config.clone(); + if let Some(creds) = response.storage_credentials.as_ref() { + let mut sorted: Vec<&StorageCredential> = creds.iter().collect(); + sorted.sort_by_key(|c| c.prefix.len()); + for cred in sorted { + config.extend(cred.config.clone()); + } + } + config.extend(user_props.clone()); + config +} + #[cfg(test)] mod tests { use std::fs::File; @@ -5070,6 +5085,7 @@ mod tests { let config_mock = create_config_mock(&mut server).await; + // GET hit twice: commit refresh + post-commit reload. let load_table_mock = server .mock("GET", "/v1/namespaces/ns1/tables/test1") .with_status(200) @@ -5078,6 +5094,7 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "load_table_response.json" )) + .expect(2) .create_async() .await; From 1054fb4e0f5646b38eafcf7aa55e7b4eac57a6cc Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 16 Jun 2026 00:29:12 +0800 Subject: [PATCH 02/27] feat(rest): per-prefix vended credentials with opt-in delegation header --- crates/catalog/rest/src/catalog.rs | 165 ++++++++++--- .../load_table_response_with_credentials.json | 78 ++++++ crates/iceberg/public-api.txt | 1 + crates/iceberg/src/io/file_io.rs | 231 ++++++++++++++++-- crates/iceberg/src/table.rs | 6 + crates/iceberg/src/transaction/mod.rs | 4 +- 6 files changed, 424 insertions(+), 61 deletions(-) create mode 100644 crates/catalog/rest/testdata/load_table_response_with_credentials.json diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 80a6c53178..7a05416e91 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -987,6 +987,7 @@ impl RestSessionCatalog { &self, metadata_location: Option<&str>, extra_config: Option>, + storage_credentials: Option<&[StorageCredential]>, ) -> Result { let mut props = self.client().await?.config.props.clone(); if let Some(config) = extra_config { @@ -1019,9 +1020,19 @@ impl RestSessionCatalog { ) })?; - let file_io = FileIOBuilder::new(factory).with_props(props).build(); + let mut builder = FileIOBuilder::new(factory).with_props(props.clone()); - Ok(file_io) + // Vended credentials are scoped per location prefix: give each its own + // storage so reads/writes use the matching credentials. + if let Some(creds) = storage_credentials { + for cred in creds { + let mut prefixed = props.clone(); + prefixed.extend(cred.config.clone()); + builder = builder.with_prefixed_props(cred.prefix.clone(), prefixed); + } + } + + Ok(builder.build()) } } @@ -1295,7 +1306,6 @@ impl SessionCatalog for RestSessionCatalog { client .http_client .request(Method::POST, client.config.tables_endpoint(namespace)) - .header("X-Iceberg-Access-Delegation", "vended-credentials") .json(&CreateTableRequest { name: creation.name, location: creation.location, @@ -1336,10 +1346,15 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `create_table` response!", ))?; - let config = table_file_io_config(&response, &self.user_config.props); + let mut base_config = response.config.clone(); + base_config.extend(self.user_config.props.clone()); let file_io = self - .load_file_io(Some(metadata_location), Some(config)) + .load_file_io( + Some(metadata_location), + Some(base_config), + response.storage_credentials.as_deref(), + ) .await?; let mut table_builder = Table::builder() @@ -1375,11 +1390,12 @@ impl SessionCatalog for RestSessionCatalog { }); let cached = cache.as_ref().and_then(|(cache, key)| cache.get(key)); + // Vended credentials are opt-in via a `header.X-Iceberg-Access-Delegation` + // catalog property (applied to every request like the Iceberg Java client); + // any returned `storage_credentials` are wired into the FileIO below. let mut request_builder = client .http_client - .request(Method::GET, client.config.table_endpoint(table_ident)) - // Opt in to vended storage credentials. - .header("X-Iceberg-Access-Delegation", "vended-credentials"); + .request(Method::GET, client.config.table_endpoint(table_ident)); if let Some((etag, _)) = &cached { request_builder = request_builder.header(header::IF_NONE_MATCH, etag); } @@ -1431,10 +1447,15 @@ impl SessionCatalog for RestSessionCatalog { .keys() .any(|key| is_short_lived_credential(key)); - let config = table_file_io_config(&response, &self.user_config.props); + let mut base_config = response.config.clone(); + base_config.extend(self.user_config.props.clone()); let file_io = self - .load_file_io(response.metadata_location.as_deref(), Some(config)) + .load_file_io( + response.metadata_location.as_deref(), + Some(base_config), + response.storage_credentials.as_deref(), + ) .await?; let mut table_builder = Table::builder() @@ -1585,7 +1606,9 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `register_table` response!", ))?; - let file_io = self.load_file_io(Some(metadata_location), None).await?; + let file_io = self + .load_file_io(Some(metadata_location), None, None) + .await?; let mut table_builder = Table::builder() .identifier(table_ident.clone()) @@ -1601,7 +1624,7 @@ impl SessionCatalog for RestSessionCatalog { async fn update_table( &self, - context: &SessionContext, + _context: &SessionContext, mut commit: TableCommit, ) -> Result { let client = self.client().await?; @@ -1664,12 +1687,11 @@ impl SessionCatalog for RestSessionCatalog { } }; - // Reload for a credentialed FileIO (commit response carries no credentials). + // The commit response carries no credentials, so build a plain FileIO; + // the transaction layer reuses the credentialed one it already holds. let file_io = self - .load_table(context, commit.identifier()) - .await? - .file_io() - .clone(); + .load_file_io(Some(&response.metadata_location), None, None) + .await?; let mut table_builder = Table::builder() .identifier(commit.identifier().clone()) @@ -1872,23 +1894,6 @@ impl RestSessionCatalogBuilder { } } -/// FileIO props: server `config`, then vended `storage_credentials` (longest prefix wins), then user props. -fn table_file_io_config( - response: &LoadTableResult, - user_props: &HashMap, -) -> HashMap { - let mut config: HashMap = response.config.clone(); - if let Some(creds) = response.storage_credentials.as_ref() { - let mut sorted: Vec<&StorageCredential> = creds.iter().collect(); - sorted.sort_by_key(|c| c.prefix.len()); - for cred in sorted { - config.extend(cred.config.clone()); - } - } - config.extend(user_props.clone()); - config -} - #[cfg(test)] mod tests { use std::fs::File; @@ -4869,6 +4874,94 @@ mod tests { rename_table_mock.assert_async().await; } + #[tokio::test] + async fn test_load_table_uses_vended_credentials() { + let mut server = Server::new_async().await; + + let config_mock = create_config_mock(&mut server).await; + + // Vended credentials are opt-in via a `header.*` catalog property (like the + // Java client). With it configured, the header is sent and the response's + // `storage-credentials` are accepted (the FileIO builds). + let load_table_mock = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("x-iceberg-access-delegation", "vended-credentials") + .with_status(200) + .with_body_from_file(format!( + "{}/testdata/{}", + env!("CARGO_MANIFEST_DIR"), + "load_table_response_with_credentials.json" + )) + .create_async() + .await; + + let props = HashMap::from([( + "header.X-Iceberg-Access-Delegation".to_string(), + "vended-credentials".to_string(), + )]); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder() + .uri(server.url()) + .props(props) + .build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + let table = catalog + .load_table(&TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .await + .unwrap(); + + assert_eq!( + "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", + table.metadata_location().unwrap() + ); + + config_mock.assert_async().await; + load_table_mock.assert_async().await; + } + + #[tokio::test] + async fn test_load_table_omits_delegation_header_by_default() { + let mut server = Server::new_async().await; + + let config_mock = create_config_mock(&mut server).await; + + // No delegation header is hardcoded: without a `header.*` prop, none is sent. + let load_table_mock = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("x-iceberg-access-delegation", mockito::Matcher::Missing) + .with_status(200) + .with_body_from_file(format!( + "{}/testdata/{}", + env!("CARGO_MANIFEST_DIR"), + "load_table_response.json" + )) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + catalog + .load_table(&TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .await + .unwrap(); + + config_mock.assert_async().await; + load_table_mock.assert_async().await; + } + #[tokio::test] async fn test_create_table() { let mut server = Server::new_async().await; @@ -5085,7 +5178,7 @@ mod tests { let config_mock = create_config_mock(&mut server).await; - // GET hit twice: commit refresh + post-commit reload. + // GET hit once: the transaction refreshes the table before committing. let load_table_mock = server .mock("GET", "/v1/namespaces/ns1/tables/test1") .with_status(200) @@ -5094,7 +5187,7 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "load_table_response.json" )) - .expect(2) + .expect(1) .create_async() .await; diff --git a/crates/catalog/rest/testdata/load_table_response_with_credentials.json b/crates/catalog/rest/testdata/load_table_response_with_credentials.json new file mode 100644 index 0000000000..c0a540fbac --- /dev/null +++ b/crates/catalog/rest/testdata/load_table_response_with_credentials.json @@ -0,0 +1,78 @@ +{ + "metadata-location": "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", + "metadata": { + "format-version": 1, + "table-uuid": "b55d9dda-6561-423a-8bfc-787980ce421f", + "location": "s3://warehouse/database/table", + "last-updated-ms": 1646787054459, + "last-column-id": 2, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": false, "type": "int"}, + {"id": 2, "name": "data", "required": false, "type": "string"} + ] + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": false, "type": "int"}, + {"id": 2, "name": "data", "required": false, "type": "string"} + ] + } + ], + "partition-spec": [], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {"owner": "bryan", "write.metadata.compression-codec": "gzip"}, + "current-snapshot-id": 3497810964824022504, + "refs": {"main": {"snapshot-id": 3497810964824022504, "type": "branch"}}, + "snapshots": [ + { + "snapshot-id": 3497810964824022504, + "timestamp-ms": 1646787054459, + "summary": { + "operation": "append", + "spark.app.id": "local-1646787004168", + "added-data-files": "1", + "added-records": "1", + "added-files-size": "697", + "changed-partition-count": "1", + "total-records": "1", + "total-files-size": "697", + "total-data-files": "1", + "total-delete-files": "0", + "total-position-deletes": "0", + "total-equality-deletes": "0" + }, + "manifest-list": "s3://warehouse/database/table/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", + "schema-id": 0 + } + ], + "snapshot-log": [{"timestamp-ms": 1646787054459, "snapshot-id": 3497810964824022504}], + "metadata-log": [ + { + "timestamp-ms": 1646787031514, + "metadata-file": "s3://warehouse/database/table/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json" + } + ] + }, + "config": {"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, + "storage-credentials": [ + { + "prefix": "s3://warehouse/database/table", + "config": { + "s3.access-key-id": "vended-key-id", + "s3.secret-access-key": "vended-secret", + "s3.session-token": "vended-token" + } + } + ] +} diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6dd6bd5f37..a21be176e6 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -878,6 +878,7 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_prefixed_props(self, prefix: impl core::convert::Into, props: impl core::iter::traits::collect::IntoIterator) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 42eec4db56..10112f8458 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::ops::Range; use std::sync::{Arc, OnceLock}; use bytes::Bytes; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, stream}; use super::storage::{ LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, @@ -67,6 +68,17 @@ pub struct FileIO { factory: Arc, /// Cached storage instance (lazily initialized) storage: Arc>>, + /// Per-prefix storages (longest prefix first) for tables that vend distinct + /// credentials per location prefix. Paths matching none use `storage` above. + prefixed: Arc>, +} + +/// A storage scoped to a location `prefix`, lazily built from its own config. +#[derive(Debug)] +struct PrefixedStorage { + prefix: String, + config: StorageConfig, + storage: OnceLock>, } mod _serde { @@ -80,12 +92,29 @@ mod _serde { pub(super) struct SerializableFileIO<'a> { pub(super) config: &'a StorageConfig, pub(super) factory: &'a Arc, + /// Per-prefix credentials travel too: a worker that deserializes this + /// FileIO has to read the same data the catalog vended them for. + pub(super) prefixed: Vec>, + } + + #[derive(Serialize)] + pub(super) struct SerializablePrefixed<'a> { + pub(super) prefix: &'a str, + pub(super) config: &'a StorageConfig, } #[derive(Deserialize)] pub(super) struct DeserializedFileIO { pub(super) config: StorageConfig, pub(super) factory: Arc, + #[serde(default)] + pub(super) prefixed: Vec, + } + + #[derive(Deserialize)] + pub(super) struct DeserializedPrefixed { + pub(super) prefix: String, + pub(super) config: StorageConfig, } } @@ -98,6 +127,7 @@ impl FileIO { config: StorageConfig::new(), factory: Arc::new(MemoryStorageFactory), storage: Arc::new(OnceLock::new()), + prefixed: Arc::new(Vec::new()), } } @@ -109,6 +139,7 @@ impl FileIO { config: StorageConfig::new(), factory: Arc::new(LocalFsStorageFactory), storage: Arc::new(OnceLock::new()), + prefixed: Arc::new(Vec::new()), } } @@ -131,6 +162,14 @@ impl FileIO { Ok(serde_json::to_vec(&_serde::SerializableFileIO { config: &self.config, factory: &self.factory, + prefixed: self + .prefixed + .iter() + .map(|p| _serde::SerializablePrefixed { + prefix: &p.prefix, + config: &p.config, + }) + .collect(), })?) } @@ -140,11 +179,26 @@ impl FileIO { /// implementation so it is registered with `typetag`. Backend-specific requirements are /// documented by each storage factory implementation. pub fn deserialize_all(bytes: &[u8]) -> Result { - let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; + let _serde::DeserializedFileIO { + config, + factory, + prefixed, + } = serde_json::from_slice(bytes)?; + // Order is preserved from `serialize_all`, so this is still longest + // prefix first. + let prefixed = prefixed + .into_iter() + .map(|p| PrefixedStorage { + prefix: p.prefix, + config: p.config, + storage: OnceLock::new(), + }) + .collect(); Ok(Self { config, factory, storage: Arc::new(OnceLock::new()), + prefixed: Arc::new(prefixed), }) } @@ -153,24 +207,31 @@ impl FileIO { &self.config } - /// Get or create the storage instance. - /// - /// The factory is invoked on first access and the result is cached - /// for all subsequent operations. - fn get_storage(&self) -> Result> { - // Check if already initialized - if let Some(storage) = self.storage.get() { - return Ok(storage.clone()); + /// Get or create the storage for `path`, routing to the longest-matching + /// prefix storage if any, else the default. Built once, then cached. + fn get_storage(&self, path: &str) -> Result> { + // `prefixed` is sorted longest-first, so the first match is most specific. + for ps in self.prefixed.iter() { + if path.starts_with(&ps.prefix) { + return Self::get_or_build(&ps.storage, &self.factory, &ps.config); + } } + Self::get_or_build(&self.storage, &self.factory, &self.config) + } - // Build the storage - let storage = self.factory.build(&self.config)?; - - // Try to set it (another thread might have set it first) - let _ = self.storage.set(storage.clone()); - - // Return whatever is in the cell (either ours or another thread's) - Ok(self.storage.get().unwrap().clone()) + /// Get a cached storage from `cell`, building it from `config` on first use. + fn get_or_build( + cell: &OnceLock>, + factory: &Arc, + config: &StorageConfig, + ) -> Result> { + if let Some(storage) = cell.get() { + return Ok(storage.clone()); + } + let storage = factory.build(config)?; + // Another thread might have set it first; keep whatever ends up in the cell. + let _ = cell.set(storage); + Ok(cell.get().unwrap().clone()) } /// Deletes file. @@ -179,7 +240,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub async fn delete(&self, path: impl AsRef) -> Result<()> { - self.get_storage()?.delete(path.as_ref()).await + self.get_storage(path.as_ref())?.delete(path.as_ref()).await } /// Remove the path and all nested dirs and files recursively. @@ -194,7 +255,9 @@ impl FileIO { /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { - self.get_storage()?.delete_prefix(path.as_ref()).await + self.get_storage(path.as_ref())? + .delete_prefix(path.as_ref()) + .await } /// Delete multiple files from a stream of paths. @@ -206,7 +269,29 @@ impl FileIO { &self, paths: impl Stream + Send + 'static, ) -> Result<()> { - self.get_storage()?.delete_stream(paths.boxed()).await + // No per-prefix storages: delete the whole batch on the default storage. + if self.prefixed.is_empty() { + return self.get_storage("")?.delete_stream(paths.boxed()).await; + } + + // Otherwise group paths by routed storage, then batch-delete each group. + let mut groups: HashMap> = HashMap::new(); + let mut paths = paths.boxed(); + while let Some(path) = paths.next().await { + let key = self + .prefixed + .iter() + .find(|ps| path.starts_with(&ps.prefix)) + .map(|ps| ps.prefix.clone()) + .unwrap_or_default(); + groups.entry(key).or_default().push(path); + } + + for batch in groups.into_values() { + let storage = self.get_storage(&batch[0])?; + storage.delete_stream(stream::iter(batch).boxed()).await?; + } + Ok(()) } /// Check file exists. @@ -215,7 +300,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub async fn exists(&self, path: impl AsRef) -> Result { - self.get_storage()?.exists(path.as_ref()).await + self.get_storage(path.as_ref())?.exists(path.as_ref()).await } /// Creates input file. @@ -224,7 +309,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_input(&self, path: impl AsRef) -> Result { - self.get_storage()?.new_input(path.as_ref()) + self.get_storage(path.as_ref())?.new_input(path.as_ref()) } /// Creates output file. @@ -233,7 +318,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_output(&self, path: impl AsRef) -> Result { - self.get_storage()?.new_output(path.as_ref()) + self.get_storage(path.as_ref())?.new_output(path.as_ref()) } } @@ -247,6 +332,8 @@ pub struct FileIOBuilder { factory: Arc, /// Storage configuration config: StorageConfig, + /// Per-location-prefix configs (prefix, config). + prefixed: Vec<(String, StorageConfig)>, } impl FileIOBuilder { @@ -255,6 +342,7 @@ impl FileIOBuilder { Self { factory, config: StorageConfig::new(), + prefixed: Vec::new(), } } @@ -275,6 +363,23 @@ impl FileIOBuilder { self } + /// Add a per-prefix storage config. Paths starting with `prefix` (longest + /// match wins) use these props instead of the default config. + pub fn with_prefixed_props( + mut self, + prefix: impl Into, + props: impl IntoIterator, + ) -> Self { + let config = StorageConfig::from_props( + props + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ); + self.prefixed.push((prefix.into(), config)); + self + } + /// Get the storage configuration. pub fn config(&self) -> &StorageConfig { &self.config @@ -282,10 +387,22 @@ impl FileIOBuilder { /// Builds [`FileIO`]. pub fn build(self) -> FileIO { + let mut prefixed: Vec = self + .prefixed + .into_iter() + .map(|(prefix, config)| PrefixedStorage { + prefix, + config, + storage: OnceLock::new(), + }) + .collect(); + // Longest prefix first so routing picks the most specific match. + prefixed.sort_by(|a, b| b.prefix.len().cmp(&a.prefix.len())); FileIO { config: self.config, factory: self.factory, storage: Arc::new(OnceLock::new()), + prefixed: Arc::new(prefixed), } } } @@ -677,4 +794,70 @@ mod tests { ); assert!(deserialized.storage.get().is_some()); } + + #[tokio::test] + async fn test_prefixed_props_sorted_by_descending_prefix_length() { + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prefixed_props("memory://a/", [("k", "short")]) + .with_prefixed_props("memory://a/longer/", [("k", "long")]) + .build(); + + // Longest prefix first so the most specific match wins at routing time. + let prefixes: Vec<&str> = file_io.prefixed.iter().map(|p| p.prefix.as_str()).collect(); + assert_eq!(prefixes, vec!["memory://a/longer/", "memory://a/"]); + } + + #[tokio::test] + async fn test_get_storage_routes_by_prefix() { + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prop("scope", "default") + .with_prefixed_props("memory://creds/", [("scope", "prefixed")]) + .build(); + + let default_a = file_io.get_storage("memory://other/x").unwrap(); + let default_b = file_io.get_storage("memory://other/y").unwrap(); + let prefixed_a = file_io.get_storage("memory://creds/x").unwrap(); + let prefixed_b = file_io.get_storage("memory://creds/y").unwrap(); + + // Repeated routing to the same bucket returns the memoized storage... + assert!(Arc::ptr_eq(&default_a, &default_b)); + assert!(Arc::ptr_eq(&prefixed_a, &prefixed_b)); + // ...and a prefix-matching path resolves to a distinct storage from the default. + assert!(!Arc::ptr_eq(&default_a, &prefixed_a)); + } + + #[tokio::test] + async fn test_delete_stream_routes_by_prefix() { + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prefixed_props("memory:/creds/", [("k", "v")]) + .build(); + + // One file under each routing bucket (default vs prefixed storage). + let default_path = "memory:/other/a.txt"; + let prefixed_path = "memory:/creds/b.txt"; + for path in [default_path, prefixed_path] { + file_io + .new_output(path) + .unwrap() + .write("x".into()) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + // delete_stream must route each path to the storage that holds it. + file_io + .delete_stream(futures::stream::iter(vec![ + default_path.to_string(), + prefixed_path.to_string(), + ])) + .await + .unwrap(); + + assert!(!file_io.exists(default_path).await.unwrap()); + assert!(!file_io.exists(prefixed_path).await.unwrap()); + } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index f33c246eb7..36b9adb400 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -228,6 +228,12 @@ impl Table { self } + /// Sets the [`Table`] `FileIO` and returns an updated instance. + pub(crate) fn with_file_io(mut self, file_io: FileIO) -> Self { + self.file_io = file_io; + self + } + /// Returns a TableBuilder to build a table pub fn builder() -> TableBuilder { TableBuilder::new() diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 2018af2fc1..7f50a643cd 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -249,7 +249,9 @@ impl Transaction { .requirements(existing_requirements) .build(); - catalog.update_table(table_commit).await + let committed = catalog.update_table(table_commit).await?; + // Reuse the credentialed FileIO we already hold; the commit response has none. + Ok(committed.with_file_io(self.table.file_io().clone())) } } From 1b8783fee9c333433745fd129c56fecb96e594ae Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 17 Jun 2026 17:05:34 +0800 Subject: [PATCH 03/27] fix(io): bound delete_stream memory by flushing per-prefix batches --- crates/iceberg/src/io/file_io.rs | 86 ++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 10112f8458..27baff4dcc 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -274,7 +274,9 @@ impl FileIO { return self.get_storage("")?.delete_stream(paths.boxed()).await; } - // Otherwise group paths by routed storage, then batch-delete each group. + // Route by prefix, flushing bounded batches as we iterate so memory stays + // bounded on large streams (like Java's `S3FileIO.deleteFiles`). + const DELETE_BATCH_SIZE: usize = 1000; let mut groups: HashMap> = HashMap::new(); let mut paths = paths.boxed(); while let Some(path) = paths.next().await { @@ -284,12 +286,24 @@ impl FileIO { .find(|ps| path.starts_with(&ps.prefix)) .map(|ps| ps.prefix.clone()) .unwrap_or_default(); - groups.entry(key).or_default().push(path); + let buf = groups.entry(key).or_default(); + buf.push(path); + if buf.len() >= DELETE_BATCH_SIZE { + let full = std::mem::take(buf); + self.get_storage(&full[0])? + .delete_stream(stream::iter(full).boxed()) + .await?; + } } + // Flush remainders. for batch in groups.into_values() { - let storage = self.get_storage(&batch[0])?; - storage.delete_stream(stream::iter(batch).boxed()).await?; + if batch.is_empty() { + continue; + } + self.get_storage(&batch[0])? + .delete_stream(stream::iter(batch).boxed()) + .await?; } Ok(()) } @@ -808,6 +822,39 @@ mod tests { assert_eq!(prefixes, vec!["memory://a/longer/", "memory://a/"]); } + #[tokio::test] + async fn test_prefixed_config_carries_credential_values() { + // Prefix config gets the vended credentials; default config keeps only base props. + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prop("s3.region", "us-east-1") + .with_prefixed_props("s3://bucket/table", [ + ("s3.region", "us-east-1"), + ("s3.access-key-id", "vended-key"), + ("s3.secret-access-key", "vended-secret"), + ]) + .build(); + + // Default: base props, no credentials. + assert_eq!( + file_io.config().get("s3.region"), + Some(&"us-east-1".to_string()) + ); + assert_eq!(file_io.config().get("s3.access-key-id"), None); + + // Prefix: base props + vended credentials. + let prefixed = &file_io.prefixed[0].config; + assert_eq!(prefixed.get("s3.region"), Some(&"us-east-1".to_string())); + assert_eq!( + prefixed.get("s3.access-key-id"), + Some(&"vended-key".to_string()) + ); + assert_eq!( + prefixed.get("s3.secret-access-key"), + Some(&"vended-secret".to_string()) + ); + } + #[tokio::test] async fn test_get_storage_routes_by_prefix() { let factory = Arc::new(MemoryStorageFactory); @@ -860,4 +907,35 @@ mod tests { assert!(!file_io.exists(default_path).await.unwrap()); assert!(!file_io.exists(prefixed_path).await.unwrap()); } + + #[tokio::test] + async fn test_delete_stream_flushes_across_batches() { + // More than the flush threshold (1000): exercises mid-stream flush + remainder. + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prefixed_props("memory:/creds/", [("k", "v")]) + .build(); + + let n = 1050; + let mut paths = Vec::with_capacity(n); + for i in 0..n { + let p = format!("memory:/creds/f{i}.txt"); + file_io + .new_output(&p) + .unwrap() + .write("x".into()) + .await + .unwrap(); + paths.push(p); + } + + file_io + .delete_stream(futures::stream::iter(paths.clone())) + .await + .unwrap(); + + for p in &paths { + assert!(!file_io.exists(p).await.unwrap()); + } + } } From 4b4e18d7ec1d49be74ffe3bd2d54b6fb78e6c915 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 10 Jul 2026 04:56:21 -0400 Subject: [PATCH 04/27] fix(rest): route vended credentials by prefix and keep them in object cache --- crates/catalog/rest/src/catalog.rs | 3 +- crates/iceberg/src/io/file_io.rs | 4 +- crates/iceberg/src/io/object_cache.rs | 11 ++++++ crates/iceberg/src/io/storage/config/mod.rs | 41 +++++++++++++++++++- crates/iceberg/src/table.rs | 42 +++++++++++++++++++++ crates/iceberg/src/transaction/mod.rs | 18 ++++++++- 6 files changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 7a05416e91..e98016f564 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1023,7 +1023,8 @@ impl RestSessionCatalog { let mut builder = FileIOBuilder::new(factory).with_props(props.clone()); // Vended credentials are scoped per location prefix: give each its own - // storage so reads/writes use the matching credentials. + // storage. Paths under no vended prefix fall back to the default `props` + // above, which carry no credentials. if let Some(creds) = storage_credentials { for cred in creds { let mut prefixed = props.clone(); diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 27baff4dcc..b49ba2ea3e 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -211,6 +211,8 @@ impl FileIO { /// prefix storage if any, else the default. Built once, then cached. fn get_storage(&self, path: &str) -> Result> { // `prefixed` is sorted longest-first, so the first match is most specific. + // Selection is by longest matching string prefix, per the Iceberg REST + // spec's storage-credentials semantics (and Java's `S3FileIO`). for ps in self.prefixed.iter() { if path.starts_with(&ps.prefix) { return Self::get_or_build(&ps.storage, &self.factory, &ps.config); @@ -411,7 +413,7 @@ impl FileIOBuilder { }) .collect(); // Longest prefix first so routing picks the most specific match. - prefixed.sort_by(|a, b| b.prefix.len().cmp(&a.prefix.len())); + prefixed.sort_by_key(|item| std::cmp::Reverse(item.prefix.len())); FileIO { config: self.config, factory: self.factory, diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 37ded38946..07902a74f7 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -126,6 +126,17 @@ impl ObjectCache { })) } + /// Returns a cache that uses `file_io` for future cache misses. + pub(crate) fn with_file_io(mut self, file_io: FileIO) -> Self { + self.file_io = file_io; + self + } + + #[cfg(test)] + pub(crate) fn file_io(&self) -> &FileIO { + &self.file_io + } + /// Retrieves an Arc [`Manifest`] from the cache /// or retrieves one from FileIO and parses it if not present pub(crate) async fn get_manifest(&self, manifest_file: &ManifestFile) -> Result> { diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index 8ea4af080e..16a76fa7cf 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -52,12 +52,23 @@ use serde::{Deserialize, Serialize}; /// This struct contains only configuration properties without specifying /// which storage backend to use. The storage type is determined by the /// explicit factory selection. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct StorageConfig { /// Configuration properties for the storage backend props: HashMap, } +impl std::fmt::Debug for StorageConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Property values may hold vended credentials (e.g. `s3.secret-access-key`, + // `s3.session-token`). Debug is reachable through the `FileIO`/`Table` + // derives, so print only the keys and never the secret values. + f.debug_struct("StorageConfig") + .field("keys", &self.props.keys().collect::>()) + .finish_non_exhaustive() + } +} + impl StorageConfig { /// Create a new empty StorageConfig. pub fn new() -> Self { @@ -152,6 +163,34 @@ mod tests { assert!(config.props().is_empty()); } + #[test] + fn test_debug_redacts_credential_values() { + let config = StorageConfig::from_props(HashMap::from([ + ("s3.access-key-id".to_string(), "vended-key".to_string()), + ( + "s3.secret-access-key".to_string(), + "super-secret".to_string(), + ), + ("s3.session-token".to_string(), "vended-token".to_string()), + ])); + + let rendered = format!("{config:?}"); + // Secret values must never appear in Debug output (reachable via FileIO/Table). + assert!( + !rendered.contains("super-secret"), + "leaked secret: {rendered}" + ); + assert!( + !rendered.contains("vended-token"), + "leaked token: {rendered}" + ); + // Keys stay visible so routing/config is still diagnosable. + assert!( + rendered.contains("s3.secret-access-key"), + "keys hidden: {rendered}" + ); + } + #[test] fn test_storage_config_get() { let config = StorageConfig::new().with_prop("region", "us-east-1"); diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 36b9adb400..cd2443d87f 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -230,6 +230,12 @@ impl Table { /// Sets the [`Table`] `FileIO` and returns an updated instance. pub(crate) fn with_file_io(mut self, file_io: FileIO) -> Self { + self.object_cache = Arc::new( + self.object_cache + .as_ref() + .clone() + .with_file_io(file_io.clone()), + ); self.file_io = file_io; self } @@ -427,7 +433,9 @@ mod tests { use crate::ErrorKind; use crate::encryption::SensitiveBytes; use crate::encryption::kms::MemoryKeyManagementClient; + use crate::io::{FileIOBuilder, MemoryStorageFactory}; use crate::spec::TableProperties; + use crate::test_utils::test_runtime; fn load_test_metadata(filename: &str) -> TableMetadata { let path = format!( @@ -537,6 +545,40 @@ mod tests { ); } + #[test] + fn test_with_file_io_updates_object_cache_file_io() { + let metadata = load_test_metadata("TableMetadataV2ValidMinimal.json"); + let original_file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("marker", "original") + .build(); + let replacement_file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("marker", "replacement") + .build(); + + let table = Table::builder() + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "table"]).unwrap()) + .file_io(original_file_io) + .runtime(test_runtime()) + .build() + .unwrap() + .with_file_io(replacement_file_io); + + assert_eq!( + table.file_io().config().get("marker").map(String::as_str), + Some("replacement") + ); + assert_eq!( + table + .object_cache() + .file_io() + .config() + .get("marker") + .map(String::as_str), + Some("replacement") + ); + } + fn make_kms() -> Arc { let kms = MemoryKeyManagementClient::new(); kms.add_master_key("master-1").unwrap(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 7f50a643cd..3d0171f300 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -243,6 +243,12 @@ impl Transaction { )?; } + // A location change moves metadata/data to a new prefix that the refresh + // load's vended credentials do not cover, so it needs a post-commit reload. + let location_changed = existing_updates + .iter() + .any(|update| matches!(update, TableUpdate::SetLocation { .. })); + let table_commit = TableCommit::builder() .ident(self.table.identifier().to_owned()) .updates(existing_updates) @@ -250,8 +256,16 @@ impl Transaction { .build(); let committed = catalog.update_table(table_commit).await?; - // Reuse the credentialed FileIO we already hold; the commit response has none. - Ok(committed.with_file_io(self.table.file_io().clone())) + if location_changed { + // The new location has its own vended credentials; the reused FileIO is + // scoped to the old prefix, so reload the table to pick them up. + catalog.load_table(committed.identifier()).await + } else { + // The commit response carries no credentials. Reuse the FileIO from the + // refresh load above (not `self.table`, which is left untouched when the + // metadata is unchanged) so freshly vended credentials are not dropped. + Ok(committed.with_file_io(refreshed.file_io().clone())) + } } } From b072d0bf48a3b353739c791b2eea147eb77c8395 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 14 Aug 2026 03:16:48 -0400 Subject: [PATCH 05/27] test(rest): cover vended credentials reaching FileIO and prefix routing --- crates/catalog/rest/src/catalog.rs | 20 ++++++++++ crates/iceberg/public-api.txt | 1 + crates/iceberg/src/io/file_io.rs | 54 ++++++++++++++++++++++++--- crates/iceberg/src/transaction/mod.rs | 35 ++++++++++++++++- 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index e98016f564..436b694193 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -4922,6 +4922,26 @@ mod tests { table.metadata_location().unwrap() ); + // The point of the feature: the vended credentials reach the FileIO, + // scoped to the prefix the server sent them for. + let file_io = table.file_io(); + let vended = file_io.config_for("s3://warehouse/database/table/data/f.parquet"); + assert_eq!( + vended.get("s3.access-key-id"), + Some(&"vended-key-id".to_string()) + ); + assert_eq!( + vended.get("s3.session-token"), + Some(&"vended-token".to_string()) + ); + // A path outside the prefix keeps the credential-free default. + assert_eq!( + file_io + .config_for("s3://warehouse/other/f.parquet") + .get("s3.access-key-id"), + None + ); + config_mock.assert_async().await; load_table_mock.assert_async().await; } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index a21be176e6..5110f74366 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -859,6 +859,7 @@ pub fn iceberg::io::AzdlsConfig::deserialize<__D>(__deserializer: __D) -> core:: pub struct iceberg::io::FileIO impl iceberg::io::FileIO pub fn iceberg::io::FileIO::config(&self) -> &iceberg::io::StorageConfig +pub fn iceberg::io::FileIO::config_for(&self, path: &str) -> &iceberg::io::StorageConfig pub async fn iceberg::io::FileIO::delete(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index b49ba2ea3e..991be4967b 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -207,18 +207,32 @@ impl FileIO { &self.config } + /// The configuration `path` routes to: the longest-matching prefix's if + /// any, else the default. Vended credentials live on the prefix configs, + /// so this answers "which credentials apply to this path". + pub fn config_for(&self, path: &str) -> &StorageConfig { + self.route(path).1 + } + /// Get or create the storage for `path`, routing to the longest-matching /// prefix storage if any, else the default. Built once, then cached. fn get_storage(&self, path: &str) -> Result> { - // `prefixed` is sorted longest-first, so the first match is most specific. - // Selection is by longest matching string prefix, per the Iceberg REST - // spec's storage-credentials semantics (and Java's `S3FileIO`). + let (cell, config) = self.route(path); + Self::get_or_build(cell, &self.factory, config) + } + + /// The storage cell and configuration serving `path`. + /// + /// `prefixed` is sorted longest-first, so the first match is the most + /// specific one, per the Iceberg REST spec's storage-credentials semantics + /// (and Java's `S3FileIO`). + fn route(&self, path: &str) -> (&OnceLock>, &StorageConfig) { for ps in self.prefixed.iter() { if path.starts_with(&ps.prefix) { - return Self::get_or_build(&ps.storage, &self.factory, &ps.config); + return (&ps.storage, &ps.config); } } - Self::get_or_build(&self.storage, &self.factory, &self.config) + (&self.storage, &self.config) } /// Get a cached storage from `cell`, building it from `config` on first use. @@ -811,6 +825,36 @@ mod tests { assert!(deserialized.storage.get().is_some()); } + #[tokio::test] + async fn test_routes_a_path_to_its_longest_matching_prefix() { + // Overlapping prefixes: the more specific credentials must win, so + // sorting alone isn't enough — the lookup has to respect that order. + // Each prefix owns a storage cell, so identity tells them apart. + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prefixed_props("memory://bucket/data", [("k", "short")]) + .with_prefixed_props("memory://bucket/data/warehouse", [("k", "long")]) + .build(); + + let long = file_io + .get_storage("memory://bucket/data/warehouse/t/f") + .unwrap(); + let short = file_io.get_storage("memory://bucket/data/other/f").unwrap(); + let default = file_io.get_storage("memory://elsewhere/f").unwrap(); + + // Routing the shortest prefix first would collapse these two. + assert!(!Arc::ptr_eq(&long, &short)); + assert!(!Arc::ptr_eq(&short, &default)); + assert!(!Arc::ptr_eq(&long, &default)); + // The same prefix keeps serving the same storage. + assert!(Arc::ptr_eq( + &long, + &file_io + .get_storage("memory://bucket/data/warehouse/other") + .unwrap() + )); + } + #[tokio::test] async fn test_prefixed_props_sorted_by_descending_prefix_length() { let factory = Arc::new(MemoryStorageFactory); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 3d0171f300..7f09f4502c 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -278,7 +278,7 @@ mod tests { use std::sync::atomic::{AtomicU32, Ordering}; use crate::catalog::MockCatalog; - use crate::io::FileIO; + use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory}; use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Literal, PartitionSpec, Struct, @@ -478,6 +478,39 @@ mod tests { .unwrap() } + #[tokio::test] + async fn test_commit_keeps_the_refreshed_file_io() { + // The commit response carries no vended credentials, so the FileIO from + // the refresh load has to survive; taking the committed table's own + // would silently drop them. + let refreshed = make_v2_table().with_file_io( + FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prefixed_props("memory://warehouse", [("s3.access-key-id", "vended")]) + .build(), + ); + let mut mock_catalog = MockCatalog::new(); + mock_catalog.expect_load_table().returning_st(move |_| { + let refreshed = refreshed.clone(); + Box::pin(async move { Ok(refreshed) }) + }); + mock_catalog + .expect_update_table() + .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) })); + + let table = create_test_transaction(&make_v2_table()) + .commit(&mock_catalog) + .await + .unwrap(); + + assert_eq!( + table + .file_io() + .config_for("memory://warehouse/t/f") + .get("s3.access-key-id"), + Some(&"vended".to_string()), + ); + } + /// Helper function to set up a mock catalog with retryable errors fn setup_mock_catalog_with_retryable_errors( success_after_attempts: Option, From b547ab709ac320d961eb0bec388eeb1c24125655 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 14 Aug 2026 10:15:45 -0400 Subject: [PATCH 06/27] docs(io): note that prefix matching is raw-string, as in Java --- crates/iceberg/src/io/file_io.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 991be4967b..8a27aade24 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -224,8 +224,10 @@ impl FileIO { /// The storage cell and configuration serving `path`. /// /// `prefixed` is sorted longest-first, so the first match is the most - /// specific one, per the Iceberg REST spec's storage-credentials semantics - /// (and Java's `S3FileIO`). + /// specific one, per the Iceberg REST spec's storage-credentials semantics. + /// + /// Matching is on the raw string, so `s3://bucket/data` also serves + /// `s3://bucket/database/` — as in Java's `S3FileIO.clientForStoragePath`. fn route(&self, path: &str) -> (&OnceLock>, &StorageConfig) { for ps in self.prefixed.iter() { if path.starts_with(&ps.prefix) { From 9fca4badf53f62122f62a690196056e9be869bf0 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 2 Sep 2026 01:46:15 -0400 Subject: [PATCH 07/27] fix(rest): keep vended credentials on register_table --- crates/catalog/rest/src/catalog.rs | 65 ++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 436b694193..28da2f9d61 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1607,8 +1607,15 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `register_table` response!", ))?; + let mut base_config = response.config.clone(); + base_config.extend(self.user_config.props.clone()); + let file_io = self - .load_file_io(Some(metadata_location), None, None) + .load_file_io( + Some(metadata_location), + Some(base_config), + response.storage_credentials.as_deref(), + ) .await?; let mut table_builder = Table::builder() @@ -1688,8 +1695,10 @@ impl SessionCatalog for RestSessionCatalog { } }; - // The commit response carries no credentials, so build a plain FileIO; - // the transaction layer reuses the credentialed one it already holds. + // The commit response carries no credentials, so this FileIO has only the + // catalog-level config. `Transaction::do_commit` swaps in the credentialed + // one from its pre-commit load; only a direct `update_table` caller sees + // this plain one. let file_io = self .load_file_io(Some(&response.metadata_location), None, None) .await?; @@ -5465,6 +5474,56 @@ mod tests { register_table_mock.assert_async().await; } + /// The register response is a `LoadTableResult`, so the vended credentials + /// it carries have to reach the table's FileIO like `load_table`'s do. + #[tokio::test] + async fn test_register_table_uses_vended_credentials() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let register_table_mock = server + .mock("POST", "/v1/namespaces/ns1/register") + .with_status(200) + .with_body_from_file(format!( + "{}/testdata/{}", + env!("CARGO_MANIFEST_DIR"), + "load_table_response_with_credentials.json" + )) + .create_async() + .await; + + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let table_ident = TableIdent::from_strs(["ns1", "test1"]).unwrap(); + let table = catalog + .register_table( + &SessionContext::empty(), + &table_ident, + "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json".to_string(), + ) + .await + .unwrap(); + + let vended = table + .file_io() + .config_for("s3://warehouse/database/table/data/f.parquet"); + assert_eq!( + vended.get("s3.access-key-id").map(String::as_str), + Some("vended-key-id") + ); + // Scoped to the prefix the server sent them for. + let outside = table + .file_io() + .config_for("s3://other-bucket/data/f.parquet"); + assert_eq!(outside.get("s3.access-key-id"), None); + // The response's table config reaches the FileIO too, as in load_table. + assert_eq!( + table.file_io().config().get("region").map(String::as_str), + Some("us-west-2") + ); + + config_mock.assert_async().await; + register_table_mock.assert_async().await; + } + #[tokio::test] async fn test_register_table_404() { let mut server = Server::new_async().await; From de524562d7e7cd5e1fdf677d3e9d27e24c13c1e0 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 9 Sep 2026 00:56:49 -0400 Subject: [PATCH 08/27] test(io): prefixed credentials survive FileIO serialization --- crates/iceberg/src/io/file_io.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 8a27aade24..807da449c4 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -795,6 +795,31 @@ mod tests { assert!(deserialized.storage.get().is_some()); } + /// Per-prefix vended credentials must survive serialization, or a worker + /// that receives this FileIO cannot read the data they were vended for. + #[tokio::test] + async fn test_prefixed_credentials_survive_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("s3.access-key-id", "default-key") + .with_prefixed_props("memory://warehouse/t", [("s3.access-key-id", "vended-key")]) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + + assert_eq!( + deserialized + .config_for("memory://warehouse/t/data/f.parquet") + .get("s3.access-key-id"), + Some(&"vended-key".to_string()) + ); + assert_eq!( + deserialized + .config_for("memory://elsewhere/f.parquet") + .get("s3.access-key-id"), + Some(&"default-key".to_string()) + ); + } + #[tokio::test] async fn test_local_fs_file_io_serialization_roundtrip() { let tmp_dir = TempDir::new().unwrap(); From 106e43a2901d92f5f74ce1fee59866a731dfd826 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 10 Sep 2026 02:11:44 -0400 Subject: [PATCH 09/27] docs(rest): note that only Transaction can reach update_table --- crates/catalog/rest/src/catalog.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 28da2f9d61..833efc2afe 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1697,8 +1697,8 @@ impl SessionCatalog for RestSessionCatalog { // The commit response carries no credentials, so this FileIO has only the // catalog-level config. `Transaction::do_commit` swaps in the credentialed - // one from its pre-commit load; only a direct `update_table` caller sees - // this plain one. + // one from its pre-commit load, and it is the only caller there can be: + // `TableCommit` is buildable inside the `iceberg` crate alone. let file_io = self .load_file_io(Some(&response.metadata_location), None, None) .await?; From 0a55ebcb41b40c9e8e21b2ecb8d1fcc893adc55c Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 20 Sep 2026 15:06:03 -0400 Subject: [PATCH 10/27] fix(io): keep cached storages out of FileIO's Debug --- crates/iceberg/src/io/file_io.rs | 136 ++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 807da449c4..6dbb6d92e9 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -60,7 +60,7 @@ use crate::Result; /// .with_prop("key", "value") /// .build(); /// ``` -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct FileIO { /// Storage configuration containing properties config: StorageConfig, @@ -74,13 +74,33 @@ pub struct FileIO { } /// A storage scoped to a location `prefix`, lazily built from its own config. -#[derive(Debug)] struct PrefixedStorage { prefix: String, config: StorageConfig, storage: OnceLock>, } +// A backend's Debug may print the raw credential map that `StorageConfig` +// redacts, so neither cached storage is shown. +impl std::fmt::Debug for FileIO { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FileIO") + .field("config", &self.config) + .field("factory", &self.factory) + .field("prefixed", &self.prefixed) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for PrefixedStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PrefixedStorage") + .field("prefix", &self.prefix) + .field("config", &self.config) + .finish_non_exhaustive() + } +} + mod _serde { use std::sync::Arc; @@ -599,10 +619,17 @@ mod tests { use bytes::Bytes; use futures::AsyncReadExt; use futures::io::AllowStdIo; + use futures::stream::BoxStream; + use serde::{Deserialize, Serialize}; use tempfile::TempDir; - use super::{FileIO, FileIOBuilder}; - use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; + use super::{ + FileIO, FileIOBuilder, InputFile, OutputFile, Storage, StorageConfig, StorageFactory, + }; + use crate::Result; + use crate::io::{ + FileMetadata, FileRead, FileWrite, LocalFsStorageFactory, MemoryStorageFactory, + }; fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -820,6 +847,107 @@ mod tests { ); } + /// Stands in for a backend whose Debug prints its raw props. + #[derive(Serialize, Deserialize)] + struct LeakyStorage(String); + + impl std::fmt::Debug for LeakyStorage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "LeakyStorage {{ secret: {} }}", self.0) + } + } + + #[typetag::serde] + #[async_trait::async_trait] + impl Storage for LeakyStorage { + async fn exists(&self, _: &str) -> Result { + Ok(false) + } + async fn metadata(&self, _: &str) -> Result { + unimplemented!() + } + async fn read(&self, _: &str) -> Result { + unimplemented!() + } + async fn reader(&self, _: &str) -> Result> { + unimplemented!() + } + async fn write(&self, _: &str, _: Bytes) -> Result<()> { + unimplemented!() + } + async fn writer(&self, _: &str) -> Result> { + unimplemented!() + } + async fn delete(&self, _: &str) -> Result<()> { + unimplemented!() + } + async fn delete_prefix(&self, _: &str) -> Result<()> { + unimplemented!() + } + async fn delete_stream(&self, _: BoxStream<'static, String>) -> Result<()> { + unimplemented!() + } + fn new_input(&self, _: &str) -> Result { + unimplemented!() + } + fn new_output(&self, _: &str) -> Result { + unimplemented!() + } + } + + #[derive(Debug, Serialize, Deserialize)] + struct LeakyFactory; + + #[typetag::serde] + impl StorageFactory for LeakyFactory { + fn build(&self, config: &StorageConfig) -> Result> { + let secret = config + .get("s3.secret-access-key") + .cloned() + .unwrap_or_default(); + Ok(Arc::new(LeakyStorage(secret))) + } + } + + /// Once a storage is initialized its own Debug is reachable through + /// `FileIO`; the vended secret must not come out that way. + #[tokio::test] + async fn test_debug_omits_initialized_storages() { + let file_io = FileIOBuilder::new(Arc::new(LeakyFactory)) + .with_prop("s3.secret-access-key", "DEFAULT-SECRET") + .with_prefixed_props("memory://warehouse/t", [( + "s3.secret-access-key", + "VENDED-SECRET", + )]) + .build(); + // Initialize both the default and the prefixed storage. + file_io.exists("memory://elsewhere/f").await.unwrap(); + file_io.exists("memory://warehouse/t/f").await.unwrap(); + + let debug = format!("{file_io:?}"); + assert!(!debug.contains("VENDED-SECRET"), "{debug}"); + assert!(!debug.contains("DEFAULT-SECRET"), "{debug}"); + // Still informative: the prefix and the redacted config keys show. + assert!(debug.contains("memory://warehouse/t"), "{debug}"); + assert!(debug.contains("s3.secret-access-key"), "{debug}"); + } + + /// Two prefixes where one nests in the other: the longest-first order + /// that routing depends on has to survive the roundtrip. + #[tokio::test] + async fn test_overlapping_prefixes_survive_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prefixed_props("memory://w/t", [("s3.access-key-id", "outer")]) + .with_prefixed_props("memory://w/t/nested", [("s3.access-key-id", "inner")]) + .build(); + + let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); + + let key = |p: &str| deserialized.config_for(p).get("s3.access-key-id").cloned(); + assert_eq!(key("memory://w/t/nested/f"), Some("inner".to_string())); + assert_eq!(key("memory://w/t/other/f"), Some("outer".to_string())); + } + #[tokio::test] async fn test_local_fs_file_io_serialization_roundtrip() { let tmp_dir = TempDir::new().unwrap(); From c4e06742aa7ea1fda54ba8027ab1342529c9152e Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 20 Sep 2026 15:06:09 -0400 Subject: [PATCH 11/27] fix(transaction): act with the refreshed FileIO and reload on any location change --- crates/iceberg/src/transaction/mod.rs | 109 ++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 7f09f4502c..90351eb4fe 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -228,7 +228,9 @@ impl Transaction { self.table = refreshed.clone(); } - let mut current_table = self.table.clone(); + // Actions read and write manifests, so they need the credentials the + // refresh load vended, whether or not the base was stale. + let mut current_table = self.table.clone().with_file_io(refreshed.file_io().clone()); let mut existing_updates: Vec = vec![]; let mut existing_requirements: Vec = vec![]; @@ -243,9 +245,7 @@ impl Transaction { )?; } - // A location change moves metadata/data to a new prefix that the refresh - // load's vended credentials do not cover, so it needs a post-commit reload. - let location_changed = existing_updates + let moves_location = existing_updates .iter() .any(|update| matches!(update, TableUpdate::SetLocation { .. })); @@ -256,6 +256,12 @@ impl Transaction { .build(); let committed = catalog.update_table(table_commit).await?; + // A location change moves metadata/data to a new prefix that the refresh + // load's vended credentials do not cover, so it needs a post-commit + // reload. Another writer may have moved the table meanwhile: a + // property-only commit carries no requirement that would catch that. + let location_changed = + moves_location || committed.metadata().location() != refreshed.metadata().location(); if location_changed { // The new location has its own vended credentials; the reused FileIO is // scoped to the old prefix, so reload the table to pick them up. @@ -277,6 +283,8 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; + use async_trait::async_trait; + use crate::catalog::MockCatalog; use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory}; use crate::memory::tests::new_memory_catalog; @@ -286,8 +294,9 @@ mod tests { }; use crate::table::{StaticTable, Table}; use crate::test_utils::{make_encrypted_table, test_runtime}; + use crate::transaction::action::{ActionCommit, TransactionAction}; use crate::transaction::{ApplyTransactionAction, Transaction}; - use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent}; + use crate::{Catalog, Error, ErrorKind, Result, TableCreation, TableIdent}; pub fn make_v1_table() -> Table { let file = File::open(format!( @@ -511,6 +520,96 @@ mod tests { ); } + fn table_with_marker(marker: &str) -> Table { + make_v2_table().with_file_io( + FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("marker", marker) + .build(), + ) + } + + fn marker_of(table: &Table) -> String { + table + .file_io() + .config() + .get("marker") + .cloned() + .unwrap_or_default() + } + + /// Records which FileIO the action was handed. + struct RecordingAction(Arc>>); + + #[async_trait] + impl TransactionAction for RecordingAction { + async fn commit(self: Arc, table: &Table) -> Result { + *self.0.lock().unwrap() = Some(marker_of(table)); + Ok(ActionCommit::new(vec![], vec![])) + } + } + + /// The refresh load may carry rotated credentials while the metadata is + /// unchanged; actions still have to run with those, not the stale ones. + #[tokio::test] + async fn test_actions_run_with_the_refreshed_file_io() { + let seen = Arc::new(std::sync::Mutex::new(None)); + let mut mock_catalog = MockCatalog::new(); + mock_catalog + .expect_load_table() + .returning_st(|_| Box::pin(async { Ok(table_with_marker("refreshed")) })); + mock_catalog + .expect_update_table() + .returning_st(|_| Box::pin(async { Ok(make_v2_table()) })); + + let tx = RecordingAction(seen.clone()) + .apply(Transaction::new(&table_with_marker("stale"))) + .unwrap(); + tx.commit(&mock_catalog).await.unwrap(); + + assert_eq!(seen.lock().unwrap().as_deref(), Some("refreshed")); + } + + /// Another writer moved the table between our refresh and our commit; the + /// committed metadata says so even though we sent no `SetLocation`. + #[tokio::test] + async fn test_commit_reloads_when_someone_else_moved_the_table() { + let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut mock_catalog = MockCatalog::new(); + let counter = loads.clone(); + mock_catalog.expect_load_table().returning_st(move |_| { + let n = counter.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(table_with_marker(if n == 0 { "first" } else { "second" })) }) + }); + mock_catalog.expect_update_table().returning_st(|_| { + Box::pin(async { + let base = make_v2_table(); + let moved = base + .metadata() + .clone() + .into_builder(None) + .set_location("s3://moved-by-someone-else/table".to_string()) + .build() + .unwrap() + .metadata; + Ok(Table::builder() + .identifier(base.identifier().clone()) + .metadata(moved) + .file_io(base.file_io().clone()) + .runtime(test_runtime()) + .build() + .unwrap()) + }) + }); + + let table = create_test_transaction(&make_v2_table()) + .commit(&mock_catalog) + .await + .unwrap(); + + assert_eq!(loads.load(Ordering::SeqCst), 2); + assert_eq!(marker_of(&table), "second"); + } + /// Helper function to set up a mock catalog with retryable errors fn setup_mock_catalog_with_retryable_errors( success_after_attempts: Option, From bdd8df059c2325e90f33ec50aa7bb4b6d68c91b2 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 20 Sep 2026 19:33:15 -0400 Subject: [PATCH 12/27] fix(transaction): keep the table's initialized storage when a reload changes nothing --- crates/iceberg/src/io/file_io.rs | 12 ++++++ crates/iceberg/src/transaction/mod.rs | 62 ++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 6dbb6d92e9..94d7f26a91 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -227,6 +227,18 @@ impl FileIO { &self.config } + /// Whether `other` would route and authenticate identically. Used to keep + /// an initialized backend when a reload brought back the same settings. + pub(crate) fn same_routing_as(&self, other: &FileIO) -> bool { + self.config == other.config + && self.prefixed.len() == other.prefixed.len() + && self + .prefixed + .iter() + .zip(other.prefixed.iter()) + .all(|(a, b)| a.prefix == b.prefix && a.config == b.config) + } + /// The configuration `path` routes to: the longest-matching prefix's if /// any, else the default. Vended credentials live on the prefix configs, /// so this answers "which credentials apply to this path". diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 90351eb4fe..1ec30f4945 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -229,8 +229,13 @@ impl Transaction { } // Actions read and write manifests, so they need the credentials the - // refresh load vended, whether or not the base was stale. - let mut current_table = self.table.clone().with_file_io(refreshed.file_io().clone()); + // refresh load vended, whether or not the base was stale. Identical + // settings keep the original: a rebuilt FileIO would drop a backend + // the table has already initialized. + let mut current_table = self.table.clone(); + if !current_table.file_io().same_routing_as(refreshed.file_io()) { + current_table = current_table.with_file_io(refreshed.file_io().clone()); + } let mut existing_updates: Vec = vec![]; let mut existing_requirements: Vec = vec![]; @@ -569,6 +574,59 @@ mod tests { assert_eq!(seen.lock().unwrap().as_deref(), Some("refreshed")); } + /// A reload that changed nothing must not cost the table its initialized + /// backend: `MemoryStorageFactory` builds a fresh, empty store each time. + #[tokio::test] + async fn test_unchanged_settings_keep_the_initialized_storage() { + let table = table_with_marker("same"); + table + .file_io() + .new_output("memory://warehouse/manifest") + .unwrap() + .write("written-before-commit".into()) + .await + .unwrap(); + + let seen = Arc::new(std::sync::Mutex::new(None)); + let action_io = seen.clone(); + let mut mock_catalog = MockCatalog::new(); + // A reload rebuilds the FileIO, as a REST catalog does. + mock_catalog + .expect_load_table() + .returning_st(|_| Box::pin(async { Ok(table_with_marker("same")) })); + mock_catalog + .expect_update_table() + .returning_st(|_| Box::pin(async { Ok(make_v2_table()) })); + + let tx = ReadingAction(action_io) + .apply(Transaction::new(&table)) + .unwrap(); + tx.commit(&mock_catalog).await.unwrap(); + + assert_eq!( + seen.lock().unwrap().as_deref(), + Some("written-before-commit") + ); + } + + /// Reads a file the table wrote before the transaction started. + struct ReadingAction(Arc>>); + + #[async_trait] + impl TransactionAction for ReadingAction { + async fn commit(self: Arc, table: &Table) -> Result { + let read = table + .file_io() + .new_input("memory://warehouse/manifest")? + .read() + .await + .ok() + .map(|b| String::from_utf8_lossy(&b).to_string()); + *self.0.lock().unwrap() = read; + Ok(ActionCommit::new(vec![], vec![])) + } + } + /// Another writer moved the table between our refresh and our commit; the /// committed metadata says so even though we sent no `SetLocation`. #[tokio::test] From 573f532839fc720387cf11571524294b7a5d091d Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Mon, 28 Sep 2026 00:14:35 -0400 Subject: [PATCH 13/27] fix(transaction): return the actions' FileIO and never replay a durable commit --- crates/iceberg/src/transaction/mod.rs | 107 +++++++++++++++++++------- 1 file changed, 80 insertions(+), 27 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 1ec30f4945..be5e276533 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -270,13 +270,20 @@ impl Transaction { if location_changed { // The new location has its own vended credentials; the reused FileIO is // scoped to the old prefix, so reload the table to pick them up. - catalog.load_table(committed.identifier()).await - } else { - // The commit response carries no credentials. Reuse the FileIO from the - // refresh load above (not `self.table`, which is left untouched when the - // metadata is unchanged) so freshly vended credentials are not dropped. - Ok(committed.with_file_io(refreshed.file_io().clone())) + match catalog.load_table(committed.identifier()).await { + Ok(reloaded) => return Ok(reloaded), + // The commit is already durable: returning this error would let the + // retry loop replay it. Keep the old credentials; a reload fixes them. + Err(e) => tracing::warn!( + "committed {} but could not reload it for the new location: {e}", + committed.identifier() + ), + } } + // The commit response carries no credentials, so keep the FileIO the + // actions ran with: the refreshed one, or the original when its settings + // were unchanged and it holds an initialized backend. + Ok(committed.with_file_io(current_table.file_io().clone())) } } @@ -601,12 +608,21 @@ mod tests { let tx = ReadingAction(action_io) .apply(Transaction::new(&table)) .unwrap(); - tx.commit(&mock_catalog).await.unwrap(); + let committed = tx.commit(&mock_catalog).await.unwrap(); assert_eq!( seen.lock().unwrap().as_deref(), Some("written-before-commit") ); + // The returned table keeps that backend too, not the reload's empty one. + let after = committed + .file_io() + .new_input("memory://warehouse/manifest") + .unwrap() + .read() + .await + .unwrap(); + assert_eq!(&after[..], b"written-before-commit"); } /// Reads a file the table wrote before the transaction started. @@ -627,6 +643,60 @@ mod tests { } } + /// What `update_table` returns once the table root has moved. + fn moved_table() -> Table { + let base = make_v2_table(); + let moved = base + .metadata() + .clone() + .into_builder(None) + .set_location("s3://moved-by-someone-else/table".to_string()) + .build() + .unwrap() + .metadata; + Table::builder() + .identifier(base.identifier().clone()) + .metadata(moved) + .file_io(base.file_io().clone()) + .runtime(test_runtime()) + .build() + .unwrap() + } + + /// The commit is durable before the reload runs, so a failed reload must + /// not send the transaction round the retry loop to be committed again. + #[tokio::test] + async fn test_a_failed_reload_does_not_replay_the_commit() { + let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut mock_catalog = MockCatalog::new(); + let counter = loads.clone(); + mock_catalog.expect_load_table().returning_st(move |_| { + let n = counter.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + if n == 1 { + // The post-commit reload. + Err(Error::new(ErrorKind::Unexpected, "503").with_retryable(true)) + } else { + Ok(make_v2_table()) + } + }) + }); + mock_catalog + .expect_update_table() + .times(1) + .returning_st(|_| Box::pin(async { Ok(moved_table()) })); + + let table = create_test_transaction(&make_v2_table()) + .commit(&mock_catalog) + .await + .unwrap(); + + assert_eq!( + table.metadata().location(), + "s3://moved-by-someone-else/table" + ); + } + /// Another writer moved the table between our refresh and our commit; the /// committed metadata says so even though we sent no `SetLocation`. #[tokio::test] @@ -638,26 +708,9 @@ mod tests { let n = counter.fetch_add(1, Ordering::SeqCst); Box::pin(async move { Ok(table_with_marker(if n == 0 { "first" } else { "second" })) }) }); - mock_catalog.expect_update_table().returning_st(|_| { - Box::pin(async { - let base = make_v2_table(); - let moved = base - .metadata() - .clone() - .into_builder(None) - .set_location("s3://moved-by-someone-else/table".to_string()) - .build() - .unwrap() - .metadata; - Ok(Table::builder() - .identifier(base.identifier().clone()) - .metadata(moved) - .file_io(base.file_io().clone()) - .runtime(test_runtime()) - .build() - .unwrap()) - }) - }); + mock_catalog + .expect_update_table() + .returning_st(|_| Box::pin(async { Ok(moved_table()) })); let table = create_test_transaction(&make_v2_table()) .commit(&mock_catalog) From e9411c7e90df0ff972346332414ecd0be5dd4861 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 01:39:51 -0400 Subject: [PATCH 14/27] fix(rest): preserve context in update_table --- crates/catalog/rest/src/catalog.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 833efc2afe..229ca17d74 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1632,7 +1632,7 @@ impl SessionCatalog for RestSessionCatalog { async fn update_table( &self, - _context: &SessionContext, + context: &SessionContext, mut commit: TableCommit, ) -> Result
{ let client = self.client().await?; From d2ff3c551d1c0a24a49b9b280a826b3a3f1554ae Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 19:41:17 -0700 Subject: [PATCH 15/27] fix(io): delete through nested prefix storages in delete_prefix --- crates/iceberg/src/io/file_io.rs | 61 ++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 94d7f26a91..2833fdb63a 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -304,10 +304,31 @@ impl FileIO { /// - If the path is a file or not exist, this function will be no-op. /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. + /// - Files under the path that route to a deeper per-prefix storage are removed through that storage. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { - self.get_storage(path.as_ref())? - .delete_prefix(path.as_ref()) - .await + let path = path.as_ref(); + let dir = if path.ends_with('/') { + path.to_string() + } else { + format!("{path}/") + }; + // A prefix that is not a directory also serves its siblings under + // `path` (raw-string matching), so that storage clears all of `path`. + for ps in self + .prefixed + .iter() + .filter(|ps| ps.prefix != path && ps.prefix.starts_with(&dir)) + { + let scope = if ps.prefix.ends_with('/') { + ps.prefix.as_str() + } else { + path + }; + Self::get_or_build(&ps.storage, &self.factory, &ps.config)? + .delete_prefix(scope) + .await?; + } + self.get_storage(path)?.delete_prefix(path).await } /// Delete multiple files from a stream of paths. @@ -1121,6 +1142,40 @@ mod tests { assert!(!file_io.exists(prefixed_path).await.unwrap()); } + #[tokio::test] + async fn test_delete_prefix_reaches_nested_prefix_storages() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prefixed_props("memory:/table/data/", [("k", "data")]) + .with_prefixed_props("memory:/table/meta", [("k", "meta")]) + .with_prefixed_props("memory:/tablex/", [("k", "sibling")]) + .build(); + let deleted = [ + "memory:/table/version-hint.text", + "memory:/table/data/a.parquet", + "memory:/table/metadata/v1.json", + "memory:/table/meta-extra/b.json", + ]; + let kept = "memory:/tablex/c.parquet"; + for path in deleted.iter().chain([&kept]) { + file_io + .new_output(path) + .unwrap() + .write("x".into()) + .await + .unwrap(); + } + + file_io.delete_prefix("memory:/table").await.unwrap(); + + let mut remaining = Vec::new(); + for path in deleted.iter().chain([&kept]) { + if file_io.exists(path).await.unwrap() { + remaining.push(*path); + } + } + assert_eq!(remaining, [kept]); + } + #[tokio::test] async fn test_delete_stream_flushes_across_batches() { // More than the flush threshold (1000): exercises mid-stream flush + remainder. From f520a9c25aa8aacccc3a680f53350e41fa2025cf Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 19:41:17 -0700 Subject: [PATCH 16/27] fix(rest): replace inherited credential sets with vended ones --- crates/catalog/rest/src/catalog.rs | 92 ++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 229ca17d74..e59e15d7ac 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -28,7 +28,10 @@ use std::time::Duration; use async_trait::async_trait; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory, + ADLS_ACCOUNT_KEY, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, + ADLS_TENANT_ID, FileIO, FileIOBuilder, GCS_CREDENTIALS_JSON, GCS_TOKEN, HF_TOKEN, + OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, + S3_SESSION_TOKEN, StorageFactory, }; use iceberg::table::Table; use iceberg::{ @@ -1027,9 +1030,10 @@ impl RestSessionCatalog { // above, which carry no credentials. if let Some(creds) = storage_credentials { for cred in creds { - let mut prefixed = props.clone(); - prefixed.extend(cred.config.clone()); - builder = builder.with_prefixed_props(cred.prefix.clone(), prefixed); + builder = builder.with_prefixed_props( + cred.prefix.clone(), + with_vended_credentials(&props, &cred.config), + ); } } @@ -1037,6 +1041,50 @@ impl RestSessionCatalog { } } +/// Storage credential properties that only work together. A key also covers +/// its scoped variants, such as `adls.sas-token.`. +const CREDENTIAL_SETS: [&[&str]; 5] = [ + &[S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN], + &[OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET], + &[GCS_CREDENTIALS_JSON, GCS_TOKEN], + &[ + ADLS_CONNECTION_STRING, + ADLS_ACCOUNT_KEY, + ADLS_SAS_TOKEN, + ADLS_TENANT_ID, + ADLS_CLIENT_ID, + ADLS_CLIENT_SECRET, + ], + &[HF_TOKEN], +]; + +fn in_credential_set(key: &str, set: &[&str]) -> bool { + set.iter().any(|name| { + key.strip_prefix(name) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) + }) +} + +/// `base` overlaid with a vended `config`. Each credential set the vended +/// config touches replaces the inherited one whole, so credentials from two +/// identities never mix; other settings, such as the endpoint, are inherited. +fn with_vended_credentials( + base: &HashMap, + config: &HashMap, +) -> HashMap { + let replaced: Vec<&[&str]> = CREDENTIAL_SETS + .into_iter() + .filter(|set| config.keys().any(|key| in_credential_set(key, set))) + .collect(); + let mut props: HashMap = base + .iter() + .filter(|(key, _)| !replaced.iter().any(|set| in_credential_set(key, set))) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + props.extend(config.clone()); + props +} + /// All requests and expected responses are derived from the REST catalog API spec: /// #[async_trait] @@ -4955,6 +5003,42 @@ mod tests { load_table_mock.assert_async().await; } + #[test] + fn test_vended_credentials_replace_inherited_credential_sets() { + let props = |pairs: &[(&str, &str)]| -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + let base = props(&[ + ("s3.endpoint", "https://s3.example.com"), + ("client.region", "us-east-1"), + ("s3.access-key-id", "catalog-key"), + ("s3.secret-access-key", "catalog-secret"), + ("s3.session-token", "catalog-token"), + ("adls.sas-token.account.dfs.core.windows.net", "catalog-sas"), + ("gcs.oauth2.token", "catalog-gcs-token"), + ]); + let vended = props(&[ + ("s3.access-key-id", "vended-key"), + ("s3.secret-access-key", "vended-secret"), + ("adls.account-key", "vended-account-key"), + ]); + + assert_eq!( + with_vended_credentials(&base, &vended), + props(&[ + ("s3.endpoint", "https://s3.example.com"), + ("client.region", "us-east-1"), + ("s3.access-key-id", "vended-key"), + ("s3.secret-access-key", "vended-secret"), + ("adls.account-key", "vended-account-key"), + ("gcs.oauth2.token", "catalog-gcs-token"), + ]) + ); + } + #[tokio::test] async fn test_load_table_omits_delegation_header_by_default() { let mut server = Server::new_async().await; From c3210722734c7b55107e8d6bd5c8afa2cd260628 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 20:05:59 -0700 Subject: [PATCH 17/27] fix(io): keep nested prefix deletes within their own prefix --- crates/iceberg/src/io/file_io.rs | 104 +++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 2833fdb63a..d30defb8f9 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -304,7 +304,9 @@ impl FileIO { /// - If the path is a file or not exist, this function will be no-op. /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. - /// - Files under the path that route to a deeper per-prefix storage are removed through that storage. + /// - Files under the path that route to a deeper per-prefix storage are removed through that + /// storage, which deletes only within its own prefix; the storage serving the path deletes + /// the rest. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { let path = path.as_ref(); let dir = if path.ends_with('/') { @@ -312,20 +314,14 @@ impl FileIO { } else { format!("{path}/") }; - // A prefix that is not a directory also serves its siblings under - // `path` (raw-string matching), so that storage clears all of `path`. + // Credentials scoped to a prefix may not reach beyond it. for ps in self .prefixed .iter() .filter(|ps| ps.prefix != path && ps.prefix.starts_with(&dir)) { - let scope = if ps.prefix.ends_with('/') { - ps.prefix.as_str() - } else { - path - }; Self::get_or_build(&ps.storage, &self.factory, &ps.config)? - .delete_prefix(scope) + .delete_prefix(&ps.prefix) .await?; } self.get_storage(path)?.delete_prefix(path).await @@ -942,6 +938,93 @@ mod tests { } } + /// Records the `delete_prefix` calls of every storage it builds, tagged + /// with the storage's `scope` property. + #[derive(Debug, Default, Serialize, Deserialize)] + struct RecordingFactory(#[serde(skip)] Arc>>); + + #[derive(Debug, Serialize, Deserialize)] + struct RecordingStorage { + scope: String, + #[serde(skip)] + calls: Arc>>, + } + + #[typetag::serde] + #[async_trait::async_trait] + impl Storage for RecordingStorage { + async fn exists(&self, _: &str) -> Result { + unimplemented!() + } + async fn metadata(&self, _: &str) -> Result { + unimplemented!() + } + async fn read(&self, _: &str) -> Result { + unimplemented!() + } + async fn reader(&self, _: &str) -> Result> { + unimplemented!() + } + async fn write(&self, _: &str, _: Bytes) -> Result<()> { + unimplemented!() + } + async fn writer(&self, _: &str) -> Result> { + unimplemented!() + } + async fn delete(&self, _: &str) -> Result<()> { + unimplemented!() + } + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.calls + .lock() + .unwrap() + .push((self.scope.clone(), path.to_string())); + Ok(()) + } + async fn delete_stream(&self, _: BoxStream<'static, String>) -> Result<()> { + unimplemented!() + } + fn new_input(&self, _: &str) -> Result { + unimplemented!() + } + fn new_output(&self, _: &str) -> Result { + unimplemented!() + } + } + + #[typetag::serde] + impl StorageFactory for RecordingFactory { + fn build(&self, config: &StorageConfig) -> Result> { + Ok(Arc::new(RecordingStorage { + scope: config.get("scope").cloned().unwrap_or_default(), + calls: self.0.clone(), + })) + } + } + + #[tokio::test] + async fn test_delete_prefix_keeps_each_storage_within_its_prefix() { + let factory = RecordingFactory::default(); + let calls = factory.0.clone(); + let file_io = FileIOBuilder::new(Arc::new(factory)) + .with_prop("scope", "default") + .with_prefixed_props("s3://bucket/table/data/", [("scope", "data")]) + .with_prefixed_props("s3://bucket/table/meta", [("scope", "meta")]) + .build(); + + file_io.delete_prefix("s3://bucket/table/").await.unwrap(); + + let expected: Vec<(String, String)> = [ + ("data", "s3://bucket/table/data/"), + ("meta", "s3://bucket/table/meta"), + ("default", "s3://bucket/table/"), + ] + .into_iter() + .map(|(scope, path)| (scope.to_string(), path.to_string())) + .collect(); + assert_eq!(*calls.lock().unwrap(), expected); + } + /// Once a storage is initialized its own Debug is reachable through /// `FileIO`; the vended secret must not come out that way. #[tokio::test] @@ -1146,14 +1229,13 @@ mod tests { async fn test_delete_prefix_reaches_nested_prefix_storages() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) .with_prefixed_props("memory:/table/data/", [("k", "data")]) - .with_prefixed_props("memory:/table/meta", [("k", "meta")]) + .with_prefixed_props("memory:/table/metadata", [("k", "metadata")]) .with_prefixed_props("memory:/tablex/", [("k", "sibling")]) .build(); let deleted = [ "memory:/table/version-hint.text", "memory:/table/data/a.parquet", "memory:/table/metadata/v1.json", - "memory:/table/meta-extra/b.json", ]; let kept = "memory:/tablex/c.parquet"; for path in deleted.iter().chain([&kept]) { From 8023fa73b7761204ba7901719f110dc1c21b343c Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 20:29:16 -0700 Subject: [PATCH 18/27] fix(io): reject nested raw prefixes and route delete_prefix by directory --- crates/iceberg/src/io/file_io.rs | 116 ++++++++++++++++++++++++------- 1 file changed, 91 insertions(+), 25 deletions(-) diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index d30defb8f9..c8cd617960 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -25,7 +25,7 @@ use futures::{Stream, StreamExt, stream}; use super::storage::{ LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, }; -use crate::Result; +use crate::{Error, ErrorKind, Result}; /// FileIO implementation, used to manipulate files in underlying storage. /// @@ -307,6 +307,12 @@ impl FileIO { /// - Files under the path that route to a deeper per-prefix storage are removed through that /// storage, which deletes only within its own prefix; the storage serving the path deletes /// the rest. + /// + /// # Errors + /// + /// Returns [`ErrorKind::FeatureUnsupported`], before deleting anything, if a per-prefix + /// storage beneath the path has a prefix that does not end with `/`: it serves a raw string + /// prefix, which cannot be deleted as a directory. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { let path = path.as_ref(); let dir = if path.ends_with('/') { @@ -315,16 +321,31 @@ impl FileIO { format!("{path}/") }; // Credentials scoped to a prefix may not reach beyond it. - for ps in self + let nested: Vec<&PrefixedStorage> = self .prefixed .iter() - .filter(|ps| ps.prefix != path && ps.prefix.starts_with(&dir)) - { + .filter(|ps| ps.prefix.len() > dir.len() && ps.prefix.starts_with(&dir)) + .collect(); + if let Some(ps) = nested.iter().find(|ps| !ps.prefix.ends_with('/')) { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "cannot delete `{path}` recursively: the credential prefix `{}` beneath it \ + is not a directory", + ps.prefix + ), + )); + } + for ps in nested { Self::get_or_build(&ps.storage, &self.factory, &ps.config)? .delete_prefix(&ps.prefix) .await?; } - self.get_storage(path)?.delete_prefix(path).await + // Route by the directory, so `table` and `table/` reach the same storage. + let (cell, config) = self.route(&dir); + Self::get_or_build(cell, &self.factory, config)? + .delete_prefix(path) + .await } /// Delete multiple files from a stream of paths. @@ -655,10 +676,10 @@ mod tests { use super::{ FileIO, FileIOBuilder, InputFile, OutputFile, Storage, StorageConfig, StorageFactory, }; - use crate::Result; use crate::io::{ FileMetadata, FileRead, FileWrite, LocalFsStorageFactory, MemoryStorageFactory, }; + use crate::{ErrorKind, Result}; fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -938,16 +959,18 @@ mod tests { } } + type DeleteLog = Arc>>; + /// Records the `delete_prefix` calls of every storage it builds, tagged /// with the storage's `scope` property. #[derive(Debug, Default, Serialize, Deserialize)] - struct RecordingFactory(#[serde(skip)] Arc>>); + struct RecordingFactory(#[serde(skip)] DeleteLog); #[derive(Debug, Serialize, Deserialize)] struct RecordingStorage { scope: String, #[serde(skip)] - calls: Arc>>, + calls: DeleteLog, } #[typetag::serde] @@ -1002,27 +1025,70 @@ mod tests { } } - #[tokio::test] - async fn test_delete_prefix_keeps_each_storage_within_its_prefix() { + /// A FileIO over a [`RecordingFactory`] with `prefixes` as (prefix, scope), + /// and the log its storages record into. + fn recording_file_io(prefixes: &[(&str, &str)]) -> (FileIO, DeleteLog) { let factory = RecordingFactory::default(); let calls = factory.0.clone(); - let file_io = FileIOBuilder::new(Arc::new(factory)) - .with_prop("scope", "default") - .with_prefixed_props("s3://bucket/table/data/", [("scope", "data")]) - .with_prefixed_props("s3://bucket/table/meta", [("scope", "meta")]) - .build(); + let mut builder = FileIOBuilder::new(Arc::new(factory)).with_prop("scope", "default"); + for (prefix, scope) in prefixes { + builder = builder.with_prefixed_props(*prefix, [("scope", *scope)]); + } + (builder.build(), calls) + } + + fn recorded(calls: &[(&str, &str)]) -> Vec<(String, String)> { + calls + .iter() + .map(|(scope, path)| (scope.to_string(), path.to_string())) + .collect() + } + + #[tokio::test] + async fn test_delete_prefix_keeps_each_storage_within_its_prefix() { + let (file_io, calls) = recording_file_io(&[ + ("s3://bucket/table/data/", "data"), + ("s3://bucket/table/metadata/", "metadata"), + ]); file_io.delete_prefix("s3://bucket/table/").await.unwrap(); - let expected: Vec<(String, String)> = [ - ("data", "s3://bucket/table/data/"), - ("meta", "s3://bucket/table/meta"), - ("default", "s3://bucket/table/"), - ] - .into_iter() - .map(|(scope, path)| (scope.to_string(), path.to_string())) - .collect(); - assert_eq!(*calls.lock().unwrap(), expected); + assert_eq!( + *calls.lock().unwrap(), + recorded(&[ + ("metadata", "s3://bucket/table/metadata/"), + ("data", "s3://bucket/table/data/"), + ("default", "s3://bucket/table/"), + ]) + ); + } + + #[tokio::test] + async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash() { + for path in ["s3://bucket/table", "s3://bucket/table/"] { + let (file_io, calls) = recording_file_io(&[("s3://bucket/table/", "vended")]); + + file_io.delete_prefix(path).await.unwrap(); + + assert_eq!(*calls.lock().unwrap(), recorded(&[("vended", path)])); + } + } + + #[tokio::test] + async fn test_delete_prefix_rejects_a_nested_raw_prefix() { + let (file_io, calls) = recording_file_io(&[ + ("s3://bucket/table/data/", "data"), + ("s3://bucket/table/meta", "meta"), + ]); + + let err = file_io + .delete_prefix("s3://bucket/table/") + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!(err.message().contains("s3://bucket/table/meta"), "{err}"); + assert!(calls.lock().unwrap().is_empty()); } /// Once a storage is initialized its own Debug is reachable through @@ -1229,7 +1295,7 @@ mod tests { async fn test_delete_prefix_reaches_nested_prefix_storages() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) .with_prefixed_props("memory:/table/data/", [("k", "data")]) - .with_prefixed_props("memory:/table/metadata", [("k", "metadata")]) + .with_prefixed_props("memory:/table/metadata/", [("k", "metadata")]) .with_prefixed_props("memory:/tablex/", [("k", "sibling")]) .build(); let deleted = [ From b34bdee249032dde07a6b9e97dbbed8b3d604ba5 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 20:34:05 -0700 Subject: [PATCH 19/27] docs: simplify vended credential comments --- crates/catalog/rest/src/catalog.rs | 27 ++------- crates/iceberg/src/io/file_io.rs | 66 ++++++--------------- crates/iceberg/src/io/storage/config/mod.rs | 7 +-- crates/iceberg/src/transaction/mod.rs | 33 +++-------- 4 files changed, 33 insertions(+), 100 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index e59e15d7ac..85b7461bbd 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1025,9 +1025,8 @@ impl RestSessionCatalog { let mut builder = FileIOBuilder::new(factory).with_props(props.clone()); - // Vended credentials are scoped per location prefix: give each its own - // storage. Paths under no vended prefix fall back to the default `props` - // above, which carry no credentials. + // Each vended prefix gets its own storage; other paths use the + // credential-free default. if let Some(creds) = storage_credentials { for cred in creds { builder = builder.with_prefixed_props( @@ -1439,9 +1438,8 @@ impl SessionCatalog for RestSessionCatalog { }); let cached = cache.as_ref().and_then(|(cache, key)| cache.get(key)); - // Vended credentials are opt-in via a `header.X-Iceberg-Access-Delegation` - // catalog property (applied to every request like the Iceberg Java client); - // any returned `storage_credentials` are wired into the FileIO below. + // Vended credentials are opt-in through a configured + // `header.X-Iceberg-Access-Delegation`, as in Java. let mut request_builder = client .http_client .request(Method::GET, client.config.table_endpoint(table_ident)); @@ -1743,10 +1741,8 @@ impl SessionCatalog for RestSessionCatalog { } }; - // The commit response carries no credentials, so this FileIO has only the - // catalog-level config. `Transaction::do_commit` swaps in the credentialed - // one from its pre-commit load, and it is the only caller there can be: - // `TableCommit` is buildable inside the `iceberg` crate alone. + // The commit response carries no credentials. `Transaction::do_commit`, + // the only possible caller, keeps the FileIO from its pre-commit load. let file_io = self .load_file_io(Some(&response.metadata_location), None, None) .await?; @@ -4938,9 +4934,6 @@ mod tests { let config_mock = create_config_mock(&mut server).await; - // Vended credentials are opt-in via a `header.*` catalog property (like the - // Java client). With it configured, the header is sent and the response's - // `storage-credentials` are accepted (the FileIO builds). let load_table_mock = server .mock("GET", "/v1/namespaces/ns1/tables/test1") .match_header("x-iceberg-access-delegation", "vended-credentials") @@ -4979,8 +4972,6 @@ mod tests { table.metadata_location().unwrap() ); - // The point of the feature: the vended credentials reach the FileIO, - // scoped to the prefix the server sent them for. let file_io = table.file_io(); let vended = file_io.config_for("s3://warehouse/database/table/data/f.parquet"); assert_eq!( @@ -4991,7 +4982,6 @@ mod tests { vended.get("s3.session-token"), Some(&"vended-token".to_string()) ); - // A path outside the prefix keeps the credential-free default. assert_eq!( file_io .config_for("s3://warehouse/other/f.parquet") @@ -5045,7 +5035,6 @@ mod tests { let config_mock = create_config_mock(&mut server).await; - // No delegation header is hardcoded: without a `header.*` prop, none is sent. let load_table_mock = server .mock("GET", "/v1/namespaces/ns1/tables/test1") .match_header("x-iceberg-access-delegation", mockito::Matcher::Missing) @@ -5558,8 +5547,6 @@ mod tests { register_table_mock.assert_async().await; } - /// The register response is a `LoadTableResult`, so the vended credentials - /// it carries have to reach the table's FileIO like `load_table`'s do. #[tokio::test] async fn test_register_table_uses_vended_credentials() { let mut server = Server::new_async().await; @@ -5593,12 +5580,10 @@ mod tests { vended.get("s3.access-key-id").map(String::as_str), Some("vended-key-id") ); - // Scoped to the prefix the server sent them for. let outside = table .file_io() .config_for("s3://other-bucket/data/f.parquet"); assert_eq!(outside.get("s3.access-key-id"), None); - // The response's table config reaches the FileIO too, as in load_table. assert_eq!( table.file_io().config().get("region").map(String::as_str), Some("us-west-2") diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index c8cd617960..66ed5b609c 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -68,8 +68,8 @@ pub struct FileIO { factory: Arc, /// Cached storage instance (lazily initialized) storage: Arc>>, - /// Per-prefix storages (longest prefix first) for tables that vend distinct - /// credentials per location prefix. Paths matching none use `storage` above. + /// Per-prefix storages, longest prefix first. Paths matching none use + /// `storage`. prefixed: Arc>, } @@ -80,8 +80,8 @@ struct PrefixedStorage { storage: OnceLock>, } -// A backend's Debug may print the raw credential map that `StorageConfig` -// redacts, so neither cached storage is shown. +// Neither cached storage is shown: a backend's Debug may print raw +// credentials. impl std::fmt::Debug for FileIO { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("FileIO") @@ -112,8 +112,7 @@ mod _serde { pub(super) struct SerializableFileIO<'a> { pub(super) config: &'a StorageConfig, pub(super) factory: &'a Arc, - /// Per-prefix credentials travel too: a worker that deserializes this - /// FileIO has to read the same data the catalog vended them for. + /// Per-prefix credentials, so a deserialized FileIO reads the same data. pub(super) prefixed: Vec>, } @@ -204,8 +203,7 @@ impl FileIO { factory, prefixed, } = serde_json::from_slice(bytes)?; - // Order is preserved from `serialize_all`, so this is still longest - // prefix first. + // `serialize_all` kept the longest-first order. let prefixed = prefixed .into_iter() .map(|p| PrefixedStorage { @@ -239,27 +237,23 @@ impl FileIO { .all(|(a, b)| a.prefix == b.prefix && a.config == b.config) } - /// The configuration `path` routes to: the longest-matching prefix's if - /// any, else the default. Vended credentials live on the prefix configs, - /// so this answers "which credentials apply to this path". + /// The configuration `path` routes to: the longest matching prefix's, + /// else the default. pub fn config_for(&self, path: &str) -> &StorageConfig { self.route(path).1 } - /// Get or create the storage for `path`, routing to the longest-matching - /// prefix storage if any, else the default. Built once, then cached. + /// The storage for `path`, built on first use. fn get_storage(&self, path: &str) -> Result> { let (cell, config) = self.route(path); Self::get_or_build(cell, &self.factory, config) } - /// The storage cell and configuration serving `path`. - /// - /// `prefixed` is sorted longest-first, so the first match is the most - /// specific one, per the Iceberg REST spec's storage-credentials semantics. + /// The storage cell and configuration serving `path`: the first, so + /// longest, matching prefix, else the default. /// /// Matching is on the raw string, so `s3://bucket/data` also serves - /// `s3://bucket/database/` — as in Java's `S3FileIO.clientForStoragePath`. + /// `s3://bucket/database/`, as in Java's `S3FileIO.clientForStoragePath`. fn route(&self, path: &str) -> (&OnceLock>, &StorageConfig) { for ps in self.prefixed.iter() { if path.starts_with(&ps.prefix) { @@ -304,15 +298,13 @@ impl FileIO { /// - If the path is a file or not exist, this function will be no-op. /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. - /// - Files under the path that route to a deeper per-prefix storage are removed through that - /// storage, which deletes only within its own prefix; the storage serving the path deletes - /// the rest. + /// - Files routed to a deeper per-prefix storage are deleted through it, within its own prefix. /// /// # Errors /// /// Returns [`ErrorKind::FeatureUnsupported`], before deleting anything, if a per-prefix - /// storage beneath the path has a prefix that does not end with `/`: it serves a raw string - /// prefix, which cannot be deleted as a directory. + /// storage beneath the path has a prefix not ending in `/`, which cannot be deleted as a + /// directory. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { let path = path.as_ref(); let dir = if path.ends_with('/') { @@ -357,13 +349,11 @@ impl FileIO { &self, paths: impl Stream + Send + 'static, ) -> Result<()> { - // No per-prefix storages: delete the whole batch on the default storage. if self.prefixed.is_empty() { return self.get_storage("")?.delete_stream(paths.boxed()).await; } - // Route by prefix, flushing bounded batches as we iterate so memory stays - // bounded on large streams (like Java's `S3FileIO.deleteFiles`). + // Flush bounded batches per prefix, as Java's `S3FileIO.deleteFiles` does. const DELETE_BATCH_SIZE: usize = 1000; let mut groups: HashMap> = HashMap::new(); let mut paths = paths.boxed(); @@ -384,7 +374,6 @@ impl FileIO { } } - // Flush remainders. for batch in groups.into_values() { if batch.is_empty() { continue; @@ -872,8 +861,6 @@ mod tests { assert!(deserialized.storage.get().is_some()); } - /// Per-prefix vended credentials must survive serialization, or a worker - /// that receives this FileIO cannot read the data they were vended for. #[tokio::test] async fn test_prefixed_credentials_survive_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) @@ -1091,8 +1078,6 @@ mod tests { assert!(calls.lock().unwrap().is_empty()); } - /// Once a storage is initialized its own Debug is reachable through - /// `FileIO`; the vended secret must not come out that way. #[tokio::test] async fn test_debug_omits_initialized_storages() { let file_io = FileIOBuilder::new(Arc::new(LeakyFactory)) @@ -1102,20 +1087,16 @@ mod tests { "VENDED-SECRET", )]) .build(); - // Initialize both the default and the prefixed storage. file_io.exists("memory://elsewhere/f").await.unwrap(); file_io.exists("memory://warehouse/t/f").await.unwrap(); let debug = format!("{file_io:?}"); assert!(!debug.contains("VENDED-SECRET"), "{debug}"); assert!(!debug.contains("DEFAULT-SECRET"), "{debug}"); - // Still informative: the prefix and the redacted config keys show. assert!(debug.contains("memory://warehouse/t"), "{debug}"); assert!(debug.contains("s3.secret-access-key"), "{debug}"); } - /// Two prefixes where one nests in the other: the longest-first order - /// that routing depends on has to survive the roundtrip. #[tokio::test] async fn test_overlapping_prefixes_survive_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) @@ -1164,9 +1145,6 @@ mod tests { #[tokio::test] async fn test_routes_a_path_to_its_longest_matching_prefix() { - // Overlapping prefixes: the more specific credentials must win, so - // sorting alone isn't enough — the lookup has to respect that order. - // Each prefix owns a storage cell, so identity tells them apart. let factory = Arc::new(MemoryStorageFactory); let file_io = FileIOBuilder::new(factory) .with_prefixed_props("memory://bucket/data", [("k", "short")]) @@ -1179,11 +1157,9 @@ mod tests { let short = file_io.get_storage("memory://bucket/data/other/f").unwrap(); let default = file_io.get_storage("memory://elsewhere/f").unwrap(); - // Routing the shortest prefix first would collapse these two. assert!(!Arc::ptr_eq(&long, &short)); assert!(!Arc::ptr_eq(&short, &default)); assert!(!Arc::ptr_eq(&long, &default)); - // The same prefix keeps serving the same storage. assert!(Arc::ptr_eq( &long, &file_io @@ -1200,14 +1176,12 @@ mod tests { .with_prefixed_props("memory://a/longer/", [("k", "long")]) .build(); - // Longest prefix first so the most specific match wins at routing time. let prefixes: Vec<&str> = file_io.prefixed.iter().map(|p| p.prefix.as_str()).collect(); assert_eq!(prefixes, vec!["memory://a/longer/", "memory://a/"]); } #[tokio::test] async fn test_prefixed_config_carries_credential_values() { - // Prefix config gets the vended credentials; default config keeps only base props. let factory = Arc::new(MemoryStorageFactory); let file_io = FileIOBuilder::new(factory) .with_prop("s3.region", "us-east-1") @@ -1218,14 +1192,12 @@ mod tests { ]) .build(); - // Default: base props, no credentials. assert_eq!( file_io.config().get("s3.region"), Some(&"us-east-1".to_string()) ); assert_eq!(file_io.config().get("s3.access-key-id"), None); - // Prefix: base props + vended credentials. let prefixed = &file_io.prefixed[0].config; assert_eq!(prefixed.get("s3.region"), Some(&"us-east-1".to_string())); assert_eq!( @@ -1251,10 +1223,8 @@ mod tests { let prefixed_a = file_io.get_storage("memory://creds/x").unwrap(); let prefixed_b = file_io.get_storage("memory://creds/y").unwrap(); - // Repeated routing to the same bucket returns the memoized storage... assert!(Arc::ptr_eq(&default_a, &default_b)); assert!(Arc::ptr_eq(&prefixed_a, &prefixed_b)); - // ...and a prefix-matching path resolves to a distinct storage from the default. assert!(!Arc::ptr_eq(&default_a, &prefixed_a)); } @@ -1265,7 +1235,6 @@ mod tests { .with_prefixed_props("memory:/creds/", [("k", "v")]) .build(); - // One file under each routing bucket (default vs prefixed storage). let default_path = "memory:/other/a.txt"; let prefixed_path = "memory:/creds/b.txt"; for path in [default_path, prefixed_path] { @@ -1278,7 +1247,6 @@ mod tests { assert!(file_io.exists(path).await.unwrap()); } - // delete_stream must route each path to the storage that holds it. file_io .delete_stream(futures::stream::iter(vec![ default_path.to_string(), @@ -1326,7 +1294,7 @@ mod tests { #[tokio::test] async fn test_delete_stream_flushes_across_batches() { - // More than the flush threshold (1000): exercises mid-stream flush + remainder. + // More than one flush batch (1000). let factory = Arc::new(MemoryStorageFactory); let file_io = FileIOBuilder::new(factory) .with_prefixed_props("memory:/creds/", [("k", "v")]) diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index 16a76fa7cf..014e324d22 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -60,9 +60,8 @@ pub struct StorageConfig { impl std::fmt::Debug for StorageConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Property values may hold vended credentials (e.g. `s3.secret-access-key`, - // `s3.session-token`). Debug is reachable through the `FileIO`/`Table` - // derives, so print only the keys and never the secret values. + // Values may hold vended credentials, and Debug is reachable through + // `FileIO` and `Table`: print keys only. f.debug_struct("StorageConfig") .field("keys", &self.props.keys().collect::>()) .finish_non_exhaustive() @@ -175,7 +174,6 @@ mod tests { ])); let rendered = format!("{config:?}"); - // Secret values must never appear in Debug output (reachable via FileIO/Table). assert!( !rendered.contains("super-secret"), "leaked secret: {rendered}" @@ -184,7 +182,6 @@ mod tests { !rendered.contains("vended-token"), "leaked token: {rendered}" ); - // Keys stay visible so routing/config is still diagnosable. assert!( rendered.contains("s3.secret-access-key"), "keys hidden: {rendered}" diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index be5e276533..ea81ce6ba2 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -228,10 +228,8 @@ impl Transaction { self.table = refreshed.clone(); } - // Actions read and write manifests, so they need the credentials the - // refresh load vended, whether or not the base was stale. Identical - // settings keep the original: a rebuilt FileIO would drop a backend - // the table has already initialized. + // Actions need the credentials the refresh load vended. Identical + // settings keep the original FileIO, whose backend may be initialized. let mut current_table = self.table.clone(); if !current_table.file_io().same_routing_as(refreshed.file_io()) { current_table = current_table.with_file_io(refreshed.file_io().clone()); @@ -261,15 +259,12 @@ impl Transaction { .build(); let committed = catalog.update_table(table_commit).await?; - // A location change moves metadata/data to a new prefix that the refresh - // load's vended credentials do not cover, so it needs a post-commit - // reload. Another writer may have moved the table meanwhile: a - // property-only commit carries no requirement that would catch that. + // A new location is outside the refreshed credentials' prefix, so reload + // after committing. Check the committed metadata: another writer may have + // moved the table, and a property-only commit has no requirement to catch it. let location_changed = moves_location || committed.metadata().location() != refreshed.metadata().location(); if location_changed { - // The new location has its own vended credentials; the reused FileIO is - // scoped to the old prefix, so reload the table to pick them up. match catalog.load_table(committed.identifier()).await { Ok(reloaded) => return Ok(reloaded), // The commit is already durable: returning this error would let the @@ -281,8 +276,7 @@ impl Transaction { } } // The commit response carries no credentials, so keep the FileIO the - // actions ran with: the refreshed one, or the original when its settings - // were unchanged and it holds an initialized backend. + // actions ran with. Ok(committed.with_file_io(current_table.file_io().clone())) } } @@ -501,9 +495,6 @@ mod tests { #[tokio::test] async fn test_commit_keeps_the_refreshed_file_io() { - // The commit response carries no vended credentials, so the FileIO from - // the refresh load has to survive; taking the committed table's own - // would silently drop them. let refreshed = make_v2_table().with_file_io( FileIOBuilder::new(Arc::new(MemoryStorageFactory)) .with_prefixed_props("memory://warehouse", [("s3.access-key-id", "vended")]) @@ -560,8 +551,6 @@ mod tests { } } - /// The refresh load may carry rotated credentials while the metadata is - /// unchanged; actions still have to run with those, not the stale ones. #[tokio::test] async fn test_actions_run_with_the_refreshed_file_io() { let seen = Arc::new(std::sync::Mutex::new(None)); @@ -581,8 +570,8 @@ mod tests { assert_eq!(seen.lock().unwrap().as_deref(), Some("refreshed")); } - /// A reload that changed nothing must not cost the table its initialized - /// backend: `MemoryStorageFactory` builds a fresh, empty store each time. + /// `MemoryStorageFactory` builds an empty store per FileIO, so a rebuilt + /// FileIO loses the data. #[tokio::test] async fn test_unchanged_settings_keep_the_initialized_storage() { let table = table_with_marker("same"); @@ -614,7 +603,6 @@ mod tests { seen.lock().unwrap().as_deref(), Some("written-before-commit") ); - // The returned table keeps that backend too, not the reload's empty one. let after = committed .file_io() .new_input("memory://warehouse/manifest") @@ -663,8 +651,6 @@ mod tests { .unwrap() } - /// The commit is durable before the reload runs, so a failed reload must - /// not send the transaction round the retry loop to be committed again. #[tokio::test] async fn test_a_failed_reload_does_not_replay_the_commit() { let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -674,7 +660,6 @@ mod tests { let n = counter.fetch_add(1, Ordering::SeqCst); Box::pin(async move { if n == 1 { - // The post-commit reload. Err(Error::new(ErrorKind::Unexpected, "503").with_retryable(true)) } else { Ok(make_v2_table()) @@ -697,8 +682,6 @@ mod tests { ); } - /// Another writer moved the table between our refresh and our commit; the - /// committed metadata says so even though we sent no `SetLocation`. #[tokio::test] async fn test_commit_reloads_when_someone_else_moved_the_table() { let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); From 6dae4188743f523e4ede96f5879f64debc76fe46 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 13 Oct 2026 00:03:36 -0700 Subject: [PATCH 20/27] refactor: drop redundant location check and overlapping tests --- crates/catalog/rest/src/catalog.rs | 27 ++++-- crates/iceberg/src/io/file_io.rs | 129 ++------------------------ crates/iceberg/src/transaction/mod.rs | 8 +- 3 files changed, 27 insertions(+), 137 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 85b7461bbd..5f72af1de1 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -1394,13 +1394,16 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `create_table` response!", ))?; - let mut base_config = response.config.clone(); - base_config.extend(self.user_config.props.clone()); + let config = response + .config + .into_iter() + .chain(self.user_config.props.clone()) + .collect(); let file_io = self .load_file_io( Some(metadata_location), - Some(base_config), + Some(config), response.storage_credentials.as_deref(), ) .await?; @@ -1494,13 +1497,16 @@ impl SessionCatalog for RestSessionCatalog { .keys() .any(|key| is_short_lived_credential(key)); - let mut base_config = response.config.clone(); - base_config.extend(self.user_config.props.clone()); + let config = response + .config + .into_iter() + .chain(self.user_config.props.clone()) + .collect(); let file_io = self .load_file_io( response.metadata_location.as_deref(), - Some(base_config), + Some(config), response.storage_credentials.as_deref(), ) .await?; @@ -1653,13 +1659,16 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `register_table` response!", ))?; - let mut base_config = response.config.clone(); - base_config.extend(self.user_config.props.clone()); + let config = response + .config + .into_iter() + .chain(self.user_config.props.clone()) + .collect(); let file_io = self .load_file_io( Some(metadata_location), - Some(base_config), + Some(config), response.storage_credentials.as_deref(), ) .await?; diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 66ed5b609c..d91878e848 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -864,24 +864,17 @@ mod tests { #[tokio::test] async fn test_prefixed_credentials_survive_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prop("s3.access-key-id", "default-key") - .with_prefixed_props("memory://warehouse/t", [("s3.access-key-id", "vended-key")]) + .with_prop("s3.access-key-id", "default") + .with_prefixed_props("memory://w/t", [("s3.access-key-id", "outer")]) + .with_prefixed_props("memory://w/t/nested", [("s3.access-key-id", "inner")]) .build(); let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); - assert_eq!( - deserialized - .config_for("memory://warehouse/t/data/f.parquet") - .get("s3.access-key-id"), - Some(&"vended-key".to_string()) - ); - assert_eq!( - deserialized - .config_for("memory://elsewhere/f.parquet") - .get("s3.access-key-id"), - Some(&"default-key".to_string()) - ); + let key = |p: &str| deserialized.config_for(p).get("s3.access-key-id").cloned(); + assert_eq!(key("memory://w/t/nested/f"), Some("inner".to_string())); + assert_eq!(key("memory://w/t/other/f"), Some("outer".to_string())); + assert_eq!(key("memory://elsewhere/f"), Some("default".to_string())); } /// Stands in for a backend whose Debug prints its raw props. @@ -1036,6 +1029,7 @@ mod tests { let (file_io, calls) = recording_file_io(&[ ("s3://bucket/table/data/", "data"), ("s3://bucket/table/metadata/", "metadata"), + ("s3://bucket/tablex/", "sibling"), ]); file_io.delete_prefix("s3://bucket/table/").await.unwrap(); @@ -1097,20 +1091,6 @@ mod tests { assert!(debug.contains("s3.secret-access-key"), "{debug}"); } - #[tokio::test] - async fn test_overlapping_prefixes_survive_serialization_roundtrip() { - let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prefixed_props("memory://w/t", [("s3.access-key-id", "outer")]) - .with_prefixed_props("memory://w/t/nested", [("s3.access-key-id", "inner")]) - .build(); - - let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); - - let key = |p: &str| deserialized.config_for(p).get("s3.access-key-id").cloned(); - assert_eq!(key("memory://w/t/nested/f"), Some("inner".to_string())); - assert_eq!(key("memory://w/t/other/f"), Some("outer".to_string())); - } - #[tokio::test] async fn test_local_fs_file_io_serialization_roundtrip() { let tmp_dir = TempDir::new().unwrap(); @@ -1168,66 +1148,6 @@ mod tests { )); } - #[tokio::test] - async fn test_prefixed_props_sorted_by_descending_prefix_length() { - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prefixed_props("memory://a/", [("k", "short")]) - .with_prefixed_props("memory://a/longer/", [("k", "long")]) - .build(); - - let prefixes: Vec<&str> = file_io.prefixed.iter().map(|p| p.prefix.as_str()).collect(); - assert_eq!(prefixes, vec!["memory://a/longer/", "memory://a/"]); - } - - #[tokio::test] - async fn test_prefixed_config_carries_credential_values() { - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prop("s3.region", "us-east-1") - .with_prefixed_props("s3://bucket/table", [ - ("s3.region", "us-east-1"), - ("s3.access-key-id", "vended-key"), - ("s3.secret-access-key", "vended-secret"), - ]) - .build(); - - assert_eq!( - file_io.config().get("s3.region"), - Some(&"us-east-1".to_string()) - ); - assert_eq!(file_io.config().get("s3.access-key-id"), None); - - let prefixed = &file_io.prefixed[0].config; - assert_eq!(prefixed.get("s3.region"), Some(&"us-east-1".to_string())); - assert_eq!( - prefixed.get("s3.access-key-id"), - Some(&"vended-key".to_string()) - ); - assert_eq!( - prefixed.get("s3.secret-access-key"), - Some(&"vended-secret".to_string()) - ); - } - - #[tokio::test] - async fn test_get_storage_routes_by_prefix() { - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prop("scope", "default") - .with_prefixed_props("memory://creds/", [("scope", "prefixed")]) - .build(); - - let default_a = file_io.get_storage("memory://other/x").unwrap(); - let default_b = file_io.get_storage("memory://other/y").unwrap(); - let prefixed_a = file_io.get_storage("memory://creds/x").unwrap(); - let prefixed_b = file_io.get_storage("memory://creds/y").unwrap(); - - assert!(Arc::ptr_eq(&default_a, &default_b)); - assert!(Arc::ptr_eq(&prefixed_a, &prefixed_b)); - assert!(!Arc::ptr_eq(&default_a, &prefixed_a)); - } - #[tokio::test] async fn test_delete_stream_routes_by_prefix() { let factory = Arc::new(MemoryStorageFactory); @@ -1259,39 +1179,6 @@ mod tests { assert!(!file_io.exists(prefixed_path).await.unwrap()); } - #[tokio::test] - async fn test_delete_prefix_reaches_nested_prefix_storages() { - let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prefixed_props("memory:/table/data/", [("k", "data")]) - .with_prefixed_props("memory:/table/metadata/", [("k", "metadata")]) - .with_prefixed_props("memory:/tablex/", [("k", "sibling")]) - .build(); - let deleted = [ - "memory:/table/version-hint.text", - "memory:/table/data/a.parquet", - "memory:/table/metadata/v1.json", - ]; - let kept = "memory:/tablex/c.parquet"; - for path in deleted.iter().chain([&kept]) { - file_io - .new_output(path) - .unwrap() - .write("x".into()) - .await - .unwrap(); - } - - file_io.delete_prefix("memory:/table").await.unwrap(); - - let mut remaining = Vec::new(); - for path in deleted.iter().chain([&kept]) { - if file_io.exists(path).await.unwrap() { - remaining.push(*path); - } - } - assert_eq!(remaining, [kept]); - } - #[tokio::test] async fn test_delete_stream_flushes_across_batches() { // More than one flush batch (1000). diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index ea81ce6ba2..1adc0d17b7 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -248,10 +248,6 @@ impl Transaction { )?; } - let moves_location = existing_updates - .iter() - .any(|update| matches!(update, TableUpdate::SetLocation { .. })); - let table_commit = TableCommit::builder() .ident(self.table.identifier().to_owned()) .updates(existing_updates) @@ -262,9 +258,7 @@ impl Transaction { // A new location is outside the refreshed credentials' prefix, so reload // after committing. Check the committed metadata: another writer may have // moved the table, and a property-only commit has no requirement to catch it. - let location_changed = - moves_location || committed.metadata().location() != refreshed.metadata().location(); - if location_changed { + if committed.metadata().location() != refreshed.metadata().location() { match catalog.load_table(committed.identifier()).await { Ok(reloaded) => return Ok(reloaded), // The commit is already durable: returning this error would let the From ebe09fc8913a8f5a36acbdd190da5ffd10af8dc7 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 18 Oct 2026 20:11:53 -0700 Subject: [PATCH 21/27] test(transaction): implement test actions as StatelessAction --- crates/iceberg/src/transaction/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 1adc0d17b7..c09ac0bbf4 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -294,7 +294,7 @@ mod tests { }; use crate::table::{StaticTable, Table}; use crate::test_utils::{make_encrypted_table, test_runtime}; - use crate::transaction::action::{ActionCommit, TransactionAction}; + use crate::transaction::action::{ActionCommit, StatelessAction}; use crate::transaction::{ApplyTransactionAction, Transaction}; use crate::{Catalog, Error, ErrorKind, Result, TableCreation, TableIdent}; @@ -535,11 +535,12 @@ mod tests { } /// Records which FileIO the action was handed. + #[derive(Clone)] struct RecordingAction(Arc>>); #[async_trait] - impl TransactionAction for RecordingAction { - async fn commit(self: Arc, table: &Table) -> Result { + impl StatelessAction for RecordingAction { + async fn commit(&self, table: &Table) -> Result { *self.0.lock().unwrap() = Some(marker_of(table)); Ok(ActionCommit::new(vec![], vec![])) } @@ -608,11 +609,12 @@ mod tests { } /// Reads a file the table wrote before the transaction started. + #[derive(Clone)] struct ReadingAction(Arc>>); #[async_trait] - impl TransactionAction for ReadingAction { - async fn commit(self: Arc, table: &Table) -> Result { + impl StatelessAction for ReadingAction { + async fn commit(&self, table: &Table) -> Result { let read = table .file_io() .new_input("memory://warehouse/manifest")? From 8aac866931a5fb7787713ae60979f2169a4caebc Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 27 Oct 2026 22:14:25 -0700 Subject: [PATCH 22/27] refactor(io): carry vended credentials in StorageConfig for storage factories to apply --- crates/catalog/rest/src/catalog.rs | 305 ++------- .../load_table_response_with_credentials.json | 78 --- crates/iceberg/public-api.txt | 22 +- crates/iceberg/src/io/file_io.rs | 590 ++---------------- crates/iceberg/src/io/object_cache.rs | 11 - crates/iceberg/src/io/storage/config/mod.rs | 138 ++-- crates/iceberg/src/table.rs | 48 -- crates/iceberg/src/transaction/mod.rs | 244 +------- crates/storage/opendal/src/lib.rs | 12 +- crates/storage/opendal/src/prefixed.rs | 373 +++++++++++ crates/storage/opendal/src/resolving.rs | 17 +- .../storage/opendal/tests/file_io_s3_test.rs | 29 +- .../opendal/tests/resolving_storage_test.rs | 28 +- 13 files changed, 674 insertions(+), 1221 deletions(-) delete mode 100644 crates/catalog/rest/testdata/load_table_response_with_credentials.json create mode 100644 crates/storage/opendal/src/prefixed.rs diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 5f72af1de1..73c5f42812 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -28,10 +28,7 @@ use std::time::Duration; use async_trait::async_trait; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - ADLS_ACCOUNT_KEY, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, - ADLS_TENANT_ID, FileIO, FileIOBuilder, GCS_CREDENTIALS_JSON, GCS_TOKEN, HF_TOKEN, - OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, - S3_SESSION_TOKEN, StorageFactory, + ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory, }; use iceberg::table::Table; use iceberg::{ @@ -990,7 +987,7 @@ impl RestSessionCatalog { &self, metadata_location: Option<&str>, extra_config: Option>, - storage_credentials: Option<&[StorageCredential]>, + storage_credentials: Option>, ) -> Result { let mut props = self.client().await?.config.props.clone(); if let Some(config) = extra_config { @@ -1023,67 +1020,17 @@ impl RestSessionCatalog { ) })?; - let mut builder = FileIOBuilder::new(factory).with_props(props.clone()); - - // Each vended prefix gets its own storage; other paths use the - // credential-free default. - if let Some(creds) = storage_credentials { - for cred in creds { - builder = builder.with_prefixed_props( - cred.prefix.clone(), - with_vended_credentials(&props, &cred.config), - ); - } - } + let file_io = FileIOBuilder::new(factory) + .with_props(props) + .with_storage_credentials(storage_credentials.into_iter().flatten().map(|credential| { + iceberg::io::StorageCredential::new(credential.prefix, credential.config) + })) + .build(); - Ok(builder.build()) + Ok(file_io) } } -/// Storage credential properties that only work together. A key also covers -/// its scoped variants, such as `adls.sas-token.`. -const CREDENTIAL_SETS: [&[&str]; 5] = [ - &[S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN], - &[OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET], - &[GCS_CREDENTIALS_JSON, GCS_TOKEN], - &[ - ADLS_CONNECTION_STRING, - ADLS_ACCOUNT_KEY, - ADLS_SAS_TOKEN, - ADLS_TENANT_ID, - ADLS_CLIENT_ID, - ADLS_CLIENT_SECRET, - ], - &[HF_TOKEN], -]; - -fn in_credential_set(key: &str, set: &[&str]) -> bool { - set.iter().any(|name| { - key.strip_prefix(name) - .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) - }) -} - -/// `base` overlaid with a vended `config`. Each credential set the vended -/// config touches replaces the inherited one whole, so credentials from two -/// identities never mix; other settings, such as the endpoint, are inherited. -fn with_vended_credentials( - base: &HashMap, - config: &HashMap, -) -> HashMap { - let replaced: Vec<&[&str]> = CREDENTIAL_SETS - .into_iter() - .filter(|set| config.keys().any(|key| in_credential_set(key, set))) - .collect(); - let mut props: HashMap = base - .iter() - .filter(|(key, _)| !replaced.iter().any(|set| in_credential_set(key, set))) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - props.extend(config.clone()); - props -} - /// All requests and expected responses are derived from the REST catalog API spec: /// #[async_trait] @@ -1404,7 +1351,7 @@ impl SessionCatalog for RestSessionCatalog { .load_file_io( Some(metadata_location), Some(config), - response.storage_credentials.as_deref(), + response.storage_credentials, ) .await?; @@ -1441,8 +1388,6 @@ impl SessionCatalog for RestSessionCatalog { }); let cached = cache.as_ref().and_then(|(cache, key)| cache.get(key)); - // Vended credentials are opt-in through a configured - // `header.X-Iceberg-Access-Delegation`, as in Java. let mut request_builder = client .http_client .request(Method::GET, client.config.table_endpoint(table_ident)); @@ -1507,7 +1452,7 @@ impl SessionCatalog for RestSessionCatalog { .load_file_io( response.metadata_location.as_deref(), Some(config), - response.storage_credentials.as_deref(), + response.storage_credentials, ) .await?; @@ -1669,7 +1614,7 @@ impl SessionCatalog for RestSessionCatalog { .load_file_io( Some(metadata_location), Some(config), - response.storage_credentials.as_deref(), + response.storage_credentials, ) .await?; @@ -1750,8 +1695,6 @@ impl SessionCatalog for RestSessionCatalog { } }; - // The commit response carries no credentials. `Transaction::do_commit`, - // the only possible caller, keeps the FileIO from its pre-commit load. let file_io = self .load_file_io(Some(&response.metadata_location), None, None) .await?; @@ -4844,6 +4787,49 @@ mod tests { load.assert_async().await; } + /// A `loadTable` body vending a storage credential, and that credential. + fn load_table_body_with_credential() -> (serde_json::Value, iceberg::io::StorageCredential) { + let credential = iceberg::io::StorageCredential::new( + "s3://warehouse/database/table", + HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), + ); + let mut body = load_table_body("v1"); + body["storage-credentials"] = + json!([{ "prefix": credential.prefix(), "config": credential.config() }]); + (body, credential) + } + + #[tokio::test] + async fn test_load_table_passes_vended_credentials_to_file_io() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let (body, credential) = load_table_body_with_credential(); + let load = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("x-iceberg-access-delegation", "vended-credentials") + .with_status(200) + .with_body(body.to_string()) + .create_async() + .await; + let catalog = session_catalog( + RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.X-Iceberg-Access-Delegation".to_string(), + "vended-credentials".to_string(), + )])) + .build(), + ); + + let table = catalog + .load_table(&SessionContext::empty(), &test_table_ident()) + .await + .unwrap(); + + assert_eq!(table.file_io().config().storage_credentials(), [credential]); + load.assert_async().await; + } + #[test] fn test_table_cache_rejects_out_of_range_expiry() { for value in ["0", "31536000000001"] { @@ -4937,143 +4923,6 @@ mod tests { rename_table_mock.assert_async().await; } - #[tokio::test] - async fn test_load_table_uses_vended_credentials() { - let mut server = Server::new_async().await; - - let config_mock = create_config_mock(&mut server).await; - - let load_table_mock = server - .mock("GET", "/v1/namespaces/ns1/tables/test1") - .match_header("x-iceberg-access-delegation", "vended-credentials") - .with_status(200) - .with_body_from_file(format!( - "{}/testdata/{}", - env!("CARGO_MANIFEST_DIR"), - "load_table_response_with_credentials.json" - )) - .create_async() - .await; - - let props = HashMap::from([( - "header.X-Iceberg-Access-Delegation".to_string(), - "vended-credentials".to_string(), - )]); - let catalog = RestCatalog::new( - SessionContext::empty(), - RestCatalogConfig::builder() - .uri(server.url()) - .props(props) - .build(), - None, - Some(Arc::new(LocalFsStorageFactory)), - Runtime::current(), - None, - ); - - let table = catalog - .load_table(&TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .await - .unwrap(); - - assert_eq!( - "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", - table.metadata_location().unwrap() - ); - - let file_io = table.file_io(); - let vended = file_io.config_for("s3://warehouse/database/table/data/f.parquet"); - assert_eq!( - vended.get("s3.access-key-id"), - Some(&"vended-key-id".to_string()) - ); - assert_eq!( - vended.get("s3.session-token"), - Some(&"vended-token".to_string()) - ); - assert_eq!( - file_io - .config_for("s3://warehouse/other/f.parquet") - .get("s3.access-key-id"), - None - ); - - config_mock.assert_async().await; - load_table_mock.assert_async().await; - } - - #[test] - fn test_vended_credentials_replace_inherited_credential_sets() { - let props = |pairs: &[(&str, &str)]| -> HashMap { - pairs - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() - }; - let base = props(&[ - ("s3.endpoint", "https://s3.example.com"), - ("client.region", "us-east-1"), - ("s3.access-key-id", "catalog-key"), - ("s3.secret-access-key", "catalog-secret"), - ("s3.session-token", "catalog-token"), - ("adls.sas-token.account.dfs.core.windows.net", "catalog-sas"), - ("gcs.oauth2.token", "catalog-gcs-token"), - ]); - let vended = props(&[ - ("s3.access-key-id", "vended-key"), - ("s3.secret-access-key", "vended-secret"), - ("adls.account-key", "vended-account-key"), - ]); - - assert_eq!( - with_vended_credentials(&base, &vended), - props(&[ - ("s3.endpoint", "https://s3.example.com"), - ("client.region", "us-east-1"), - ("s3.access-key-id", "vended-key"), - ("s3.secret-access-key", "vended-secret"), - ("adls.account-key", "vended-account-key"), - ("gcs.oauth2.token", "catalog-gcs-token"), - ]) - ); - } - - #[tokio::test] - async fn test_load_table_omits_delegation_header_by_default() { - let mut server = Server::new_async().await; - - let config_mock = create_config_mock(&mut server).await; - - let load_table_mock = server - .mock("GET", "/v1/namespaces/ns1/tables/test1") - .match_header("x-iceberg-access-delegation", mockito::Matcher::Missing) - .with_status(200) - .with_body_from_file(format!( - "{}/testdata/{}", - env!("CARGO_MANIFEST_DIR"), - "load_table_response.json" - )) - .create_async() - .await; - - let catalog = RestCatalog::new( - SessionContext::empty(), - RestCatalogConfig::builder().uri(server.url()).build(), - None, - Some(Arc::new(LocalFsStorageFactory)), - Runtime::current(), - None, - ); - - catalog - .load_table(&TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .await - .unwrap(); - - config_mock.assert_async().await; - load_table_mock.assert_async().await; - } - #[tokio::test] async fn test_create_table() { let mut server = Server::new_async().await; @@ -5290,7 +5139,6 @@ mod tests { let config_mock = create_config_mock(&mut server).await; - // GET hit once: the transaction refreshes the table before committing. let load_table_mock = server .mock("GET", "/v1/namespaces/ns1/tables/test1") .with_status(200) @@ -5299,7 +5147,6 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "load_table_response.json" )) - .expect(1) .create_async() .await; @@ -5557,49 +5404,31 @@ mod tests { } #[tokio::test] - async fn test_register_table_uses_vended_credentials() { + async fn test_register_table_passes_vended_config_and_credentials_to_file_io() { let mut server = Server::new_async().await; - let config_mock = create_config_mock(&mut server).await; - let register_table_mock = server + let _config = create_config_mock(&mut server).await; + let (body, credential) = load_table_body_with_credential(); + let register = server .mock("POST", "/v1/namespaces/ns1/register") .with_status(200) - .with_body_from_file(format!( - "{}/testdata/{}", - env!("CARGO_MANIFEST_DIR"), - "load_table_response_with_credentials.json" - )) + .with_body(body.to_string()) .create_async() .await; - let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); - let table_ident = TableIdent::from_strs(["ns1", "test1"]).unwrap(); + let table = catalog .register_table( &SessionContext::empty(), - &table_ident, - "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json".to_string(), + &test_table_ident(), + "s3://warehouse/v1.metadata.json".to_string(), ) .await .unwrap(); - let vended = table - .file_io() - .config_for("s3://warehouse/database/table/data/f.parquet"); - assert_eq!( - vended.get("s3.access-key-id").map(String::as_str), - Some("vended-key-id") - ); - let outside = table - .file_io() - .config_for("s3://other-bucket/data/f.parquet"); - assert_eq!(outside.get("s3.access-key-id"), None); - assert_eq!( - table.file_io().config().get("region").map(String::as_str), - Some("us-west-2") - ); - - config_mock.assert_async().await; - register_table_mock.assert_async().await; + let config = table.file_io().config(); + assert_eq!(config.get("region").map(String::as_str), Some("us-west-2")); + assert_eq!(config.storage_credentials(), [credential]); + register.assert_async().await; } #[tokio::test] diff --git a/crates/catalog/rest/testdata/load_table_response_with_credentials.json b/crates/catalog/rest/testdata/load_table_response_with_credentials.json deleted file mode 100644 index c0a540fbac..0000000000 --- a/crates/catalog/rest/testdata/load_table_response_with_credentials.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "metadata-location": "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", - "metadata": { - "format-version": 1, - "table-uuid": "b55d9dda-6561-423a-8bfc-787980ce421f", - "location": "s3://warehouse/database/table", - "last-updated-ms": 1646787054459, - "last-column-id": 2, - "schema": { - "type": "struct", - "schema-id": 0, - "fields": [ - {"id": 1, "name": "id", "required": false, "type": "int"}, - {"id": 2, "name": "data", "required": false, "type": "string"} - ] - }, - "current-schema-id": 0, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - {"id": 1, "name": "id", "required": false, "type": "int"}, - {"id": 2, "name": "data", "required": false, "type": "string"} - ] - } - ], - "partition-spec": [], - "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": []}], - "last-partition-id": 999, - "default-sort-order-id": 0, - "sort-orders": [{"order-id": 0, "fields": []}], - "properties": {"owner": "bryan", "write.metadata.compression-codec": "gzip"}, - "current-snapshot-id": 3497810964824022504, - "refs": {"main": {"snapshot-id": 3497810964824022504, "type": "branch"}}, - "snapshots": [ - { - "snapshot-id": 3497810964824022504, - "timestamp-ms": 1646787054459, - "summary": { - "operation": "append", - "spark.app.id": "local-1646787004168", - "added-data-files": "1", - "added-records": "1", - "added-files-size": "697", - "changed-partition-count": "1", - "total-records": "1", - "total-files-size": "697", - "total-data-files": "1", - "total-delete-files": "0", - "total-position-deletes": "0", - "total-equality-deletes": "0" - }, - "manifest-list": "s3://warehouse/database/table/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", - "schema-id": 0 - } - ], - "snapshot-log": [{"timestamp-ms": 1646787054459, "snapshot-id": 3497810964824022504}], - "metadata-log": [ - { - "timestamp-ms": 1646787031514, - "metadata-file": "s3://warehouse/database/table/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json" - } - ] - }, - "config": {"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, - "storage-credentials": [ - { - "prefix": "s3://warehouse/database/table", - "config": { - "s3.access-key-id": "vended-key-id", - "s3.secret-access-key": "vended-secret", - "s3.session-token": "vended-token" - } - } - ] -} diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 5110f74366..71b7f20010 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -859,7 +859,6 @@ pub fn iceberg::io::AzdlsConfig::deserialize<__D>(__deserializer: __D) -> core:: pub struct iceberg::io::FileIO impl iceberg::io::FileIO pub fn iceberg::io::FileIO::config(&self) -> &iceberg::io::StorageConfig -pub fn iceberg::io::FileIO::config_for(&self, path: &str) -> &iceberg::io::StorageConfig pub async fn iceberg::io::FileIO::delete(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> @@ -879,9 +878,9 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self -pub fn iceberg::io::FileIOBuilder::with_prefixed_props(self, prefix: impl core::convert::Into, props: impl core::iter::traits::collect::IntoIterator) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::io::FileIOBuilder::with_storage_credentials(self, credentials: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::clone(&self) -> iceberg::io::FileIOBuilder impl core::fmt::Debug for iceberg::io::FileIOBuilder @@ -1102,8 +1101,10 @@ pub fn iceberg::io::StorageConfig::from_props(props: std::collections::hash::map pub fn iceberg::io::StorageConfig::get(&self, key: &str) -> core::option::Option<&alloc::string::String> pub fn iceberg::io::StorageConfig::new() -> Self pub fn iceberg::io::StorageConfig::props(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageConfig::storage_credentials(&self) -> &[iceberg::io::StorageCredential] pub fn iceberg::io::StorageConfig::with_prop(self, key: impl core::convert::Into, value: impl core::convert::Into) -> Self pub fn iceberg::io::StorageConfig::with_props(self, props: impl core::iter::traits::collect::IntoIterator, impl core::convert::Into)>) -> Self +pub fn iceberg::io::StorageConfig::with_storage_credentials(self, credentials: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::clone(&self) -> iceberg::io::StorageConfig impl core::cmp::Eq for iceberg::io::StorageConfig @@ -1133,6 +1134,23 @@ impl serde_core::ser::Serialize for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::StorageCredential +impl iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::io::StorageCredential::prefix(&self) -> &str +impl core::clone::Clone for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential +impl core::cmp::Eq for iceberg::io::StorageCredential +impl core::cmp::PartialEq for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::eq(&self, other: &iceberg::io::StorageCredential) -> bool +impl core::fmt::Debug for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential +impl serde_core::ser::Serialize for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub const iceberg::io::ADLS_ACCOUNT_KEY: &str pub const iceberg::io::ADLS_ACCOUNT_NAME: &str pub const iceberg::io::ADLS_AUTHORITY_HOST: &str diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index d91878e848..ce0775bb56 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -15,17 +15,17 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::ops::Range; use std::sync::{Arc, OnceLock}; use bytes::Bytes; -use futures::{Stream, StreamExt, stream}; +use futures::{Stream, StreamExt}; use super::storage::{ - LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, + LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageCredential, + StorageFactory, }; -use crate::{Error, ErrorKind, Result}; +use crate::Result; /// FileIO implementation, used to manipulate files in underlying storage. /// @@ -60,7 +60,7 @@ use crate::{Error, ErrorKind, Result}; /// .with_prop("key", "value") /// .build(); /// ``` -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct FileIO { /// Storage configuration containing properties config: StorageConfig, @@ -68,37 +68,6 @@ pub struct FileIO { factory: Arc, /// Cached storage instance (lazily initialized) storage: Arc>>, - /// Per-prefix storages, longest prefix first. Paths matching none use - /// `storage`. - prefixed: Arc>, -} - -/// A storage scoped to a location `prefix`, lazily built from its own config. -struct PrefixedStorage { - prefix: String, - config: StorageConfig, - storage: OnceLock>, -} - -// Neither cached storage is shown: a backend's Debug may print raw -// credentials. -impl std::fmt::Debug for FileIO { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FileIO") - .field("config", &self.config) - .field("factory", &self.factory) - .field("prefixed", &self.prefixed) - .finish_non_exhaustive() - } -} - -impl std::fmt::Debug for PrefixedStorage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PrefixedStorage") - .field("prefix", &self.prefix) - .field("config", &self.config) - .finish_non_exhaustive() - } } mod _serde { @@ -112,28 +81,12 @@ mod _serde { pub(super) struct SerializableFileIO<'a> { pub(super) config: &'a StorageConfig, pub(super) factory: &'a Arc, - /// Per-prefix credentials, so a deserialized FileIO reads the same data. - pub(super) prefixed: Vec>, - } - - #[derive(Serialize)] - pub(super) struct SerializablePrefixed<'a> { - pub(super) prefix: &'a str, - pub(super) config: &'a StorageConfig, } #[derive(Deserialize)] pub(super) struct DeserializedFileIO { pub(super) config: StorageConfig, pub(super) factory: Arc, - #[serde(default)] - pub(super) prefixed: Vec, - } - - #[derive(Deserialize)] - pub(super) struct DeserializedPrefixed { - pub(super) prefix: String, - pub(super) config: StorageConfig, } } @@ -146,7 +99,6 @@ impl FileIO { config: StorageConfig::new(), factory: Arc::new(MemoryStorageFactory), storage: Arc::new(OnceLock::new()), - prefixed: Arc::new(Vec::new()), } } @@ -158,7 +110,6 @@ impl FileIO { config: StorageConfig::new(), factory: Arc::new(LocalFsStorageFactory), storage: Arc::new(OnceLock::new()), - prefixed: Arc::new(Vec::new()), } } @@ -181,14 +132,6 @@ impl FileIO { Ok(serde_json::to_vec(&_serde::SerializableFileIO { config: &self.config, factory: &self.factory, - prefixed: self - .prefixed - .iter() - .map(|p| _serde::SerializablePrefixed { - prefix: &p.prefix, - config: &p.config, - }) - .collect(), })?) } @@ -198,25 +141,11 @@ impl FileIO { /// implementation so it is registered with `typetag`. Backend-specific requirements are /// documented by each storage factory implementation. pub fn deserialize_all(bytes: &[u8]) -> Result { - let _serde::DeserializedFileIO { - config, - factory, - prefixed, - } = serde_json::from_slice(bytes)?; - // `serialize_all` kept the longest-first order. - let prefixed = prefixed - .into_iter() - .map(|p| PrefixedStorage { - prefix: p.prefix, - config: p.config, - storage: OnceLock::new(), - }) - .collect(); + let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; Ok(Self { config, factory, storage: Arc::new(OnceLock::new()), - prefixed: Arc::new(prefixed), }) } @@ -225,57 +154,24 @@ impl FileIO { &self.config } - /// Whether `other` would route and authenticate identically. Used to keep - /// an initialized backend when a reload brought back the same settings. - pub(crate) fn same_routing_as(&self, other: &FileIO) -> bool { - self.config == other.config - && self.prefixed.len() == other.prefixed.len() - && self - .prefixed - .iter() - .zip(other.prefixed.iter()) - .all(|(a, b)| a.prefix == b.prefix && a.config == b.config) - } - - /// The configuration `path` routes to: the longest matching prefix's, - /// else the default. - pub fn config_for(&self, path: &str) -> &StorageConfig { - self.route(path).1 - } - - /// The storage for `path`, built on first use. - fn get_storage(&self, path: &str) -> Result> { - let (cell, config) = self.route(path); - Self::get_or_build(cell, &self.factory, config) - } - - /// The storage cell and configuration serving `path`: the first, so - /// longest, matching prefix, else the default. + /// Get or create the storage instance. /// - /// Matching is on the raw string, so `s3://bucket/data` also serves - /// `s3://bucket/database/`, as in Java's `S3FileIO.clientForStoragePath`. - fn route(&self, path: &str) -> (&OnceLock>, &StorageConfig) { - for ps in self.prefixed.iter() { - if path.starts_with(&ps.prefix) { - return (&ps.storage, &ps.config); - } - } - (&self.storage, &self.config) - } - - /// Get a cached storage from `cell`, building it from `config` on first use. - fn get_or_build( - cell: &OnceLock>, - factory: &Arc, - config: &StorageConfig, - ) -> Result> { - if let Some(storage) = cell.get() { + /// The factory is invoked on first access and the result is cached + /// for all subsequent operations. + fn get_storage(&self) -> Result> { + // Check if already initialized + if let Some(storage) = self.storage.get() { return Ok(storage.clone()); } - let storage = factory.build(config)?; - // Another thread might have set it first; keep whatever ends up in the cell. - let _ = cell.set(storage); - Ok(cell.get().unwrap().clone()) + + // Build the storage + let storage = self.factory.build(&self.config)?; + + // Try to set it (another thread might have set it first) + let _ = self.storage.set(storage.clone()); + + // Return whatever is in the cell (either ours or another thread's) + Ok(self.storage.get().unwrap().clone()) } /// Deletes file. @@ -284,7 +180,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub async fn delete(&self, path: impl AsRef) -> Result<()> { - self.get_storage(path.as_ref())?.delete(path.as_ref()).await + self.get_storage()?.delete(path.as_ref()).await } /// Remove the path and all nested dirs and files recursively. @@ -298,46 +194,8 @@ impl FileIO { /// - If the path is a file or not exist, this function will be no-op. /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. - /// - Files routed to a deeper per-prefix storage are deleted through it, within its own prefix. - /// - /// # Errors - /// - /// Returns [`ErrorKind::FeatureUnsupported`], before deleting anything, if a per-prefix - /// storage beneath the path has a prefix not ending in `/`, which cannot be deleted as a - /// directory. pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { - let path = path.as_ref(); - let dir = if path.ends_with('/') { - path.to_string() - } else { - format!("{path}/") - }; - // Credentials scoped to a prefix may not reach beyond it. - let nested: Vec<&PrefixedStorage> = self - .prefixed - .iter() - .filter(|ps| ps.prefix.len() > dir.len() && ps.prefix.starts_with(&dir)) - .collect(); - if let Some(ps) = nested.iter().find(|ps| !ps.prefix.ends_with('/')) { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - format!( - "cannot delete `{path}` recursively: the credential prefix `{}` beneath it \ - is not a directory", - ps.prefix - ), - )); - } - for ps in nested { - Self::get_or_build(&ps.storage, &self.factory, &ps.config)? - .delete_prefix(&ps.prefix) - .await?; - } - // Route by the directory, so `table` and `table/` reach the same storage. - let (cell, config) = self.route(&dir); - Self::get_or_build(cell, &self.factory, config)? - .delete_prefix(path) - .await + self.get_storage()?.delete_prefix(path.as_ref()).await } /// Delete multiple files from a stream of paths. @@ -349,40 +207,7 @@ impl FileIO { &self, paths: impl Stream + Send + 'static, ) -> Result<()> { - if self.prefixed.is_empty() { - return self.get_storage("")?.delete_stream(paths.boxed()).await; - } - - // Flush bounded batches per prefix, as Java's `S3FileIO.deleteFiles` does. - const DELETE_BATCH_SIZE: usize = 1000; - let mut groups: HashMap> = HashMap::new(); - let mut paths = paths.boxed(); - while let Some(path) = paths.next().await { - let key = self - .prefixed - .iter() - .find(|ps| path.starts_with(&ps.prefix)) - .map(|ps| ps.prefix.clone()) - .unwrap_or_default(); - let buf = groups.entry(key).or_default(); - buf.push(path); - if buf.len() >= DELETE_BATCH_SIZE { - let full = std::mem::take(buf); - self.get_storage(&full[0])? - .delete_stream(stream::iter(full).boxed()) - .await?; - } - } - - for batch in groups.into_values() { - if batch.is_empty() { - continue; - } - self.get_storage(&batch[0])? - .delete_stream(stream::iter(batch).boxed()) - .await?; - } - Ok(()) + self.get_storage()?.delete_stream(paths.boxed()).await } /// Check file exists. @@ -391,7 +216,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub async fn exists(&self, path: impl AsRef) -> Result { - self.get_storage(path.as_ref())?.exists(path.as_ref()).await + self.get_storage()?.exists(path.as_ref()).await } /// Creates input file. @@ -400,7 +225,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_input(&self, path: impl AsRef) -> Result { - self.get_storage(path.as_ref())?.new_input(path.as_ref()) + self.get_storage()?.new_input(path.as_ref()) } /// Creates output file. @@ -409,7 +234,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_output(&self, path: impl AsRef) -> Result { - self.get_storage(path.as_ref())?.new_output(path.as_ref()) + self.get_storage()?.new_output(path.as_ref()) } } @@ -423,8 +248,6 @@ pub struct FileIOBuilder { factory: Arc, /// Storage configuration config: StorageConfig, - /// Per-location-prefix configs (prefix, config). - prefixed: Vec<(String, StorageConfig)>, } impl FileIOBuilder { @@ -433,7 +256,6 @@ impl FileIOBuilder { Self { factory, config: StorageConfig::new(), - prefixed: Vec::new(), } } @@ -454,20 +276,14 @@ impl FileIOBuilder { self } - /// Add a per-prefix storage config. Paths starting with `prefix` (longest - /// match wins) use these props instead of the default config. - pub fn with_prefixed_props( + /// Add credentials that apply to locations under their prefixes. + /// + /// See [`StorageConfig::with_storage_credentials`]. + pub fn with_storage_credentials( mut self, - prefix: impl Into, - props: impl IntoIterator, + credentials: impl IntoIterator, ) -> Self { - let config = StorageConfig::from_props( - props - .into_iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - ); - self.prefixed.push((prefix.into(), config)); + self.config = self.config.with_storage_credentials(credentials); self } @@ -478,22 +294,10 @@ impl FileIOBuilder { /// Builds [`FileIO`]. pub fn build(self) -> FileIO { - let mut prefixed: Vec = self - .prefixed - .into_iter() - .map(|(prefix, config)| PrefixedStorage { - prefix, - config, - storage: OnceLock::new(), - }) - .collect(); - // Longest prefix first so routing picks the most specific match. - prefixed.sort_by_key(|item| std::cmp::Reverse(item.prefix.len())); FileIO { config: self.config, factory: self.factory, storage: Arc::new(OnceLock::new()), - prefixed: Arc::new(prefixed), } } } @@ -658,17 +462,10 @@ mod tests { use bytes::Bytes; use futures::AsyncReadExt; use futures::io::AllowStdIo; - use futures::stream::BoxStream; - use serde::{Deserialize, Serialize}; use tempfile::TempDir; - use super::{ - FileIO, FileIOBuilder, InputFile, OutputFile, Storage, StorageConfig, StorageFactory, - }; - use crate::io::{ - FileMetadata, FileRead, FileWrite, LocalFsStorageFactory, MemoryStorageFactory, - }; - use crate::{ErrorKind, Result}; + use super::{FileIO, FileIOBuilder}; + use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -861,236 +658,6 @@ mod tests { assert!(deserialized.storage.get().is_some()); } - #[tokio::test] - async fn test_prefixed_credentials_survive_serialization_roundtrip() { - let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prop("s3.access-key-id", "default") - .with_prefixed_props("memory://w/t", [("s3.access-key-id", "outer")]) - .with_prefixed_props("memory://w/t/nested", [("s3.access-key-id", "inner")]) - .build(); - - let deserialized = FileIO::deserialize_all(&file_io.serialize_all().unwrap()).unwrap(); - - let key = |p: &str| deserialized.config_for(p).get("s3.access-key-id").cloned(); - assert_eq!(key("memory://w/t/nested/f"), Some("inner".to_string())); - assert_eq!(key("memory://w/t/other/f"), Some("outer".to_string())); - assert_eq!(key("memory://elsewhere/f"), Some("default".to_string())); - } - - /// Stands in for a backend whose Debug prints its raw props. - #[derive(Serialize, Deserialize)] - struct LeakyStorage(String); - - impl std::fmt::Debug for LeakyStorage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "LeakyStorage {{ secret: {} }}", self.0) - } - } - - #[typetag::serde] - #[async_trait::async_trait] - impl Storage for LeakyStorage { - async fn exists(&self, _: &str) -> Result { - Ok(false) - } - async fn metadata(&self, _: &str) -> Result { - unimplemented!() - } - async fn read(&self, _: &str) -> Result { - unimplemented!() - } - async fn reader(&self, _: &str) -> Result> { - unimplemented!() - } - async fn write(&self, _: &str, _: Bytes) -> Result<()> { - unimplemented!() - } - async fn writer(&self, _: &str) -> Result> { - unimplemented!() - } - async fn delete(&self, _: &str) -> Result<()> { - unimplemented!() - } - async fn delete_prefix(&self, _: &str) -> Result<()> { - unimplemented!() - } - async fn delete_stream(&self, _: BoxStream<'static, String>) -> Result<()> { - unimplemented!() - } - fn new_input(&self, _: &str) -> Result { - unimplemented!() - } - fn new_output(&self, _: &str) -> Result { - unimplemented!() - } - } - - #[derive(Debug, Serialize, Deserialize)] - struct LeakyFactory; - - #[typetag::serde] - impl StorageFactory for LeakyFactory { - fn build(&self, config: &StorageConfig) -> Result> { - let secret = config - .get("s3.secret-access-key") - .cloned() - .unwrap_or_default(); - Ok(Arc::new(LeakyStorage(secret))) - } - } - - type DeleteLog = Arc>>; - - /// Records the `delete_prefix` calls of every storage it builds, tagged - /// with the storage's `scope` property. - #[derive(Debug, Default, Serialize, Deserialize)] - struct RecordingFactory(#[serde(skip)] DeleteLog); - - #[derive(Debug, Serialize, Deserialize)] - struct RecordingStorage { - scope: String, - #[serde(skip)] - calls: DeleteLog, - } - - #[typetag::serde] - #[async_trait::async_trait] - impl Storage for RecordingStorage { - async fn exists(&self, _: &str) -> Result { - unimplemented!() - } - async fn metadata(&self, _: &str) -> Result { - unimplemented!() - } - async fn read(&self, _: &str) -> Result { - unimplemented!() - } - async fn reader(&self, _: &str) -> Result> { - unimplemented!() - } - async fn write(&self, _: &str, _: Bytes) -> Result<()> { - unimplemented!() - } - async fn writer(&self, _: &str) -> Result> { - unimplemented!() - } - async fn delete(&self, _: &str) -> Result<()> { - unimplemented!() - } - async fn delete_prefix(&self, path: &str) -> Result<()> { - self.calls - .lock() - .unwrap() - .push((self.scope.clone(), path.to_string())); - Ok(()) - } - async fn delete_stream(&self, _: BoxStream<'static, String>) -> Result<()> { - unimplemented!() - } - fn new_input(&self, _: &str) -> Result { - unimplemented!() - } - fn new_output(&self, _: &str) -> Result { - unimplemented!() - } - } - - #[typetag::serde] - impl StorageFactory for RecordingFactory { - fn build(&self, config: &StorageConfig) -> Result> { - Ok(Arc::new(RecordingStorage { - scope: config.get("scope").cloned().unwrap_or_default(), - calls: self.0.clone(), - })) - } - } - - /// A FileIO over a [`RecordingFactory`] with `prefixes` as (prefix, scope), - /// and the log its storages record into. - fn recording_file_io(prefixes: &[(&str, &str)]) -> (FileIO, DeleteLog) { - let factory = RecordingFactory::default(); - let calls = factory.0.clone(); - let mut builder = FileIOBuilder::new(Arc::new(factory)).with_prop("scope", "default"); - for (prefix, scope) in prefixes { - builder = builder.with_prefixed_props(*prefix, [("scope", *scope)]); - } - (builder.build(), calls) - } - - fn recorded(calls: &[(&str, &str)]) -> Vec<(String, String)> { - calls - .iter() - .map(|(scope, path)| (scope.to_string(), path.to_string())) - .collect() - } - - #[tokio::test] - async fn test_delete_prefix_keeps_each_storage_within_its_prefix() { - let (file_io, calls) = recording_file_io(&[ - ("s3://bucket/table/data/", "data"), - ("s3://bucket/table/metadata/", "metadata"), - ("s3://bucket/tablex/", "sibling"), - ]); - - file_io.delete_prefix("s3://bucket/table/").await.unwrap(); - - assert_eq!( - *calls.lock().unwrap(), - recorded(&[ - ("metadata", "s3://bucket/table/metadata/"), - ("data", "s3://bucket/table/data/"), - ("default", "s3://bucket/table/"), - ]) - ); - } - - #[tokio::test] - async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash() { - for path in ["s3://bucket/table", "s3://bucket/table/"] { - let (file_io, calls) = recording_file_io(&[("s3://bucket/table/", "vended")]); - - file_io.delete_prefix(path).await.unwrap(); - - assert_eq!(*calls.lock().unwrap(), recorded(&[("vended", path)])); - } - } - - #[tokio::test] - async fn test_delete_prefix_rejects_a_nested_raw_prefix() { - let (file_io, calls) = recording_file_io(&[ - ("s3://bucket/table/data/", "data"), - ("s3://bucket/table/meta", "meta"), - ]); - - let err = file_io - .delete_prefix("s3://bucket/table/") - .await - .unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); - assert!(err.message().contains("s3://bucket/table/meta"), "{err}"); - assert!(calls.lock().unwrap().is_empty()); - } - - #[tokio::test] - async fn test_debug_omits_initialized_storages() { - let file_io = FileIOBuilder::new(Arc::new(LeakyFactory)) - .with_prop("s3.secret-access-key", "DEFAULT-SECRET") - .with_prefixed_props("memory://warehouse/t", [( - "s3.secret-access-key", - "VENDED-SECRET", - )]) - .build(); - file_io.exists("memory://elsewhere/f").await.unwrap(); - file_io.exists("memory://warehouse/t/f").await.unwrap(); - - let debug = format!("{file_io:?}"); - assert!(!debug.contains("VENDED-SECRET"), "{debug}"); - assert!(!debug.contains("DEFAULT-SECRET"), "{debug}"); - assert!(debug.contains("memory://warehouse/t"), "{debug}"); - assert!(debug.contains("s3.secret-access-key"), "{debug}"); - } - #[tokio::test] async fn test_local_fs_file_io_serialization_roundtrip() { let tmp_dir = TempDir::new().unwrap(); @@ -1122,91 +689,4 @@ mod tests { ); assert!(deserialized.storage.get().is_some()); } - - #[tokio::test] - async fn test_routes_a_path_to_its_longest_matching_prefix() { - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prefixed_props("memory://bucket/data", [("k", "short")]) - .with_prefixed_props("memory://bucket/data/warehouse", [("k", "long")]) - .build(); - - let long = file_io - .get_storage("memory://bucket/data/warehouse/t/f") - .unwrap(); - let short = file_io.get_storage("memory://bucket/data/other/f").unwrap(); - let default = file_io.get_storage("memory://elsewhere/f").unwrap(); - - assert!(!Arc::ptr_eq(&long, &short)); - assert!(!Arc::ptr_eq(&short, &default)); - assert!(!Arc::ptr_eq(&long, &default)); - assert!(Arc::ptr_eq( - &long, - &file_io - .get_storage("memory://bucket/data/warehouse/other") - .unwrap() - )); - } - - #[tokio::test] - async fn test_delete_stream_routes_by_prefix() { - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prefixed_props("memory:/creds/", [("k", "v")]) - .build(); - - let default_path = "memory:/other/a.txt"; - let prefixed_path = "memory:/creds/b.txt"; - for path in [default_path, prefixed_path] { - file_io - .new_output(path) - .unwrap() - .write("x".into()) - .await - .unwrap(); - assert!(file_io.exists(path).await.unwrap()); - } - - file_io - .delete_stream(futures::stream::iter(vec![ - default_path.to_string(), - prefixed_path.to_string(), - ])) - .await - .unwrap(); - - assert!(!file_io.exists(default_path).await.unwrap()); - assert!(!file_io.exists(prefixed_path).await.unwrap()); - } - - #[tokio::test] - async fn test_delete_stream_flushes_across_batches() { - // More than one flush batch (1000). - let factory = Arc::new(MemoryStorageFactory); - let file_io = FileIOBuilder::new(factory) - .with_prefixed_props("memory:/creds/", [("k", "v")]) - .build(); - - let n = 1050; - let mut paths = Vec::with_capacity(n); - for i in 0..n { - let p = format!("memory:/creds/f{i}.txt"); - file_io - .new_output(&p) - .unwrap() - .write("x".into()) - .await - .unwrap(); - paths.push(p); - } - - file_io - .delete_stream(futures::stream::iter(paths.clone())) - .await - .unwrap(); - - for p in &paths { - assert!(!file_io.exists(p).await.unwrap()); - } - } } diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 07902a74f7..37ded38946 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -126,17 +126,6 @@ impl ObjectCache { })) } - /// Returns a cache that uses `file_io` for future cache misses. - pub(crate) fn with_file_io(mut self, file_io: FileIO) -> Self { - self.file_io = file_io; - self - } - - #[cfg(test)] - pub(crate) fn file_io(&self) -> &FileIO { - &self.file_io - } - /// Retrieves an Arc [`Manifest`] from the cache /// or retrieves one from FileIO and parses it if not present pub(crate) async fn get_manifest(&self, manifest_file: &ManifestFile) -> Result> { diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index 014e324d22..a0e71a5501 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -37,7 +37,7 @@ mod hf; mod oss; mod s3; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; pub use azdls::*; pub use gcs::*; @@ -52,20 +52,13 @@ use serde::{Deserialize, Serialize}; /// This struct contains only configuration properties without specifying /// which storage backend to use. The storage type is determined by the /// explicit factory selection. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct StorageConfig { /// Configuration properties for the storage backend props: HashMap, -} - -impl std::fmt::Debug for StorageConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Values may hold vended credentials, and Debug is reachable through - // `FileIO` and `Table`: print keys only. - f.debug_struct("StorageConfig") - .field("keys", &self.props.keys().collect::>()) - .finish_non_exhaustive() - } + /// Credentials that apply to locations under their prefixes + #[serde(default, skip_serializing_if = "Vec::is_empty")] + storage_credentials: Vec, } impl StorageConfig { @@ -73,6 +66,7 @@ impl StorageConfig { pub fn new() -> Self { Self { props: HashMap::new(), + storage_credentials: Vec::new(), } } @@ -82,7 +76,10 @@ impl StorageConfig { /// /// * `props` - Configuration properties for the storage backend pub fn from_props(props: HashMap) -> Self { - Self { props } + Self { + props, + storage_credentials: Vec::new(), + } } /// Get all configuration properties. @@ -131,6 +128,64 @@ impl StorageConfig { .extend(props.into_iter().map(|(k, v)| (k.into(), v.into()))); self } + + /// Get the credentials that apply to locations under their prefixes. + pub fn storage_credentials(&self) -> &[StorageCredential] { + &self.storage_credentials + } + + /// Add credentials that apply to locations under their prefixes. + /// + /// A factory that supports them overlays the props with the config of a + /// location's longest matching credential. + pub fn with_storage_credentials( + mut self, + credentials: impl IntoIterator, + ) -> Self { + self.storage_credentials.extend(credentials); + self + } +} + +/// Storage properties, such as a vended credential, that apply to locations +/// under a prefix, like Java's `StorageCredential`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StorageCredential { + /// Location prefix, such as `s3://bucket/table` + prefix: String, + /// Storage properties holding the credential + config: HashMap, +} + +impl StorageCredential { + /// Create a credential for locations under `prefix`. An empty prefix covers none. + pub fn new(prefix: impl Into, config: HashMap) -> Self { + Self { + prefix: prefix.into(), + config, + } + } + + /// Return the location prefix this credential applies to. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// Return the storage properties holding the credential. + pub fn config(&self) -> &HashMap { + &self.config + } +} + +// The config values are secrets. +impl std::fmt::Debug for StorageCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let config_keys: BTreeSet<_> = self.config.keys().collect(); + f.debug_struct("StorageCredential") + .field("prefix", &self.prefix) + .field("config_keys", &config_keys) + .finish() + } } #[cfg(test)] @@ -162,32 +217,6 @@ mod tests { assert!(config.props().is_empty()); } - #[test] - fn test_debug_redacts_credential_values() { - let config = StorageConfig::from_props(HashMap::from([ - ("s3.access-key-id".to_string(), "vended-key".to_string()), - ( - "s3.secret-access-key".to_string(), - "super-secret".to_string(), - ), - ("s3.session-token".to_string(), "vended-token".to_string()), - ])); - - let rendered = format!("{config:?}"); - assert!( - !rendered.contains("super-secret"), - "leaked secret: {rendered}" - ); - assert!( - !rendered.contains("vended-token"), - "leaked token: {rendered}" - ); - assert!( - rendered.contains("s3.secret-access-key"), - "keys hidden: {rendered}" - ); - } - #[test] fn test_storage_config_get() { let config = StorageConfig::new().with_prop("region", "us-east-1"); @@ -236,6 +265,37 @@ mod tests { assert_eq!(config, deserialized); } + #[test] + fn test_storage_config_serialization_keeps_storage_credentials() { + let config = StorageConfig::new() + .with_prop("region", "us-east-1") + .with_storage_credentials([StorageCredential::new( + "s3://bucket/table", + HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), + )]); + + let serialized = serde_json::to_string(&config).unwrap(); + let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(config, deserialized); + } + + #[test] + fn test_storage_credential_debug_omits_config_values() { + let credential = StorageCredential::new( + "s3://bucket/table", + HashMap::from([ + ("s3.secret-access-key".to_string(), "secret".to_string()), + ("s3.access-key-id".to_string(), "key".to_string()), + ]), + ); + + assert_eq!( + format!("{credential:?}"), + r#"StorageCredential { prefix: "s3://bucket/table", config_keys: {"s3.access-key-id", "s3.secret-access-key"} }"# + ); + } + #[test] fn test_storage_config_clone_independence() { let original = StorageConfig::new().with_prop("region", "us-east-1"); diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index cd2443d87f..f33c246eb7 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -228,18 +228,6 @@ impl Table { self } - /// Sets the [`Table`] `FileIO` and returns an updated instance. - pub(crate) fn with_file_io(mut self, file_io: FileIO) -> Self { - self.object_cache = Arc::new( - self.object_cache - .as_ref() - .clone() - .with_file_io(file_io.clone()), - ); - self.file_io = file_io; - self - } - /// Returns a TableBuilder to build a table pub fn builder() -> TableBuilder { TableBuilder::new() @@ -433,9 +421,7 @@ mod tests { use crate::ErrorKind; use crate::encryption::SensitiveBytes; use crate::encryption::kms::MemoryKeyManagementClient; - use crate::io::{FileIOBuilder, MemoryStorageFactory}; use crate::spec::TableProperties; - use crate::test_utils::test_runtime; fn load_test_metadata(filename: &str) -> TableMetadata { let path = format!( @@ -545,40 +531,6 @@ mod tests { ); } - #[test] - fn test_with_file_io_updates_object_cache_file_io() { - let metadata = load_test_metadata("TableMetadataV2ValidMinimal.json"); - let original_file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prop("marker", "original") - .build(); - let replacement_file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prop("marker", "replacement") - .build(); - - let table = Table::builder() - .metadata(metadata) - .identifier(TableIdent::from_strs(["ns", "table"]).unwrap()) - .file_io(original_file_io) - .runtime(test_runtime()) - .build() - .unwrap() - .with_file_io(replacement_file_io); - - assert_eq!( - table.file_io().config().get("marker").map(String::as_str), - Some("replacement") - ); - assert_eq!( - table - .object_cache() - .file_io() - .config() - .get("marker") - .map(String::as_str), - Some("replacement") - ); - } - fn make_kms() -> Arc { let kms = MemoryKeyManagementClient::new(); kms.add_master_key("master-1").unwrap(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index c09ac0bbf4..2018af2fc1 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -228,12 +228,7 @@ impl Transaction { self.table = refreshed.clone(); } - // Actions need the credentials the refresh load vended. Identical - // settings keep the original FileIO, whose backend may be initialized. let mut current_table = self.table.clone(); - if !current_table.file_io().same_routing_as(refreshed.file_io()) { - current_table = current_table.with_file_io(refreshed.file_io().clone()); - } let mut existing_updates: Vec = vec![]; let mut existing_requirements: Vec = vec![]; @@ -254,24 +249,7 @@ impl Transaction { .requirements(existing_requirements) .build(); - let committed = catalog.update_table(table_commit).await?; - // A new location is outside the refreshed credentials' prefix, so reload - // after committing. Check the committed metadata: another writer may have - // moved the table, and a property-only commit has no requirement to catch it. - if committed.metadata().location() != refreshed.metadata().location() { - match catalog.load_table(committed.identifier()).await { - Ok(reloaded) => return Ok(reloaded), - // The commit is already durable: returning this error would let the - // retry loop replay it. Keep the old credentials; a reload fixes them. - Err(e) => tracing::warn!( - "committed {} but could not reload it for the new location: {e}", - committed.identifier() - ), - } - } - // The commit response carries no credentials, so keep the FileIO the - // actions ran with. - Ok(committed.with_file_io(current_table.file_io().clone())) + catalog.update_table(table_commit).await } } @@ -283,10 +261,8 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; - use async_trait::async_trait; - use crate::catalog::MockCatalog; - use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory}; + use crate::io::FileIO; use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Literal, PartitionSpec, Struct, @@ -294,9 +270,8 @@ mod tests { }; use crate::table::{StaticTable, Table}; use crate::test_utils::{make_encrypted_table, test_runtime}; - use crate::transaction::action::{ActionCommit, StatelessAction}; use crate::transaction::{ApplyTransactionAction, Transaction}; - use crate::{Catalog, Error, ErrorKind, Result, TableCreation, TableIdent}; + use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent}; pub fn make_v1_table() -> Table { let file = File::open(format!( @@ -487,219 +462,6 @@ mod tests { .unwrap() } - #[tokio::test] - async fn test_commit_keeps_the_refreshed_file_io() { - let refreshed = make_v2_table().with_file_io( - FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prefixed_props("memory://warehouse", [("s3.access-key-id", "vended")]) - .build(), - ); - let mut mock_catalog = MockCatalog::new(); - mock_catalog.expect_load_table().returning_st(move |_| { - let refreshed = refreshed.clone(); - Box::pin(async move { Ok(refreshed) }) - }); - mock_catalog - .expect_update_table() - .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) })); - - let table = create_test_transaction(&make_v2_table()) - .commit(&mock_catalog) - .await - .unwrap(); - - assert_eq!( - table - .file_io() - .config_for("memory://warehouse/t/f") - .get("s3.access-key-id"), - Some(&"vended".to_string()), - ); - } - - fn table_with_marker(marker: &str) -> Table { - make_v2_table().with_file_io( - FileIOBuilder::new(Arc::new(MemoryStorageFactory)) - .with_prop("marker", marker) - .build(), - ) - } - - fn marker_of(table: &Table) -> String { - table - .file_io() - .config() - .get("marker") - .cloned() - .unwrap_or_default() - } - - /// Records which FileIO the action was handed. - #[derive(Clone)] - struct RecordingAction(Arc>>); - - #[async_trait] - impl StatelessAction for RecordingAction { - async fn commit(&self, table: &Table) -> Result { - *self.0.lock().unwrap() = Some(marker_of(table)); - Ok(ActionCommit::new(vec![], vec![])) - } - } - - #[tokio::test] - async fn test_actions_run_with_the_refreshed_file_io() { - let seen = Arc::new(std::sync::Mutex::new(None)); - let mut mock_catalog = MockCatalog::new(); - mock_catalog - .expect_load_table() - .returning_st(|_| Box::pin(async { Ok(table_with_marker("refreshed")) })); - mock_catalog - .expect_update_table() - .returning_st(|_| Box::pin(async { Ok(make_v2_table()) })); - - let tx = RecordingAction(seen.clone()) - .apply(Transaction::new(&table_with_marker("stale"))) - .unwrap(); - tx.commit(&mock_catalog).await.unwrap(); - - assert_eq!(seen.lock().unwrap().as_deref(), Some("refreshed")); - } - - /// `MemoryStorageFactory` builds an empty store per FileIO, so a rebuilt - /// FileIO loses the data. - #[tokio::test] - async fn test_unchanged_settings_keep_the_initialized_storage() { - let table = table_with_marker("same"); - table - .file_io() - .new_output("memory://warehouse/manifest") - .unwrap() - .write("written-before-commit".into()) - .await - .unwrap(); - - let seen = Arc::new(std::sync::Mutex::new(None)); - let action_io = seen.clone(); - let mut mock_catalog = MockCatalog::new(); - // A reload rebuilds the FileIO, as a REST catalog does. - mock_catalog - .expect_load_table() - .returning_st(|_| Box::pin(async { Ok(table_with_marker("same")) })); - mock_catalog - .expect_update_table() - .returning_st(|_| Box::pin(async { Ok(make_v2_table()) })); - - let tx = ReadingAction(action_io) - .apply(Transaction::new(&table)) - .unwrap(); - let committed = tx.commit(&mock_catalog).await.unwrap(); - - assert_eq!( - seen.lock().unwrap().as_deref(), - Some("written-before-commit") - ); - let after = committed - .file_io() - .new_input("memory://warehouse/manifest") - .unwrap() - .read() - .await - .unwrap(); - assert_eq!(&after[..], b"written-before-commit"); - } - - /// Reads a file the table wrote before the transaction started. - #[derive(Clone)] - struct ReadingAction(Arc>>); - - #[async_trait] - impl StatelessAction for ReadingAction { - async fn commit(&self, table: &Table) -> Result { - let read = table - .file_io() - .new_input("memory://warehouse/manifest")? - .read() - .await - .ok() - .map(|b| String::from_utf8_lossy(&b).to_string()); - *self.0.lock().unwrap() = read; - Ok(ActionCommit::new(vec![], vec![])) - } - } - - /// What `update_table` returns once the table root has moved. - fn moved_table() -> Table { - let base = make_v2_table(); - let moved = base - .metadata() - .clone() - .into_builder(None) - .set_location("s3://moved-by-someone-else/table".to_string()) - .build() - .unwrap() - .metadata; - Table::builder() - .identifier(base.identifier().clone()) - .metadata(moved) - .file_io(base.file_io().clone()) - .runtime(test_runtime()) - .build() - .unwrap() - } - - #[tokio::test] - async fn test_a_failed_reload_does_not_replay_the_commit() { - let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut mock_catalog = MockCatalog::new(); - let counter = loads.clone(); - mock_catalog.expect_load_table().returning_st(move |_| { - let n = counter.fetch_add(1, Ordering::SeqCst); - Box::pin(async move { - if n == 1 { - Err(Error::new(ErrorKind::Unexpected, "503").with_retryable(true)) - } else { - Ok(make_v2_table()) - } - }) - }); - mock_catalog - .expect_update_table() - .times(1) - .returning_st(|_| Box::pin(async { Ok(moved_table()) })); - - let table = create_test_transaction(&make_v2_table()) - .commit(&mock_catalog) - .await - .unwrap(); - - assert_eq!( - table.metadata().location(), - "s3://moved-by-someone-else/table" - ); - } - - #[tokio::test] - async fn test_commit_reloads_when_someone_else_moved_the_table() { - let loads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut mock_catalog = MockCatalog::new(); - let counter = loads.clone(); - mock_catalog.expect_load_table().returning_st(move |_| { - let n = counter.fetch_add(1, Ordering::SeqCst); - Box::pin(async move { Ok(table_with_marker(if n == 0 { "first" } else { "second" })) }) - }); - mock_catalog - .expect_update_table() - .returning_st(|_| Box::pin(async { Ok(moved_table()) })); - - let table = create_test_transaction(&make_v2_table()) - .commit(&mock_catalog) - .await - .unwrap(); - - assert_eq!(loads.load(Ordering::SeqCst), 2); - assert_eq!(marker_of(&table), "second"); - } - /// Helper function to set up a mock catalog with retryable errors fn setup_mock_catalog_with_retryable_errors( success_after_attempts: Option, diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 55b651ae1f..dcc3b3f4d1 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -44,6 +44,7 @@ use iceberg_property_macro::Properties; use opendal::Operator; use opendal::layers::{RetryLayer, TimeoutLayer}; use opendal::options::WriteOptions; +use prefixed::PrefixedStorage; use serde::{Deserialize, Serialize}; use utils::from_opendal_error; @@ -109,6 +110,7 @@ cfg_if! { } } +mod prefixed; mod resolving; pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory}; @@ -218,6 +220,8 @@ fn parse_positive_ms(value: &str) -> Result { /// Maps scheme to the corresponding OpenDalStorage storage variant. /// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances. /// +/// Supports [storage credentials](iceberg::io::StorageConfig::with_storage_credentials). +/// /// # Serialization /// /// The receiving binary must enable the feature corresponding to the serialized backend variant. @@ -277,8 +281,14 @@ where #[typetag::serde(name = "OpenDalStorageFactory")] impl StorageFactory for OpenDalStorageFactory { - #[allow(unused_variables)] fn build(&self, config: &StorageConfig) -> Result> { + PrefixedStorage::build(config, |config| self.build_storage(config)) + } +} + +impl OpenDalStorageFactory { + #[allow(unused_variables)] + fn build_storage(&self, config: &StorageConfig) -> Result> { let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs new file mode 100644 index 0000000000..5aacb477ec --- /dev/null +++ b/crates/storage/opendal/src/prefixed.rs @@ -0,0 +1,373 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Storage that serves each location with its prefix's credential. + +use std::collections::HashMap; +use std::fmt::{self, Debug}; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::{self, BoxStream}; +use iceberg::Result; +use iceberg::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, +}; +use serde::{Deserialize, Serialize}; + +/// Per-prefix batch size of `delete_stream`, as in Java's `S3FileIO.deleteFiles`. +const DELETE_BATCH_SIZE: usize = 1000; + +/// Serves each path with the credential of its longest matching prefix, as +/// Java's `S3FileIO.clientForStoragePath` does, else without credentials. +#[derive(Serialize, Deserialize)] +pub(crate) struct PrefixedStorage { + default: Arc, + /// Longest prefix first. + prefixed: Vec<(String, Arc)>, +} + +impl PrefixedStorage { + /// Builds the storage for `config` with `build`, which is passed configs + /// without storage credentials. + pub(crate) fn build( + config: &StorageConfig, + build: impl Fn(&StorageConfig) -> Result>, + ) -> Result> { + if config.storage_credentials().is_empty() { + return build(config); + } + Ok(Arc::new(Self::new(config, build)?)) + } + + fn new( + config: &StorageConfig, + build: impl Fn(&StorageConfig) -> Result>, + ) -> Result { + let base = StorageConfig::from_props(config.props().clone()); + // The last credential for a prefix wins, as in Java's `clientByPrefix`. + let credentials: HashMap<&str, &HashMap> = config + .storage_credentials() + .iter() + .filter(|credential| !credential.prefix().is_empty()) + .map(|credential| (credential.prefix(), credential.config())) + .collect(); + let mut prefixed = credentials + .into_iter() + .map(|(prefix, credential)| { + let config = base.clone().with_props(credential.clone()); + Ok((prefix.to_string(), build(&config)?)) + }) + .collect::>>()?; + prefixed.sort_by_key(|(prefix, _)| std::cmp::Reverse(prefix.len())); + Ok(Self { + default: build(&base)?, + prefixed, + }) + } + + /// Matches raw strings, as Java does: `s3://b/data` covers `s3://b/database`. + fn prefix_index(&self, path: &str) -> Option { + self.prefixed + .iter() + .position(|(prefix, _)| path.starts_with(prefix.as_str())) + } + + fn storage_at(&self, index: Option) -> &dyn Storage { + match index { + Some(index) => self.prefixed[index].1.as_ref(), + None => self.default.as_ref(), + } + } + + fn storage_for(&self, path: &str) -> &dyn Storage { + self.storage_at(self.prefix_index(path)) + } +} + +// A backend's Debug may print the credentials it was built with. +impl Debug for PrefixedStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let prefixes: Vec<_> = self.prefixed.iter().map(|(prefix, _)| prefix).collect(); + f.debug_struct("PrefixedStorage") + .field("default", &self.default) + .field("prefixes", &prefixes) + .finish_non_exhaustive() + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for PrefixedStorage { + async fn exists(&self, path: &str) -> Result { + self.storage_for(path).exists(path).await + } + + async fn metadata(&self, path: &str) -> Result { + self.storage_for(path).metadata(path).await + } + + async fn read(&self, path: &str) -> Result { + self.storage_for(path).read(path).await + } + + async fn reader(&self, path: &str) -> Result> { + self.storage_for(path).reader(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.storage_for(path).write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.storage_for(path).writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.storage_for(path).delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + // Route as a directory, so an `s3://b/t/` prefix covers `s3://b/t`. + let dir = format!("{}/", path.strip_suffix('/').unwrap_or(path)); + self.storage_for(&dir).delete_prefix(path).await + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + let mut batches: HashMap, Vec> = HashMap::new(); + while let Some(path) = paths.next().await { + let index = self.prefix_index(&path); + let batch = batches.entry(index).or_default(); + batch.push(path); + if batch.len() >= DELETE_BATCH_SIZE { + let batch = std::mem::take(batch); + self.storage_at(index) + .delete_stream(stream::iter(batch).boxed()) + .await?; + } + } + for (index, batch) in batches { + if !batch.is_empty() { + self.storage_at(index) + .delete_stream(stream::iter(batch).boxed()) + .await?; + } + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + self.storage_for(path).new_input(path) + } + + fn new_output(&self, path: &str) -> Result { + self.storage_for(path).new_output(path) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use iceberg::io::StorageCredential; + + use super::*; + + type CallLog = Arc>>; + + /// Records each call's path, or `delete_stream`'s batch size, with its `scope` prop. + #[derive(Debug, Serialize, Deserialize)] + struct RecordingStorage { + props: HashMap, + #[serde(skip)] + calls: CallLog, + } + + impl RecordingStorage { + fn record(&self, arg: impl Into) { + self.calls + .lock() + .unwrap() + .push((self.props["scope"].clone(), arg.into())); + } + } + + #[async_trait] + #[typetag::serde] + impl Storage for RecordingStorage { + async fn exists(&self, path: &str) -> Result { + self.record(path); + Ok(true) + } + + async fn metadata(&self, _path: &str) -> Result { + unimplemented!() + } + + async fn read(&self, _path: &str) -> Result { + unimplemented!() + } + + async fn reader(&self, _path: &str) -> Result> { + unimplemented!() + } + + async fn write(&self, _path: &str, _bs: Bytes) -> Result<()> { + unimplemented!() + } + + async fn writer(&self, _path: &str) -> Result> { + unimplemented!() + } + + async fn delete(&self, _path: &str) -> Result<()> { + unimplemented!() + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.record(path); + Ok(()) + } + + async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> { + self.record(paths.count().await.to_string()); + Ok(()) + } + + fn new_input(&self, _path: &str) -> Result { + unimplemented!() + } + + fn new_output(&self, _path: &str) -> Result { + unimplemented!() + } + } + + /// A storage with `scope` "default" and a credential per (prefix, scope). + fn recording_storage(credentials: &[(&str, &str)]) -> (PrefixedStorage, CallLog) { + let config = StorageConfig::new() + .with_prop("scope", "default") + .with_storage_credentials(credentials.iter().map(|(prefix, scope)| { + StorageCredential::new( + *prefix, + HashMap::from([("scope".to_string(), scope.to_string())]), + ) + })); + let calls = CallLog::default(); + let storage = PrefixedStorage::new(&config, |config| { + Ok(Arc::new(RecordingStorage { + props: config.props().clone(), + calls: calls.clone(), + })) + }) + .unwrap(); + (storage, calls) + } + + fn recorded(calls: &[(&str, &str)]) -> Vec<(String, String)> { + calls + .iter() + .map(|(scope, arg)| (scope.to_string(), arg.to_string())) + .collect() + } + + #[tokio::test] + async fn test_routes_a_path_to_its_longest_matching_prefix() { + let (storage, calls) = recording_storage(&[ + ("s3://bucket/data", "short"), + ("s3://bucket/data/warehouse", "long"), + ("", "empty"), + ]); + + for path in [ + "s3://bucket/data/warehouse/f", + "s3://bucket/data/other/f", + "s3://bucket/database/f", + "s3://bucket/elsewhere/f", + ] { + storage.exists(path).await.unwrap(); + } + + assert_eq!( + *calls.lock().unwrap(), + recorded(&[ + ("long", "s3://bucket/data/warehouse/f"), + ("short", "s3://bucket/data/other/f"), + ("short", "s3://bucket/database/f"), + ("default", "s3://bucket/elsewhere/f"), + ]) + ); + } + + #[tokio::test] + async fn test_a_later_credential_replaces_an_earlier_one_for_its_prefix() { + let (storage, calls) = recording_storage(&[ + ("s3://bucket/table/", "earlier"), + ("s3://bucket/table/", "later"), + ]); + + storage.exists("s3://bucket/table/f").await.unwrap(); + + assert_eq!( + *calls.lock().unwrap(), + recorded(&[("later", "s3://bucket/table/f")]) + ); + } + + #[tokio::test] + async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash() { + for path in ["s3://bucket/table", "s3://bucket/table/"] { + let (storage, calls) = recording_storage(&[("s3://bucket/table/", "vended")]); + + storage.delete_prefix(path).await.unwrap(); + + assert_eq!(*calls.lock().unwrap(), recorded(&[("vended", path)])); + } + } + + #[tokio::test] + async fn test_delete_stream_batches_paths_per_prefix() { + let (storage, calls) = recording_storage(&[("s3://bucket/table/", "vended")]); + let paths = (0..DELETE_BATCH_SIZE + 1) + .map(|i| format!("s3://bucket/table/{i}")) + .chain(["s3://bucket/other/f".to_string()]); + + storage + .delete_stream(stream::iter(paths).boxed()) + .await + .unwrap(); + + let mut calls = calls.lock().unwrap().clone(); + calls.sort(); + assert_eq!( + calls, + recorded(&[("default", "1"), ("vended", "1"), ("vended", "1000")]) + ); + } + + #[test] + fn test_debug_omits_prefixed_storages() { + let (storage, _) = recording_storage(&[("s3://bucket/table/", "vended-secret")]); + + let debug = format!("{storage:?}"); + + assert!(!debug.contains("vended-secret"), "{debug}"); + assert!(debug.contains("s3://bucket/table/"), "{debug}"); + } +} diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 47459c42c8..970c30cb7f 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -33,6 +33,7 @@ use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; use url::Url; +use crate::prefixed::PrefixedStorage; #[cfg(feature = "opendal-s3")] use crate::s3::CustomAwsCredentialLoader; use crate::{OpenDalClientConfig, OpenDalStorage}; @@ -160,6 +161,8 @@ fn build_storage_for_scheme( /// delegates operations to the appropriate [`OpenDalStorage`] variant based on /// the path scheme. /// +/// Supports [storage credentials](iceberg::io::StorageConfig::with_storage_credentials). +/// /// # Serialization /// /// Serialization fails when a custom S3 credential loader is configured because the loader holds @@ -216,12 +219,14 @@ impl OpenDalResolvingStorageFactory { #[typetag::serde] impl StorageFactory for OpenDalResolvingStorageFactory { fn build(&self, config: &StorageConfig) -> Result> { - Ok(Arc::new(OpenDalResolvingStorage { - props: config.props().clone(), - storages: RwLock::new(HashMap::new()), - #[cfg(feature = "opendal-s3")] - customized_credential_load: self.customized_credential_load.clone(), - })) + PrefixedStorage::build(config, |config| { + Ok(Arc::new(OpenDalResolvingStorage { + props: config.props().clone(), + storages: RwLock::new(HashMap::new()), + #[cfg(feature = "opendal-s3")] + customized_credential_load: self.customized_credential_load.clone(), + })) + }) } } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 14e54dcf59..f38f05cbb7 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -21,13 +21,14 @@ //! Each test uses unique file paths based on module path to avoid conflicts. #[cfg(feature = "opendal-s3")] mod tests { + use std::collections::HashMap; use std::sync::Arc; use bytes::Bytes; use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, - S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, + S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, StorageCredential, }; use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, @@ -415,4 +416,30 @@ mod tests { // Should succeed with no-op file_io.delete_stream(stream).await.unwrap(); } + + #[tokio::test] + async fn test_file_io_s3_storage_credential_authorizes_its_prefix() { + set_up(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(vec![ + (S3_ENDPOINT, get_object_store_endpoint()), + (S3_ACCESS_KEY_ID, "invalid".to_string()), + (S3_SECRET_ACCESS_KEY, "invalid".to_string()), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .with_storage_credentials([StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + )]) + .build(); + + assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); + assert!(file_io.exists("s3://bucket1/other/f").await.is_err()); + } } diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index bcc3d51842..b5a26b0bf2 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -26,11 +26,12 @@ feature = "opendal-memory" ))] mod tests { + use std::collections::HashMap; use std::sync::Arc; use iceberg::io::{ FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, - S3_SECRET_ACCESS_KEY, + S3_SECRET_ACCESS_KEY, StorageCredential, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; @@ -239,6 +240,31 @@ mod tests { ); } + #[tokio::test] + async fn test_storage_credential_authorizes_its_prefix_after_serialization() { + set_up(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_props(vec![ + (S3_ENDPOINT, get_object_store_endpoint()), + (S3_ACCESS_KEY_ID, "invalid".to_string()), + (S3_SECRET_ACCESS_KEY, "invalid".to_string()), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .with_storage_credentials([StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + )]) + .build(); + let file_io = roundtrip_file_io(&file_io); + + assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); + assert!(file_io.exists("s3://bucket1/other/f").await.is_err()); + } + #[tokio::test] async fn test_invalid_scheme() { let file_io = get_resolving_file_io(); From 78c8cb5f62d155d62c3c2960c141e85bd3e95708 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 28 Oct 2026 05:10:35 -0700 Subject: [PATCH 23/27] refactor(io): build StorageConfig with TypedBuilder and reuse StorageCredential in REST --- crates/catalog/rest/public-api.txt | 17 +------- crates/catalog/rest/src/catalog.rs | 13 +++--- crates/catalog/rest/src/types.rs | 14 +------ crates/iceberg/public-api.txt | 3 +- crates/iceberg/src/io/file_io.rs | 4 +- crates/iceberg/src/io/storage/config/mod.rs | 44 +++++++++------------ crates/storage/opendal/src/lib.rs | 2 +- crates/storage/opendal/src/prefixed.rs | 24 ++++++----- crates/storage/opendal/src/resolving.rs | 2 +- 9 files changed, 48 insertions(+), 75 deletions(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index a0c6fd03ce..5f89dadfce 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -196,7 +196,7 @@ pub struct iceberg_catalog_rest::LoadTableResult pub iceberg_catalog_rest::LoadTableResult::config: std::collections::hash::map::HashMap pub iceberg_catalog_rest::LoadTableResult::metadata: iceberg::spec::table_metadata::TableMetadata pub iceberg_catalog_rest::LoadTableResult::metadata_location: core::option::Option -pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option> +pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option> impl core::clone::Clone for iceberg_catalog_rest::LoadTableResult pub fn iceberg_catalog_rest::LoadTableResult::clone(&self) -> iceberg_catalog_rest::LoadTableResult impl core::cmp::Eq for iceberg_catalog_rest::LoadTableResult @@ -355,21 +355,6 @@ impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())> -pub struct iceberg_catalog_rest::StorageCredential -pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap -pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String -impl core::clone::Clone for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::clone(&self) -> iceberg_catalog_rest::StorageCredential -impl core::cmp::Eq for iceberg_catalog_rest::StorageCredential -impl core::cmp::PartialEq for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::eq(&self, other: &iceberg_catalog_rest::StorageCredential) -> bool -impl core::fmt::Debug for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::marker::StructuralPartialEq for iceberg_catalog_rest::StorageCredential -impl serde_core::ser::Serialize for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg_catalog_rest::UpdateNamespacePropertiesRequest pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::removals: core::option::Option> pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::updates: std::collections::hash::map::HashMap diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 73c5f42812..47859fe6d3 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -28,7 +28,8 @@ use std::time::Duration; use async_trait::async_trait; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory, + ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageCredential, + StorageFactory, }; use iceberg::table::Table; use iceberg::{ @@ -55,7 +56,7 @@ use crate::response::HttpResponse; use crate::types::{ CatalogConfig, CommitTableRequest, CommitTableResponse, CreateNamespaceRequest, CreateTableRequest, ListNamespaceResponse, ListTablesResponse, LoadTableResult, - NamespaceResponse, RegisterTableRequest, RenameTableRequest, StorageCredential, + NamespaceResponse, RegisterTableRequest, RenameTableRequest, }; /// REST catalog URI @@ -1022,9 +1023,7 @@ impl RestSessionCatalog { let file_io = FileIOBuilder::new(factory) .with_props(props) - .with_storage_credentials(storage_credentials.into_iter().flatten().map(|credential| { - iceberg::io::StorageCredential::new(credential.prefix, credential.config) - })) + .with_storage_credentials(storage_credentials.into_iter().flatten()) .build(); Ok(file_io) @@ -4788,8 +4787,8 @@ mod tests { } /// A `loadTable` body vending a storage credential, and that credential. - fn load_table_body_with_credential() -> (serde_json::Value, iceberg::io::StorageCredential) { - let credential = iceberg::io::StorageCredential::new( + fn load_table_body_with_credential() -> (serde_json::Value, StorageCredential) { + let credential = StorageCredential::new( "s3://warehouse/database/table", HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), ); diff --git a/crates/catalog/rest/src/types.rs b/crates/catalog/rest/src/types.rs index 390521229e..d3e2e4fda3 100644 --- a/crates/catalog/rest/src/types.rs +++ b/crates/catalog/rest/src/types.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; +use iceberg::io::StorageCredential; use iceberg::spec::{Schema, SortOrder, TableMetadata, UnboundPartitionSpec}; use iceberg::{ Error, ErrorKind, Namespace, NamespaceIdent, TableIdent, TableRequirement, TableUpdate, @@ -231,19 +232,6 @@ pub struct LoadTableResult { pub storage_credentials: Option>, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -/// Storage credential for a specific location prefix. -/// -/// Indicates a storage location prefix where the credential is relevant. Clients should -/// choose the most specific prefix (by selecting the longest prefix) if several credentials -/// of the same type are available. -pub struct StorageCredential { - /// Storage location prefix where this credential is relevant - pub prefix: String, - /// Configuration map containing credential information - pub config: HashMap, -} - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Request to create a new table in a namespace. diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 71b7f20010..ab6d74a4e5 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1104,7 +1104,6 @@ pub fn iceberg::io::StorageConfig::props(&self) -> &std::collections::hash::map: pub fn iceberg::io::StorageConfig::storage_credentials(&self) -> &[iceberg::io::StorageCredential] pub fn iceberg::io::StorageConfig::with_prop(self, key: impl core::convert::Into, value: impl core::convert::Into) -> Self pub fn iceberg::io::StorageConfig::with_props(self, props: impl core::iter::traits::collect::IntoIterator, impl core::convert::Into)>) -> Self -pub fn iceberg::io::StorageConfig::with_storage_credentials(self, credentials: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::clone(&self) -> iceberg::io::StorageConfig impl core::cmp::Eq for iceberg::io::StorageConfig @@ -1130,6 +1129,8 @@ pub fn iceberg::io::StorageConfig::default() -> iceberg::io::StorageConfig impl core::fmt::Debug for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::io::StorageConfig +impl iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::builder() -> StorageConfigBuilder<((), ())> impl serde_core::ser::Serialize for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index ce0775bb56..a79619bf7d 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -278,12 +278,12 @@ impl FileIOBuilder { /// Add credentials that apply to locations under their prefixes. /// - /// See [`StorageConfig::with_storage_credentials`]. + /// See [`StorageConfig::storage_credentials`]. pub fn with_storage_credentials( mut self, credentials: impl IntoIterator, ) -> Self { - self.config = self.config.with_storage_credentials(credentials); + self.config.storage_credentials.extend(credentials); self } diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index a0e71a5501..aebf5a20de 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -37,7 +37,7 @@ mod hf; mod oss; mod s3; -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; pub use azdls::*; pub use gcs::*; @@ -46,19 +46,22 @@ pub use hf::*; pub use oss::*; pub use s3::*; use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; /// Configuration properties for storage backends. /// /// This struct contains only configuration properties without specifying /// which storage backend to use. The storage type is determined by the /// explicit factory selection. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default, TypedBuilder)] pub struct StorageConfig { /// Configuration properties for the storage backend + #[builder(default)] props: HashMap, /// Credentials that apply to locations under their prefixes + #[builder(default)] #[serde(default, skip_serializing_if = "Vec::is_empty")] - storage_credentials: Vec, + pub(crate) storage_credentials: Vec, } impl StorageConfig { @@ -130,20 +133,11 @@ impl StorageConfig { } /// Get the credentials that apply to locations under their prefixes. - pub fn storage_credentials(&self) -> &[StorageCredential] { - &self.storage_credentials - } - - /// Add credentials that apply to locations under their prefixes. /// /// A factory that supports them overlays the props with the config of a /// location's longest matching credential. - pub fn with_storage_credentials( - mut self, - credentials: impl IntoIterator, - ) -> Self { - self.storage_credentials.extend(credentials); - self + pub fn storage_credentials(&self) -> &[StorageCredential] { + &self.storage_credentials } } @@ -180,10 +174,9 @@ impl StorageCredential { // The config values are secrets. impl std::fmt::Debug for StorageCredential { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let config_keys: BTreeSet<_> = self.config.keys().collect(); f.debug_struct("StorageCredential") .field("prefix", &self.prefix) - .field("config_keys", &config_keys) + .field("config_keys", &self.config.keys()) .finish() } } @@ -267,12 +260,16 @@ mod tests { #[test] fn test_storage_config_serialization_keeps_storage_credentials() { - let config = StorageConfig::new() - .with_prop("region", "us-east-1") - .with_storage_credentials([StorageCredential::new( + let config = StorageConfig::builder() + .props(HashMap::from([( + "region".to_string(), + "us-east-1".to_string(), + )])) + .storage_credentials(vec![StorageCredential::new( "s3://bucket/table", HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), - )]); + )]) + .build(); let serialized = serde_json::to_string(&config).unwrap(); let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap(); @@ -284,15 +281,12 @@ mod tests { fn test_storage_credential_debug_omits_config_values() { let credential = StorageCredential::new( "s3://bucket/table", - HashMap::from([ - ("s3.secret-access-key".to_string(), "secret".to_string()), - ("s3.access-key-id".to_string(), "key".to_string()), - ]), + HashMap::from([("s3.secret-access-key".to_string(), "secret".to_string())]), ); assert_eq!( format!("{credential:?}"), - r#"StorageCredential { prefix: "s3://bucket/table", config_keys: {"s3.access-key-id", "s3.secret-access-key"} }"# + r#"StorageCredential { prefix: "s3://bucket/table", config_keys: ["s3.secret-access-key"] }"# ); } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index dcc3b3f4d1..144d8372c3 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -220,7 +220,7 @@ fn parse_positive_ms(value: &str) -> Result { /// Maps scheme to the corresponding OpenDalStorage storage variant. /// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances. /// -/// Supports [storage credentials](iceberg::io::StorageConfig::with_storage_credentials). +/// Supports [storage credentials](iceberg::io::StorageConfig::storage_credentials). /// /// # Serialization /// diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs index 5aacb477ec..7ffc7a42a2 100644 --- a/crates/storage/opendal/src/prefixed.rs +++ b/crates/storage/opendal/src/prefixed.rs @@ -104,7 +104,11 @@ impl PrefixedStorage { // A backend's Debug may print the credentials it was built with. impl Debug for PrefixedStorage { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let prefixes: Vec<_> = self.prefixed.iter().map(|(prefix, _)| prefix).collect(); + let prefixes = fmt::from_fn(|f| { + f.debug_list() + .entries(self.prefixed.iter().map(|(prefix, _)| prefix)) + .finish() + }); f.debug_struct("PrefixedStorage") .field("default", &self.default) .field("prefixes", &prefixes) @@ -261,14 +265,16 @@ mod tests { /// A storage with `scope` "default" and a credential per (prefix, scope). fn recording_storage(credentials: &[(&str, &str)]) -> (PrefixedStorage, CallLog) { - let config = StorageConfig::new() - .with_prop("scope", "default") - .with_storage_credentials(credentials.iter().map(|(prefix, scope)| { - StorageCredential::new( - *prefix, - HashMap::from([("scope".to_string(), scope.to_string())]), - ) - })); + let scope = |scope: &str| HashMap::from([("scope".to_string(), scope.to_string())]); + let config = StorageConfig::builder() + .props(scope("default")) + .storage_credentials( + credentials + .iter() + .map(|(prefix, credential)| StorageCredential::new(*prefix, scope(credential))) + .collect(), + ) + .build(); let calls = CallLog::default(); let storage = PrefixedStorage::new(&config, |config| { Ok(Arc::new(RecordingStorage { diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 970c30cb7f..4c23ac4208 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -161,7 +161,7 @@ fn build_storage_for_scheme( /// delegates operations to the appropriate [`OpenDalStorage`] variant based on /// the path scheme. /// -/// Supports [storage credentials](iceberg::io::StorageConfig::with_storage_credentials). +/// Supports [storage credentials](iceberg::io::StorageConfig::storage_credentials). /// /// # Serialization /// From 2b1aab8fb499fd4c067ed47eedaa33bf762161dd Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 28 Oct 2026 23:36:25 -0700 Subject: [PATCH 24/27] refactor(io): pass a built StorageConfig through FileIOBuilder::with_config --- crates/catalog/rest/src/catalog.rs | 11 ++++--- crates/iceberg/public-api.txt | 2 +- crates/iceberg/src/io/file_io.rs | 27 ++++++++------- crates/iceberg/src/io/storage/config/mod.rs | 2 +- .../storage/opendal/tests/file_io_s3_test.rs | 33 ++++++++++--------- .../opendal/tests/resolving_storage_test.rs | 23 +++++++------ 6 files changed, 55 insertions(+), 43 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 47859fe6d3..b556fa51b9 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -28,8 +28,8 @@ use std::time::Duration; use async_trait::async_trait; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageCredential, - StorageFactory, + ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageConfig, + StorageCredential, StorageFactory, }; use iceberg::table::Table; use iceberg::{ @@ -1021,10 +1021,11 @@ impl RestSessionCatalog { ) })?; - let file_io = FileIOBuilder::new(factory) - .with_props(props) - .with_storage_credentials(storage_credentials.into_iter().flatten()) + let config = StorageConfig::builder() + .props(props) + .storage_credentials(storage_credentials.unwrap_or_default()) .build(); + let file_io = FileIOBuilder::new(factory).with_config(config).build(); Ok(file_io) } diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index ab6d74a4e5..0c2047d893 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -878,9 +878,9 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_config(self, config: iceberg::io::StorageConfig) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self -pub fn iceberg::io::FileIOBuilder::with_storage_credentials(self, credentials: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::clone(&self) -> iceberg::io::FileIOBuilder impl core::fmt::Debug for iceberg::io::FileIOBuilder diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index a79619bf7d..7029ed4074 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -22,8 +22,7 @@ use bytes::Bytes; use futures::{Stream, StreamExt}; use super::storage::{ - LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageCredential, - StorageFactory, + LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, }; use crate::Result; @@ -276,14 +275,9 @@ impl FileIOBuilder { self } - /// Add credentials that apply to locations under their prefixes. - /// - /// See [`StorageConfig::storage_credentials`]. - pub fn with_storage_credentials( - mut self, - credentials: impl IntoIterator, - ) -> Self { - self.config.storage_credentials.extend(credentials); + /// Replace the storage configuration set so far. + pub fn with_config(mut self, config: StorageConfig) -> Self { + self.config = config; self } @@ -465,7 +459,7 @@ mod tests { use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; - use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; + use crate::io::{LocalFsStorageFactory, MemoryStorageFactory, StorageConfig}; fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -613,6 +607,17 @@ mod tests { assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + #[test] + fn test_file_io_builder_with_config_replaces_config() { + let config = StorageConfig::new().with_prop("key2", "value2"); + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("key1", "value1") + .with_config(config.clone()) + .build(); + + assert_eq!(file_io.config(), &config); + } + #[tokio::test] async fn test_memory_file_io_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index aebf5a20de..a5cbe28fa0 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -61,7 +61,7 @@ pub struct StorageConfig { /// Credentials that apply to locations under their prefixes #[builder(default)] #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) storage_credentials: Vec, + storage_credentials: Vec, } impl StorageConfig { diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index f38f05cbb7..057fe513fb 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -28,7 +28,7 @@ mod tests { use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, - S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, StorageCredential, + S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, StorageConfig, StorageCredential, }; use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, @@ -420,23 +420,26 @@ mod tests { #[tokio::test] async fn test_file_io_s3_storage_credential_authorizes_its_prefix() { set_up(); + let config = StorageConfig::builder() + .props(HashMap::from([ + (S3_ENDPOINT.to_string(), get_object_store_endpoint()), + (S3_ACCESS_KEY_ID.to_string(), "invalid".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "invalid".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ])) + .storage_credentials(vec![StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + )]) + .build(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None, })) - .with_props(vec![ - (S3_ENDPOINT, get_object_store_endpoint()), - (S3_ACCESS_KEY_ID, "invalid".to_string()), - (S3_SECRET_ACCESS_KEY, "invalid".to_string()), - (S3_REGION, "us-east-1".to_string()), - (S3_PATH_STYLE_ACCESS, "true".to_string()), - ]) - .with_storage_credentials([StorageCredential::new( - "s3://bucket1/vended/", - HashMap::from([ - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - ]), - )]) + .with_config(config) .build(); assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index b5a26b0bf2..056cef450b 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -31,7 +31,7 @@ mod tests { use iceberg::io::{ FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, - S3_SECRET_ACCESS_KEY, StorageCredential, + S3_SECRET_ACCESS_KEY, StorageConfig, StorageCredential, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; @@ -243,15 +243,15 @@ mod tests { #[tokio::test] async fn test_storage_credential_authorizes_its_prefix_after_serialization() { set_up(); - let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) - .with_props(vec![ - (S3_ENDPOINT, get_object_store_endpoint()), - (S3_ACCESS_KEY_ID, "invalid".to_string()), - (S3_SECRET_ACCESS_KEY, "invalid".to_string()), - (S3_REGION, "us-east-1".to_string()), - (S3_PATH_STYLE_ACCESS, "true".to_string()), - ]) - .with_storage_credentials([StorageCredential::new( + let config = StorageConfig::builder() + .props(HashMap::from([ + (S3_ENDPOINT.to_string(), get_object_store_endpoint()), + (S3_ACCESS_KEY_ID.to_string(), "invalid".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "invalid".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ])) + .storage_credentials(vec![StorageCredential::new( "s3://bucket1/vended/", HashMap::from([ (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), @@ -259,6 +259,9 @@ mod tests { ]), )]) .build(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_config(config) + .build(); let file_io = roundtrip_file_io(&file_io); assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); From 19dc179ffb1380e95e123e9052d6e0fea17b6d38 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 3 Nov 2026 07:02:45 -0800 Subject: [PATCH 25/27] refactor(storage/opendal): hold OpenDalStorage in PrefixedStorage and validate credential prefixes --- crates/catalog/rest/src/catalog.rs | 3 +- crates/iceberg/public-api.txt | 4 +- crates/iceberg/src/io/storage/config/mod.rs | 63 ++++- crates/storage/opendal/src/lib.rs | 36 +-- crates/storage/opendal/src/prefixed.rs | 246 +++++++----------- crates/storage/opendal/src/resolving.rs | 177 ++++++++++--- .../storage/opendal/tests/file_io_s3_test.rs | 17 +- .../opendal/tests/resolving_storage_test.rs | 17 +- 8 files changed, 330 insertions(+), 233 deletions(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index b556fa51b9..7f25e59799 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -4792,7 +4792,8 @@ mod tests { let credential = StorageCredential::new( "s3://warehouse/database/table", HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), - ); + ) + .unwrap(); let mut body = load_table_body("v1"); body["storage-credentials"] = json!([{ "prefix": credential.prefix(), "config": credential.config() }]); diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 0c2047d893..18157216cd 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1138,7 +1138,7 @@ pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core pub struct iceberg::io::StorageCredential impl iceberg::io::StorageCredential pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap -pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> iceberg::Result pub fn iceberg::io::StorageCredential::prefix(&self) -> &str impl core::clone::Clone for iceberg::io::StorageCredential pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential @@ -1151,7 +1151,7 @@ impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential impl serde_core::ser::Serialize for iceberg::io::StorageCredential pub fn iceberg::io::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageCredential -pub fn iceberg::io::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub fn iceberg::io::StorageCredential::deserialize>(deserializer: D) -> core::result::Result::Error> pub const iceberg::io::ADLS_ACCOUNT_KEY: &str pub const iceberg::io::ADLS_ACCOUNT_NAME: &str pub const iceberg::io::ADLS_AUTHORITY_HOST: &str diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index a5cbe28fa0..5d54ab2c3b 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -45,9 +45,11 @@ pub use hdfs_native::*; pub use hf::*; pub use oss::*; pub use s3::*; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use typed_builder::TypedBuilder; +use crate::{Error, ErrorKind, Result}; + /// Configuration properties for storage backends. /// /// This struct contains only configuration properties without specifying @@ -143,7 +145,7 @@ impl StorageConfig { /// Storage properties, such as a vended credential, that apply to locations /// under a prefix, like Java's `StorageCredential`. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Serialize)] pub struct StorageCredential { /// Location prefix, such as `s3://bucket/table` prefix: String, @@ -152,12 +154,20 @@ pub struct StorageCredential { } impl StorageCredential { - /// Create a credential for locations under `prefix`. An empty prefix covers none. - pub fn new(prefix: impl Into, config: HashMap) -> Self { - Self { - prefix: prefix.into(), - config, + /// Create a credential for locations under `prefix`. + /// + /// # Errors + /// + /// Returns an error if `prefix` is empty. + pub fn new(prefix: impl Into, config: HashMap) -> Result { + let prefix = prefix.into(); + if prefix.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid storage credential: prefix must be non-empty", + )); } + Ok(Self { prefix, config }) } /// Return the location prefix this credential applies to. @@ -171,6 +181,19 @@ impl StorageCredential { } } +impl<'de> Deserialize<'de> for StorageCredential { + fn deserialize>(deserializer: D) -> std::result::Result { + #[derive(Deserialize)] + struct Fields { + prefix: String, + config: HashMap, + } + + let Fields { prefix, config } = Fields::deserialize(deserializer)?; + Self::new(prefix, config).map_err(serde::de::Error::custom) + } +} + // The config values are secrets. impl std::fmt::Debug for StorageCredential { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -265,10 +288,13 @@ mod tests { "region".to_string(), "us-east-1".to_string(), )])) - .storage_credentials(vec![StorageCredential::new( - "s3://bucket/table", - HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), - )]) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket/table", + HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), + ) + .unwrap(), + ]) .build(); let serialized = serde_json::to_string(&config).unwrap(); @@ -282,7 +308,8 @@ mod tests { let credential = StorageCredential::new( "s3://bucket/table", HashMap::from([("s3.secret-access-key".to_string(), "secret".to_string())]), - ); + ) + .unwrap(); assert_eq!( format!("{credential:?}"), @@ -290,6 +317,18 @@ mod tests { ); } + #[test] + fn test_storage_credential_rejects_an_empty_prefix() { + let config = HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]); + + let err = StorageCredential::new("", config).unwrap_err(); + let deserialized = + serde_json::from_str::(r#"{"prefix":"","config":{}}"#); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(deserialized.is_err()); + } + #[test] fn test_storage_config_clone_independence() { let original = StorageConfig::new().with_prop("region", "us-east-1"); diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 144d8372c3..f24ae681e5 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -288,52 +288,52 @@ impl StorageFactory for OpenDalStorageFactory { impl OpenDalStorageFactory { #[allow(unused_variables)] - fn build_storage(&self, config: &StorageConfig) -> Result> { + fn build_storage(&self, config: &StorageConfig) -> Result { let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] - OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory { + OpenDalStorageFactory::Memory => Ok(OpenDalStorage::Memory { operator: memory_config_build()?, client_config, - })), + }), #[cfg(feature = "opendal-fs")] - OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })), + OpenDalStorageFactory::Fs => Ok(OpenDalStorage::LocalFs { client_config }), #[cfg(feature = "opendal-s3")] OpenDalStorageFactory::S3 { customized_credential_load, - } => Ok(Arc::new(OpenDalStorage::S3 { + } => Ok(OpenDalStorage::S3 { config: s3_config_parse(config.props().clone())?.into(), multipart_part_size: s3_multipart_part_size_parse(config.props())?, customized_credential_load: customized_credential_load.clone(), client_config: s3_client_config(config.props(), client_config)?, - })), + }), #[cfg(feature = "opendal-gcs")] - OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { + OpenDalStorageFactory::Gcs => Ok(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative( - HdfsNativeStorage::new( + OpenDalStorageFactory::HdfsNative => { + Ok(OpenDalStorage::HdfsNative(HdfsNativeStorage::new( hdfs_native_config_parse(config.props().clone())?, client_config, - ), - ))), + ))) + } #[cfg(feature = "opendal-oss")] - OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { + OpenDalStorageFactory::Oss => Ok(OpenDalStorage::Oss { config: oss_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-azdls")] - OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls { + OpenDalStorageFactory::Azdls => Ok(OpenDalStorage::Azdls { config: azdls_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-hf")] - OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf { + OpenDalStorageFactory::Hf => Ok(OpenDalStorage::Hf { config: hf_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(all( not(feature = "opendal-memory"), not(feature = "opendal-fs"), diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs index 7ffc7a42a2..d5a9603bf6 100644 --- a/crates/storage/opendal/src/prefixed.rs +++ b/crates/storage/opendal/src/prefixed.rs @@ -31,53 +31,56 @@ use iceberg::io::{ }; use serde::{Deserialize, Serialize}; -/// Per-prefix batch size of `delete_stream`, as in Java's `S3FileIO.deleteFiles`. -const DELETE_BATCH_SIZE: usize = 1000; +use crate::OpenDalStorage; + +/// Batch size of `delete_stream`, as in Java's `S3FileIO.deleteFiles`. +pub(crate) const DELETE_BATCH_SIZE: usize = 1000; /// Serves each path with the credential of its longest matching prefix, as /// Java's `S3FileIO.clientForStoragePath` does, else without credentials. #[derive(Serialize, Deserialize)] pub(crate) struct PrefixedStorage { - default: Arc, + default: OpenDalStorage, /// Longest prefix first. - prefixed: Vec<(String, Arc)>, + prefixed: Vec<(String, OpenDalStorage)>, } impl PrefixedStorage { - /// Builds the storage for `config` with `build`, which is passed configs + /// Builds the storage for `config` with `build_fn`, which is passed configs /// without storage credentials. pub(crate) fn build( config: &StorageConfig, - build: impl Fn(&StorageConfig) -> Result>, + build_fn: impl Fn(&StorageConfig) -> Result, ) -> Result> { if config.storage_credentials().is_empty() { - return build(config); + return Ok(Arc::new(build_fn(config)?)); } - Ok(Arc::new(Self::new(config, build)?)) + Ok(Arc::new(Self::new(config, build_fn)?)) } - fn new( + /// Builds one storage per credential of `config` and one for its props + /// alone, with `build_fn` as in [`Self::build`]. + pub(crate) fn new( config: &StorageConfig, - build: impl Fn(&StorageConfig) -> Result>, + build_fn: impl Fn(&StorageConfig) -> Result, ) -> Result { let base = StorageConfig::from_props(config.props().clone()); // The last credential for a prefix wins, as in Java's `clientByPrefix`. let credentials: HashMap<&str, &HashMap> = config .storage_credentials() .iter() - .filter(|credential| !credential.prefix().is_empty()) .map(|credential| (credential.prefix(), credential.config())) .collect(); let mut prefixed = credentials .into_iter() .map(|(prefix, credential)| { let config = base.clone().with_props(credential.clone()); - Ok((prefix.to_string(), build(&config)?)) + Ok((prefix.to_string(), build_fn(&config)?)) }) .collect::>>()?; prefixed.sort_by_key(|(prefix, _)| std::cmp::Reverse(prefix.len())); Ok(Self { - default: build(&base)?, + default: build_fn(&base)?, prefixed, }) } @@ -89,14 +92,14 @@ impl PrefixedStorage { .position(|(prefix, _)| path.starts_with(prefix.as_str())) } - fn storage_at(&self, index: Option) -> &dyn Storage { + fn storage_at(&self, index: Option) -> &OpenDalStorage { match index { - Some(index) => self.prefixed[index].1.as_ref(), - None => self.default.as_ref(), + Some(index) => &self.prefixed[index].1, + None => &self.default, } } - fn storage_for(&self, path: &str) -> &dyn Storage { + pub(crate) fn storage_for(&self, path: &str) -> &OpenDalStorage { self.storage_at(self.prefix_index(path)) } } @@ -185,191 +188,126 @@ impl Storage for PrefixedStorage { } } -#[cfg(test)] +#[cfg(all(test, feature = "opendal-memory"))] mod tests { - use std::sync::Mutex; + use std::time::Duration; use iceberg::io::StorageCredential; use super::*; + use crate::{OPENDAL_IO_TIMEOUT_MS, OpenDalStorageFactory}; - type CallLog = Arc>>; - - /// Records each call's path, or `delete_stream`'s batch size, with its `scope` prop. - #[derive(Debug, Serialize, Deserialize)] - struct RecordingStorage { - props: HashMap, - #[serde(skip)] - calls: CallLog, - } - - impl RecordingStorage { - fn record(&self, arg: impl Into) { - self.calls - .lock() - .unwrap() - .push((self.props["scope"].clone(), arg.into())); - } - } - - #[async_trait] - #[typetag::serde] - impl Storage for RecordingStorage { - async fn exists(&self, path: &str) -> Result { - self.record(path); - Ok(true) - } - - async fn metadata(&self, _path: &str) -> Result { - unimplemented!() - } - - async fn read(&self, _path: &str) -> Result { - unimplemented!() - } - - async fn reader(&self, _path: &str) -> Result> { - unimplemented!() - } - - async fn write(&self, _path: &str, _bs: Bytes) -> Result<()> { - unimplemented!() - } - - async fn writer(&self, _path: &str) -> Result> { - unimplemented!() - } - - async fn delete(&self, _path: &str) -> Result<()> { - unimplemented!() - } - - async fn delete_prefix(&self, path: &str) -> Result<()> { - self.record(path); - Ok(()) - } - - async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> { - self.record(paths.count().await.to_string()); - Ok(()) - } - - fn new_input(&self, _path: &str) -> Result { - unimplemented!() - } - - fn new_output(&self, _path: &str) -> Result { - unimplemented!() - } + fn timeout_ms(ms: u64) -> HashMap { + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), ms.to_string())]) } - /// A storage with `scope` "default" and a credential per (prefix, scope). - fn recording_storage(credentials: &[(&str, &str)]) -> (PrefixedStorage, CallLog) { - let scope = |scope: &str| HashMap::from([("scope".to_string(), scope.to_string())]); + /// Memory storages with an IO timeout of 1s by default and of `ms` under each `prefix`. + fn memory_storage(credentials: &[(&str, u64)]) -> PrefixedStorage { let config = StorageConfig::builder() - .props(scope("default")) + .props(timeout_ms(1000)) .storage_credentials( credentials .iter() - .map(|(prefix, credential)| StorageCredential::new(*prefix, scope(credential))) + .map(|(prefix, ms)| StorageCredential::new(*prefix, timeout_ms(*ms)).unwrap()) .collect(), ) .build(); - let calls = CallLog::default(); - let storage = PrefixedStorage::new(&config, |config| { - Ok(Arc::new(RecordingStorage { - props: config.props().clone(), - calls: calls.clone(), - })) + PrefixedStorage::new(&config, |config| { + OpenDalStorageFactory::Memory.build_storage(config) }) - .unwrap(); - (storage, calls) + .unwrap() } - fn recorded(calls: &[(&str, &str)]) -> Vec<(String, String)> { - calls - .iter() - .map(|(scope, arg)| (scope.to_string(), arg.to_string())) - .collect() + fn timeout_for(storage: &PrefixedStorage, path: &str) -> Duration { + storage.storage_for(path).client_config().io_timeout() } - #[tokio::test] - async fn test_routes_a_path_to_its_longest_matching_prefix() { - let (storage, calls) = recording_storage(&[ - ("s3://bucket/data", "short"), - ("s3://bucket/data/warehouse", "long"), - ("", "empty"), - ]); - - for path in [ - "s3://bucket/data/warehouse/f", - "s3://bucket/data/other/f", - "s3://bucket/database/f", - "s3://bucket/elsewhere/f", - ] { - storage.exists(path).await.unwrap(); - } + #[test] + fn test_routes_a_path_to_its_longest_matching_prefix() { + let storage = memory_storage(&[("memory:/data", 2000), ("memory:/data/warehouse", 3000)]); + + let timeouts: Vec<_> = [ + "memory:/data/warehouse/f", + "memory:/data/other/f", + "memory:/database/f", + "memory:/elsewhere/f", + ] + .into_iter() + .map(|path| timeout_for(&storage, path)) + .collect(); assert_eq!( - *calls.lock().unwrap(), - recorded(&[ - ("long", "s3://bucket/data/warehouse/f"), - ("short", "s3://bucket/data/other/f"), - ("short", "s3://bucket/database/f"), - ("default", "s3://bucket/elsewhere/f"), - ]) + timeouts, + [3000, 2000, 2000, 1000].map(Duration::from_millis) ); } - #[tokio::test] - async fn test_a_later_credential_replaces_an_earlier_one_for_its_prefix() { - let (storage, calls) = recording_storage(&[ - ("s3://bucket/table/", "earlier"), - ("s3://bucket/table/", "later"), - ]); - - storage.exists("s3://bucket/table/f").await.unwrap(); + #[test] + fn test_a_later_credential_replaces_an_earlier_one_for_its_prefix() { + let storage = memory_storage(&[("memory:/table/", 2000), ("memory:/table/", 3000)]); assert_eq!( - *calls.lock().unwrap(), - recorded(&[("later", "s3://bucket/table/f")]) + timeout_for(&storage, "memory:/table/f"), + Duration::from_millis(3000) ); } #[tokio::test] async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash() { - for path in ["s3://bucket/table", "s3://bucket/table/"] { - let (storage, calls) = recording_storage(&[("s3://bucket/table/", "vended")]); + for path in ["memory:/table", "memory:/table/"] { + let storage = memory_storage(&[("memory:/table/", 2000)]); + storage + .write("memory:/table/f", Bytes::from("data")) + .await + .unwrap(); storage.delete_prefix(path).await.unwrap(); - assert_eq!(*calls.lock().unwrap(), recorded(&[("vended", path)])); + assert!(!storage.exists("memory:/table/f").await.unwrap(), "{path}"); } } #[tokio::test] - async fn test_delete_stream_batches_paths_per_prefix() { - let (storage, calls) = recording_storage(&[("s3://bucket/table/", "vended")]); - let paths = (0..DELETE_BATCH_SIZE + 1) - .map(|i| format!("s3://bucket/table/{i}")) - .chain(["s3://bucket/other/f".to_string()]); + async fn test_delete_stream_deletes_each_path_with_its_prefix_storage() { + let storage = memory_storage(&[("memory:/table/", 2000)]); + let paths: Vec<_> = (0..DELETE_BATCH_SIZE + 1) + .map(|i| format!("memory:/table/{i}")) + .chain(["memory:/other/f".to_string()]) + .collect(); + for path in &paths { + storage.write(path, Bytes::from("data")).await.unwrap(); + } storage - .delete_stream(stream::iter(paths).boxed()) + .delete_stream(stream::iter(paths.clone()).boxed()) .await .unwrap(); - let mut calls = calls.lock().unwrap().clone(); - calls.sort(); - assert_eq!( - calls, - recorded(&[("default", "1"), ("vended", "1"), ("vended", "1000")]) - ); + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } } + #[cfg(feature = "opendal-s3")] #[test] fn test_debug_omits_prefixed_storages() { - let (storage, _) = recording_storage(&[("s3://bucket/table/", "vended-secret")]); + let config = StorageConfig::builder() + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket/table/", + HashMap::from([( + iceberg::io::S3_SECRET_ACCESS_KEY.to_string(), + "vended-secret".to_string(), + )]), + ) + .unwrap(), + ]) + .build(); + let factory = OpenDalStorageFactory::S3 { + customized_credential_load: None, + }; + let storage = + PrefixedStorage::new(&config, |config| factory.build_storage(config)).unwrap(); let debug = format!("{storage:?}"); diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 4c23ac4208..6ec59078fe 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -27,13 +27,13 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredential, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; use url::Url; -use crate::prefixed::PrefixedStorage; +use crate::prefixed::{DELETE_BATCH_SIZE, PrefixedStorage}; #[cfg(feature = "opendal-s3")] use crate::s3::CustomAwsCredentialLoader; use crate::{OpenDalClientConfig, OpenDalStorage}; @@ -219,14 +219,13 @@ impl OpenDalResolvingStorageFactory { #[typetag::serde] impl StorageFactory for OpenDalResolvingStorageFactory { fn build(&self, config: &StorageConfig) -> Result> { - PrefixedStorage::build(config, |config| { - Ok(Arc::new(OpenDalResolvingStorage { - props: config.props().clone(), - storages: RwLock::new(HashMap::new()), - #[cfg(feature = "opendal-s3")] - customized_credential_load: self.customized_credential_load.clone(), - })) - }) + Ok(Arc::new(OpenDalResolvingStorage { + props: config.props().clone(), + storage_credentials: config.storage_credentials().to_vec(), + storages: RwLock::new(HashMap::new()), + #[cfg(feature = "opendal-s3")] + customized_credential_load: self.customized_credential_load.clone(), + })) } } @@ -240,9 +239,12 @@ impl StorageFactory for OpenDalResolvingStorageFactory { pub struct OpenDalResolvingStorage { /// Configuration properties shared across all backends. props: HashMap, + /// Credentials that apply to locations under their prefixes. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + storage_credentials: Vec, /// Cache of canonical scheme to storage mappings. #[serde(skip, default)] - storages: RwLock>>, + storages: RwLock>>, /// Custom AWS credential loader for S3 storage. #[cfg(feature = "opendal-s3")] #[serde(skip)] @@ -251,8 +253,8 @@ pub struct OpenDalResolvingStorage { impl OpenDalResolvingStorage { /// Resolve the storage for the given path by extracting the canonical scheme and - /// returning the cached or newly-created [`OpenDalStorage`]. - fn resolve(&self, path: &str) -> Result> { + /// returning the cached or newly-created storage for it. + fn resolve(&self, path: &str) -> Result> { let scheme = extract_scheme(path)?; // Fast path: check read lock first. @@ -277,12 +279,30 @@ impl OpenDalResolvingStorage { return Ok(storage.clone()); } - let storage = build_storage_for_scheme( - scheme, - &self.props, - #[cfg(feature = "opendal-s3")] - &self.customized_credential_load, - )?; + // A prefix such as `s3://bucket/` or `s3` names its scheme, and never + // matches the paths of another. + let storage_credentials = self + .storage_credentials + .iter() + .filter(|credential| { + let prefix = credential.prefix(); + let name = prefix.split_once(':').map_or(prefix, |(name, _)| name); + parse_scheme(&name.to_ascii_lowercase()).map_or(true, |s| s == scheme) + }) + .cloned() + .collect(); + let config = StorageConfig::builder() + .props(self.props.clone()) + .storage_credentials(storage_credentials) + .build(); + let storage = PrefixedStorage::new(&config, |config| { + build_storage_for_scheme( + scheme, + config.props(), + #[cfg(feature = "opendal-s3")] + &self.customized_credential_load, + ) + })?; let storage = Arc::new(storage); cache.insert(scheme, storage.clone()); Ok(storage) @@ -330,10 +350,21 @@ impl Storage for OpenDalResolvingStorage { let mut grouped: HashMap<&'static str, Vec> = HashMap::new(); while let Some(path) = paths.next().await { let scheme = extract_scheme(&path)?; - grouped.entry(scheme).or_default().push(path); + let batch = grouped.entry(scheme).or_default(); + batch.push(path); + if batch.len() >= DELETE_BATCH_SIZE { + let batch = std::mem::take(batch); + let storage = self.resolve(&batch[0])?; + storage + .delete_stream(futures::stream::iter(batch).boxed()) + .await?; + } } for (_, paths) in grouped { + if paths.is_empty() { + continue; + } let storage = self.resolve(&paths[0])?; storage .delete_stream(futures::stream::iter(paths).boxed()) @@ -343,17 +374,11 @@ impl Storage for OpenDalResolvingStorage { } fn new_input(&self, path: &str) -> Result { - Ok(InputFile::new( - Arc::new(self.resolve(path)?.as_ref().clone()), - path.to_string(), - )) + self.resolve(path)?.new_input(path) } fn new_output(&self, path: &str) -> Result { - Ok(OutputFile::new( - Arc::new(self.resolve(path)?.as_ref().clone()), - path.to_string(), - )) + self.resolve(path)?.new_output(path) } } @@ -399,6 +424,7 @@ mod tests { fn empty_resolving_storage() -> OpenDalResolvingStorage { OpenDalResolvingStorage { props: HashMap::new(), + storage_credentials: Vec::new(), storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: None, @@ -431,20 +457,106 @@ mod tests { for path in paths { let resolved = storage.resolve(path).unwrap(); assert_eq!( - resolved.client_config().io_timeout(), + resolved.storage_for(path).client_config().io_timeout(), Duration::from_secs(45), "{path}" ); } } + #[cfg(feature = "opendal-memory")] + #[test] + fn test_resolve_ignores_credentials_for_other_schemes() { + let mut storage = empty_resolving_storage(); + storage.storage_credentials = ["s3://bucket/", "s3"] + .map(|prefix| { + StorageCredential::new( + prefix, + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), "invalid".to_string())]), + ) + .unwrap() + }) + .to_vec(); + + assert!(storage.resolve("memory:/key").is_ok()); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_resolve_applies_a_scheme_name_prefix_to_its_aliases() { + let mut storage = empty_resolving_storage(); + storage.storage_credentials = vec![ + StorageCredential::new( + "s3", + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), "2000".to_string())]), + ) + .unwrap(), + ]; + + let timeouts = ["s3://bucket/key", "s3a://bucket/key"].map(|path| { + storage + .resolve(path) + .unwrap() + .storage_for(path) + .client_config() + .io_timeout() + }); + + assert_eq!(timeouts, [Duration::from_secs(2); 2]); + } + + /// A resolving storage holding one delete batch of memory files, and their paths. + #[cfg(feature = "opendal-memory")] + async fn storage_with_a_full_batch() -> (OpenDalResolvingStorage, Vec) { + let storage = empty_resolving_storage(); + let paths: Vec<_> = (0..DELETE_BATCH_SIZE) + .map(|i| format!("memory:/table/{i}")) + .collect(); + for path in &paths { + storage.write(path, Bytes::from("data")).await.unwrap(); + } + (storage, paths) + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test(start_paused = true)] + async fn test_delete_stream_deletes_a_full_batch_before_the_stream_ends() { + let (storage, paths) = storage_with_a_full_batch().await; + let stream = futures::stream::iter(paths.clone()) + .chain(futures::stream::pending()) + .boxed(); + + tokio::time::timeout(Duration::from_secs(1), storage.delete_stream(stream)) + .await + .unwrap_err(); + + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test] + async fn test_delete_stream_deletes_a_stream_of_exactly_one_batch() { + let (storage, paths) = storage_with_a_full_batch().await; + + storage + .delete_stream(futures::stream::iter(paths.clone()).boxed()) + .await + .unwrap(); + + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } + } + #[cfg(feature = "opendal-s3")] #[test] fn test_resolve_s3_aliases_share_instance() { let storage = empty_resolving_storage(); // All three S3-family schemes must collapse to a single cached - // `Arc` so that catalogs handing the resolver a mix + // storage so that catalogs handing the resolver a mix // of `s3://`, `s3a://`, `s3n://` paths don't rebuild operators. let a = storage.resolve("s3://bucket/key").unwrap(); let b = storage.resolve("s3a://bucket/key").unwrap(); @@ -459,10 +571,11 @@ mod tests { fn test_resolve_hdfs_native_returns_hdfs_variant() { let storage = empty_resolving_storage(); - let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); + let path = "hdfs://nameservice1/a/b"; + let resolved = storage.resolve(path).unwrap(); assert!( - matches!(&*resolved, OpenDalStorage::HdfsNative(_)), + matches!(resolved.storage_for(path), OpenDalStorage::HdfsNative(_)), "expected HdfsNative variant, got {resolved:?}" ); } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 057fe513fb..f178ece6d3 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -428,13 +428,16 @@ mod tests { (S3_REGION.to_string(), "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), ])) - .storage_credentials(vec![StorageCredential::new( - "s3://bucket1/vended/", - HashMap::from([ - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - ]), - )]) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + ) + .unwrap(), + ]) .build(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None, diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index 056cef450b..c11a948c6b 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -251,13 +251,16 @@ mod tests { (S3_REGION.to_string(), "us-east-1".to_string()), (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), ])) - .storage_credentials(vec![StorageCredential::new( - "s3://bucket1/vended/", - HashMap::from([ - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - ]), - )]) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + ) + .unwrap(), + ]) .build(); let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) .with_config(config) From ad2e55c2f96eddd9f7b1780c82d24330a802a92f Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 4 Nov 2026 07:04:50 -0800 Subject: [PATCH 26/27] test(storage/opendal): parameterize PrefixedStorage tests with rstest --- Cargo.lock | 1 + crates/storage/opendal/Cargo.toml | 1 + crates/storage/opendal/src/prefixed.rs | 77 +++++++++++--------------- 3 files changed, 33 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 667120ff16..bbfe017ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3163,6 +3163,7 @@ dependencies = [ "reqsign-aws-v4", "reqsign-core", "reqwest", + "rstest", "serde", "serde_json", "tempfile", diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 83541e89c5..3203d7a03b 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -73,6 +73,7 @@ url = { workspace = true } async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } reqwest = { workspace = true } +rstest = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "test-util"] } diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs index d5a9603bf6..f9a88d399b 100644 --- a/crates/storage/opendal/src/prefixed.rs +++ b/crates/storage/opendal/src/prefixed.rs @@ -193,6 +193,7 @@ mod tests { use std::time::Duration; use iceberg::io::StorageCredential; + use rstest::rstest; use super::*; use crate::{OPENDAL_IO_TIMEOUT_MS, OpenDalStorageFactory}; @@ -222,23 +223,20 @@ mod tests { storage.storage_for(path).client_config().io_timeout() } - #[test] - fn test_routes_a_path_to_its_longest_matching_prefix() { + #[rstest] + #[case::longest_prefix("memory:/data/warehouse/f", 3000)] + #[case::shorter_prefix("memory:/data/other/f", 2000)] + #[case::prefix_of_a_longer_name("memory:/database/f", 2000)] + #[case::no_prefix("memory:/elsewhere/f", 1000)] + fn test_routes_a_path_to_its_longest_matching_prefix( + #[case] path: &str, + #[case] expected_ms: u64, + ) { let storage = memory_storage(&[("memory:/data", 2000), ("memory:/data/warehouse", 3000)]); - let timeouts: Vec<_> = [ - "memory:/data/warehouse/f", - "memory:/data/other/f", - "memory:/database/f", - "memory:/elsewhere/f", - ] - .into_iter() - .map(|path| timeout_for(&storage, path)) - .collect(); - assert_eq!( - timeouts, - [3000, 2000, 2000, 1000].map(Duration::from_millis) + timeout_for(&storage, path), + Duration::from_millis(expected_ms) ); } @@ -252,19 +250,20 @@ mod tests { ); } + #[rstest] + #[case::without_slash("memory:/table")] + #[case::with_slash("memory:/table/")] #[tokio::test] - async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash() { - for path in ["memory:/table", "memory:/table/"] { - let storage = memory_storage(&[("memory:/table/", 2000)]); - storage - .write("memory:/table/f", Bytes::from("data")) - .await - .unwrap(); + async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash(#[case] path: &str) { + let storage = memory_storage(&[("memory:/table/", 2000)]); + storage + .write("memory:/table/f", Bytes::from("data")) + .await + .unwrap(); - storage.delete_prefix(path).await.unwrap(); + storage.delete_prefix(path).await.unwrap(); - assert!(!storage.exists("memory:/table/f").await.unwrap(), "{path}"); - } + assert!(!storage.exists("memory:/table/f").await.unwrap()); } #[tokio::test] @@ -288,30 +287,16 @@ mod tests { } } - #[cfg(feature = "opendal-s3")] #[test] fn test_debug_omits_prefixed_storages() { - let config = StorageConfig::builder() - .storage_credentials(vec![ - StorageCredential::new( - "s3://bucket/table/", - HashMap::from([( - iceberg::io::S3_SECRET_ACCESS_KEY.to_string(), - "vended-secret".to_string(), - )]), - ) - .unwrap(), - ]) - .build(); - let factory = OpenDalStorageFactory::S3 { - customized_credential_load: None, - }; - let storage = - PrefixedStorage::new(&config, |config| factory.build_storage(config)).unwrap(); - - let debug = format!("{storage:?}"); + let storage = memory_storage(&[("memory:/table/", 2000)]); - assert!(!debug.contains("vended-secret"), "{debug}"); - assert!(debug.contains("s3://bucket/table/"), "{debug}"); + assert_eq!( + format!("{storage:?}"), + format!( + r#"PrefixedStorage {{ default: {:?}, prefixes: ["memory:/table/"], .. }}"#, + storage.default + ) + ); } } From 5cb1bc64dca17fef2a66d0ba60ddbeb28bad26f6 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 5 Nov 2026 03:36:12 -0800 Subject: [PATCH 27/27] refactor(io): keep the last storage credential per prefix in StorageConfig --- crates/iceberg/src/io/storage/config/mod.rs | 71 +++++++++++++++++++-- crates/storage/opendal/src/prefixed.rs | 23 ++----- 2 files changed, 70 insertions(+), 24 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index 5d54ab2c3b..242e6f5a67 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -37,7 +37,7 @@ mod hf; mod oss; mod s3; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; pub use azdls::*; pub use gcs::*; @@ -61,11 +61,38 @@ pub struct StorageConfig { #[builder(default)] props: HashMap, /// Credentials that apply to locations under their prefixes - #[builder(default)] - #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[builder( + default, + setter(transform = |storage_credentials: Vec| { + last_credential_per_prefix(storage_credentials) + }) + )] + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "deserialize_storage_credentials" + )] storage_credentials: Vec, } +/// Keeps the last credential for each prefix, as Java's `S3FileIO.clientByPrefix` does. +fn last_credential_per_prefix(credentials: Vec) -> Vec { + let mut prefixes = HashSet::new(); + let mut credentials: Vec<_> = credentials + .into_iter() + .rev() + .filter(|credential| prefixes.insert(credential.prefix.clone())) + .collect(); + credentials.reverse(); + credentials +} + +fn deserialize_storage_credentials<'de, D: Deserializer<'de>>( + deserializer: D, +) -> std::result::Result, D::Error> { + Vec::deserialize(deserializer).map(last_credential_per_prefix) +} + impl StorageConfig { /// Create a new empty StorageConfig. pub fn new() -> Self { @@ -136,8 +163,9 @@ impl StorageConfig { /// Get the credentials that apply to locations under their prefixes. /// - /// A factory that supports them overlays the props with the config of a - /// location's longest matching credential. + /// There is one per prefix, the last one given for it. A factory that + /// supports them overlays the props with the config of a location's + /// longest matching credential. pub fn storage_credentials(&self) -> &[StorageCredential] { &self.storage_credentials } @@ -303,6 +331,39 @@ mod tests { assert_eq!(config, deserialized); } + #[test] + fn test_storage_config_keeps_the_last_credential_for_each_prefix() { + let credential = |prefix: &str, access_key_id: &str| { + StorageCredential::new( + prefix, + HashMap::from([("s3.access-key-id".to_string(), access_key_id.to_string())]), + ) + .unwrap() + }; + + let credentials = vec![ + credential("s3://bucket/a", "1"), + credential("s3://bucket/b", "2"), + credential("s3://bucket/a", "3"), + ]; + let expected = [ + credential("s3://bucket/b", "2"), + credential("s3://bucket/a", "3"), + ]; + + let built = StorageConfig::builder() + .storage_credentials(credentials.clone()) + .build(); + let deserialized: StorageConfig = serde_json::from_value(serde_json::json!({ + "props": {}, + "storage_credentials": credentials, + })) + .unwrap(); + + assert_eq!(built.storage_credentials(), expected); + assert_eq!(deserialized.storage_credentials(), expected); + } + #[test] fn test_storage_credential_debug_omits_config_values() { let credential = StorageCredential::new( diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs index f9a88d399b..20d90c107f 100644 --- a/crates/storage/opendal/src/prefixed.rs +++ b/crates/storage/opendal/src/prefixed.rs @@ -65,17 +65,12 @@ impl PrefixedStorage { build_fn: impl Fn(&StorageConfig) -> Result, ) -> Result { let base = StorageConfig::from_props(config.props().clone()); - // The last credential for a prefix wins, as in Java's `clientByPrefix`. - let credentials: HashMap<&str, &HashMap> = config + let mut prefixed = config .storage_credentials() .iter() - .map(|credential| (credential.prefix(), credential.config())) - .collect(); - let mut prefixed = credentials - .into_iter() - .map(|(prefix, credential)| { - let config = base.clone().with_props(credential.clone()); - Ok((prefix.to_string(), build_fn(&config)?)) + .map(|credential| { + let config = base.clone().with_props(credential.config().clone()); + Ok((credential.prefix().to_string(), build_fn(&config)?)) }) .collect::>>()?; prefixed.sort_by_key(|(prefix, _)| std::cmp::Reverse(prefix.len())); @@ -240,16 +235,6 @@ mod tests { ); } - #[test] - fn test_a_later_credential_replaces_an_earlier_one_for_its_prefix() { - let storage = memory_storage(&[("memory:/table/", 2000), ("memory:/table/", 3000)]); - - assert_eq!( - timeout_for(&storage, "memory:/table/f"), - Duration::from_millis(3000) - ); - } - #[rstest] #[case::without_slash("memory:/table")] #[case::with_slash("memory:/table/")]