Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
9519a87
feat(rest): use vended storage credentials from LoadTable response
plusplusjiajia Jun 15, 2026
1054fb4
feat(rest): per-prefix vended credentials with opt-in delegation header
plusplusjiajia Jun 15, 2026
1b8783f
fix(io): bound delete_stream memory by flushing per-prefix batches
plusplusjiajia Jun 17, 2026
4b4e18d
fix(rest): route vended credentials by prefix and keep them in object…
plusplusjiajia Jul 10, 2026
b072d0b
test(rest): cover vended credentials reaching FileIO and prefix routing
plusplusjiajia Aug 14, 2026
b547ab7
docs(io): note that prefix matching is raw-string, as in Java
plusplusjiajia Aug 14, 2026
9fca4ba
fix(rest): keep vended credentials on register_table
plusplusjiajia Sep 2, 2026
de52456
test(io): prefixed credentials survive FileIO serialization
plusplusjiajia Sep 9, 2026
106e43a
docs(rest): note that only Transaction can reach update_table
plusplusjiajia Sep 10, 2026
0a55ebc
fix(io): keep cached storages out of FileIO's Debug
plusplusjiajia Sep 20, 2026
c4e0674
fix(transaction): act with the refreshed FileIO and reload on any loc…
plusplusjiajia Sep 20, 2026
bdd8df0
fix(transaction): keep the table's initialized storage when a reload …
plusplusjiajia Sep 20, 2026
573f532
fix(transaction): return the actions' FileIO and never replay a durab…
plusplusjiajia Sep 28, 2026
e9411c7
fix(rest): preserve context in update_table
plusplusjiajia Oct 9, 2026
d2ff3c5
fix(io): delete through nested prefix storages in delete_prefix
plusplusjiajia Oct 10, 2026
f520a9c
fix(rest): replace inherited credential sets with vended ones
plusplusjiajia Oct 10, 2026
c321072
fix(io): keep nested prefix deletes within their own prefix
plusplusjiajia Oct 10, 2026
8023fa7
fix(io): reject nested raw prefixes and route delete_prefix by directory
plusplusjiajia Oct 10, 2026
b34bdee
docs: simplify vended credential comments
plusplusjiajia Oct 10, 2026
6dae418
refactor: drop redundant location check and overlapping tests
plusplusjiajia Oct 13, 2026
ebe09fc
test(transaction): implement test actions as StatelessAction
plusplusjiajia Oct 19, 2026
8aac866
refactor(io): carry vended credentials in StorageConfig for storage f…
plusplusjiajia Oct 28, 2026
78c8cb5
refactor(io): build StorageConfig with TypedBuilder and reuse Storage…
plusplusjiajia Oct 28, 2026
2b1aab8
refactor(io): pass a built StorageConfig through FileIOBuilder::with_…
plusplusjiajia Oct 29, 2026
19dc179
refactor(storage/opendal): hold OpenDalStorage in PrefixedStorage and…
plusplusjiajia Nov 3, 2026
ad2e55c
test(storage/opendal): parameterize PrefixedStorage tests with rstest
plusplusjiajia Nov 4, 2026
5cb1bc6
refactor(io): keep the last storage credential per prefix in StorageC…
plusplusjiajia Nov 5, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 1 addition & 16 deletions crates/catalog/rest/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ pub struct iceberg_catalog_rest::LoadTableResult
pub iceberg_catalog_rest::LoadTableResult::config: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
pub iceberg_catalog_rest::LoadTableResult::metadata: iceberg::spec::table_metadata::TableMetadata
pub iceberg_catalog_rest::LoadTableResult::metadata_location: core::option::Option<alloc::string::String>
pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option<alloc::vec::Vec<iceberg_catalog_rest::StorageCredential>>
pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option<alloc::vec::Vec<iceberg::io::storage::config::StorageCredential>>
impl core::clone::Clone for iceberg_catalog_rest::LoadTableResult
pub fn iceberg_catalog_rest::LoadTableResult::clone(&self) -> iceberg_catalog_rest::LoadTableResult
impl core::cmp::Eq for iceberg_catalog_rest::LoadTableResult
Expand Down Expand Up @@ -355,21 +355,6 @@ impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl iceberg_catalog_rest::SigV4Signer
pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())>
pub struct iceberg_catalog_rest::StorageCredential
pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String
impl core::clone::Clone for iceberg_catalog_rest::StorageCredential
pub fn iceberg_catalog_rest::StorageCredential::clone(&self) -> iceberg_catalog_rest::StorageCredential
impl core::cmp::Eq for iceberg_catalog_rest::StorageCredential
impl core::cmp::PartialEq for iceberg_catalog_rest::StorageCredential
pub fn iceberg_catalog_rest::StorageCredential::eq(&self, other: &iceberg_catalog_rest::StorageCredential) -> bool
impl core::fmt::Debug for iceberg_catalog_rest::StorageCredential
pub fn iceberg_catalog_rest::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl core::marker::StructuralPartialEq for iceberg_catalog_rest::StorageCredential
impl serde_core::ser::Serialize for iceberg_catalog_rest::StorageCredential
pub fn iceberg_catalog_rest::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::StorageCredential
pub fn iceberg_catalog_rest::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
pub struct iceberg_catalog_rest::UpdateNamespacePropertiesRequest
pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::removals: core::option::Option<alloc::vec::Vec<alloc::string::String>>
pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::updates: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
Expand Down
110 changes: 104 additions & 6 deletions crates/catalog/rest/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ use std::time::Duration;
use async_trait::async_trait;
use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory};
use iceberg::io::{
ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory,
ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageConfig,
StorageCredential, StorageFactory,
};
use iceberg::table::Table;
use iceberg::{
Expand Down Expand Up @@ -987,6 +988,7 @@ impl RestSessionCatalog {
&self,
metadata_location: Option<&str>,
extra_config: Option<HashMap<String, String>>,
storage_credentials: Option<Vec<StorageCredential>>,
Comment thread
blackmwk marked this conversation as resolved.
Resolved
) -> Result<FileIO> {
let mut props = self.client().await?.config.props.clone();
if let Some(config) = extra_config {
Expand Down Expand Up @@ -1019,7 +1021,11 @@ impl RestSessionCatalog {
)
})?;

let file_io = FileIOBuilder::new(factory).with_props(props).build();
let config = StorageConfig::builder()
.props(props)
.storage_credentials(storage_credentials.unwrap_or_default())
.build();
let file_io = FileIOBuilder::new(factory).with_config(config).build();

Ok(file_io)
}
Expand Down Expand Up @@ -1342,7 +1348,11 @@ impl SessionCatalog for RestSessionCatalog {
.collect();

let file_io = self
.load_file_io(Some(metadata_location), Some(config))
.load_file_io(
Some(metadata_location),
Some(config),
response.storage_credentials,
)
.await?;

let mut table_builder = Table::builder()
Expand Down Expand Up @@ -1439,7 +1449,11 @@ impl SessionCatalog for RestSessionCatalog {
.collect();

let file_io = self
.load_file_io(response.metadata_location.as_deref(), Some(config))
.load_file_io(
response.metadata_location.as_deref(),
Some(config),
response.storage_credentials,
)
.await?;

let mut table_builder = Table::builder()
Expand Down Expand Up @@ -1590,7 +1604,19 @@ impl SessionCatalog for RestSessionCatalog {
"Metadata location missing in `register_table` response!",
))?;

let file_io = self.load_file_io(Some(metadata_location), None).await?;
let config = response
.config
.into_iter()
.chain(self.user_config.props.clone())
.collect();

let file_io = self
.load_file_io(
Some(metadata_location),
Some(config),
response.storage_credentials,
)
.await?;

let mut table_builder = Table::builder()
.identifier(table_ident.clone())
Expand Down Expand Up @@ -1670,7 +1696,7 @@ impl SessionCatalog for RestSessionCatalog {
};

let file_io = self
.load_file_io(Some(&response.metadata_location), None)
.load_file_io(Some(&response.metadata_location), None, None)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wonder what the risks are here of not creating FileIo here without the credentials? Is that idea that after we have credential refresh we will just automatically refresh on the next read / write?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@xanderbailey Good question. No risk on the Transaction::commit path: do_commit loads before committing and swaps that load's credentialed FileIO back in afterwards (with_file_io), reloading on SetLocation. Refresh isn't needed here. Only a direct Catalog::update_table caller sees the plain FileIO, and the commit response has no credentials to give it by spec — I've pointed the comment at do_commit.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess in Java they have TableOperations which carries the FileIo rather than having to build a new one after the update 🤔 . There are advantages to the way our commit path works so not saying we should look to move in that direction. Just trying to work out how we could make this more bullet-proof.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@xanderbailey Good comparison — I think we're actually closer to it than it looks. UpdateTableRequest in Java doesn't carry a FileIO either; it stays on TableOperations and commit() leaves it alone. do_commit keeping the table's FileIO across update_table is the same idea, just spelled out because we build a fresh Table rather than mutating ops in place.

On bullet-proofing: TableCommit::build() is pub(crate), so do_commit ends up being the only caller update_table can have. I've added a note to the comment so that's easier to see

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update: I've since taken the Transaction change out of this PR to keep it within storage, so the table Transaction::commit returns doesn't carry the vended credentials yet. I'll fix that in a follow-up.

.await?;

let mut table_builder = Table::builder()
Expand Down Expand Up @@ -4761,6 +4787,50 @@ mod tests {
load.assert_async().await;
}

/// A `loadTable` body vending a storage credential, and that credential.
fn load_table_body_with_credential() -> (serde_json::Value, StorageCredential) {
let credential = StorageCredential::new(
"s3://warehouse/database/table",
HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]),
)
.unwrap();
let mut body = load_table_body("v1");
body["storage-credentials"] =
json!([{ "prefix": credential.prefix(), "config": credential.config() }]);
(body, credential)
}

#[tokio::test]
async fn test_load_table_passes_vended_credentials_to_file_io() {
let mut server = Server::new_async().await;
let _config = create_config_mock(&mut server).await;
let (body, credential) = load_table_body_with_credential();
let load = server
.mock("GET", "/v1/namespaces/ns1/tables/test1")
.match_header("x-iceberg-access-delegation", "vended-credentials")
.with_status(200)
.with_body(body.to_string())
.create_async()
.await;
let catalog = session_catalog(
RestCatalogConfig::builder()
.uri(server.url())
.props(HashMap::from([(
"header.X-Iceberg-Access-Delegation".to_string(),
"vended-credentials".to_string(),
)]))
.build(),
);

let table = catalog
.load_table(&SessionContext::empty(), &test_table_ident())
.await
.unwrap();

assert_eq!(table.file_io().config().storage_credentials(), [credential]);
load.assert_async().await;
}

#[test]
fn test_table_cache_rejects_out_of_range_expiry() {
for value in ["0", "31536000000001"] {
Expand Down Expand Up @@ -5334,6 +5404,34 @@ mod tests {
register_table_mock.assert_async().await;
}

#[tokio::test]
async fn test_register_table_passes_vended_config_and_credentials_to_file_io() {
let mut server = Server::new_async().await;
let _config = create_config_mock(&mut server).await;
let (body, credential) = load_table_body_with_credential();
let register = server
.mock("POST", "/v1/namespaces/ns1/register")
.with_status(200)
.with_body(body.to_string())
.create_async()
.await;
let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build());

let table = catalog
.register_table(
&SessionContext::empty(),
&test_table_ident(),
"s3://warehouse/v1.metadata.json".to_string(),
)
.await
.unwrap();

let config = table.file_io().config();
assert_eq!(config.get("region").map(String::as_str), Some("us-west-2"));
assert_eq!(config.storage_credentials(), [credential]);
register.assert_async().await;
}

#[tokio::test]
async fn test_register_table_404() {
let mut server = Server::new_async().await;
Expand Down
14 changes: 1 addition & 13 deletions crates/catalog/rest/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

use std::collections::HashMap;

use iceberg::io::StorageCredential;
use iceberg::spec::{Schema, SortOrder, TableMetadata, UnboundPartitionSpec};
use iceberg::{
Error, ErrorKind, Namespace, NamespaceIdent, TableIdent, TableRequirement, TableUpdate,
Expand Down Expand Up @@ -231,19 +232,6 @@ pub struct LoadTableResult {
pub storage_credentials: Option<Vec<StorageCredential>>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
/// Storage credential for a specific location prefix.
///
/// Indicates a storage location prefix where the credential is relevant. Clients should
/// choose the most specific prefix (by selecting the longest prefix) if several credentials
/// of the same type are available.
pub struct StorageCredential {
/// Storage location prefix where this credential is relevant
pub prefix: String,
/// Configuration map containing credential information
pub config: HashMap<String, String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
/// Request to create a new table in a namespace.
Expand Down
21 changes: 21 additions & 0 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,7 @@ impl iceberg::io::FileIOBuilder
pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO
pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig
pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc<dyn iceberg::io::StorageFactory>) -> Self
pub fn iceberg::io::FileIOBuilder::with_config(self, config: iceberg::io::StorageConfig) -> Self
pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self
pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator<Item = (impl alloc::string::ToString, impl alloc::string::ToString)>) -> Self
impl core::clone::Clone for iceberg::io::FileIOBuilder
Expand Down Expand Up @@ -1100,6 +1101,7 @@ pub fn iceberg::io::StorageConfig::from_props(props: std::collections::hash::map
pub fn iceberg::io::StorageConfig::get(&self, key: &str) -> core::option::Option<&alloc::string::String>
pub fn iceberg::io::StorageConfig::new() -> Self
pub fn iceberg::io::StorageConfig::props(&self) -> &std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
pub fn iceberg::io::StorageConfig::storage_credentials(&self) -> &[iceberg::io::StorageCredential]
pub fn iceberg::io::StorageConfig::with_prop(self, key: impl core::convert::Into<alloc::string::String>, value: impl core::convert::Into<alloc::string::String>) -> Self
pub fn iceberg::io::StorageConfig::with_props(self, props: impl core::iter::traits::collect::IntoIterator<Item = (impl core::convert::Into<alloc::string::String>, impl core::convert::Into<alloc::string::String>)>) -> Self
impl core::clone::Clone for iceberg::io::StorageConfig
Expand Down Expand Up @@ -1127,10 +1129,29 @@ pub fn iceberg::io::StorageConfig::default() -> iceberg::io::StorageConfig
impl core::fmt::Debug for iceberg::io::StorageConfig
pub fn iceberg::io::StorageConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl core::marker::StructuralPartialEq for iceberg::io::StorageConfig
impl iceberg::io::StorageConfig
pub fn iceberg::io::StorageConfig::builder() -> StorageConfigBuilder<((), ())>
impl serde_core::ser::Serialize for iceberg::io::StorageConfig
pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig
pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
pub struct iceberg::io::StorageCredential
impl iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>
pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into<alloc::string::String>, config: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>) -> iceberg::Result<Self>
pub fn iceberg::io::StorageCredential::prefix(&self) -> &str
impl core::clone::Clone for iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential
impl core::cmp::Eq for iceberg::io::StorageCredential
impl core::cmp::PartialEq for iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::eq(&self, other: &iceberg::io::StorageCredential) -> bool
impl core::fmt::Debug for iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential
impl serde_core::ser::Serialize for iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageCredential
pub fn iceberg::io::StorageCredential::deserialize<D: serde_core::de::Deserializer<'de>>(deserializer: D) -> core::result::Result<Self, <D as serde_core::de::Deserializer>::Error>
pub const iceberg::io::ADLS_ACCOUNT_KEY: &str
pub const iceberg::io::ADLS_ACCOUNT_NAME: &str
pub const iceberg::io::ADLS_AUTHORITY_HOST: &str
Expand Down
19 changes: 18 additions & 1 deletion crates/iceberg/src/io/file_io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,12 @@ impl FileIOBuilder {
self
}

/// Replace the storage configuration set so far.
pub fn with_config(mut self, config: StorageConfig) -> Self {
self.config = config;
self
}

/// Get the storage configuration.
pub fn config(&self) -> &StorageConfig {
&self.config
Expand Down Expand Up @@ -453,7 +459,7 @@ mod tests {
use tempfile::TempDir;

use super::{FileIO, FileIOBuilder};
use crate::io::{LocalFsStorageFactory, MemoryStorageFactory};
use crate::io::{LocalFsStorageFactory, MemoryStorageFactory, StorageConfig};

fn create_local_file_io() -> FileIO {
FileIO::new_with_fs()
Expand Down Expand Up @@ -601,6 +607,17 @@ mod tests {
assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string()));
}

#[test]
fn test_file_io_builder_with_config_replaces_config() {
let config = StorageConfig::new().with_prop("key2", "value2");
let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory))
.with_prop("key1", "value1")
.with_config(config.clone())
.build();

assert_eq!(file_io.config(), &config);
}

#[tokio::test]
async fn test_memory_file_io_serialization_roundtrip() {
let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory))
Expand Down
Loading
Loading