From ce5e3ae68b3df13096be2f0cbf2fa61101c8789f Mon Sep 17 00:00:00 2001 From: Mike Williams Date: Wed, 11 Nov 2026 14:31:09 +1100 Subject: [PATCH] fix: read table files from non-Glue REST catalogs, using vended credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every `bergr rest … table …` command failed for REST catalogs other than AWS Glue's ("StorageFactory must be provided for REST catalog table operations"). Since iceberg 0.11, `RestCatalog` needs a storage factory to load a table, and only the Glue path supplied one. The plain REST catalog now gets an `OpenDalResolvingStorageFactory`, which picks storage by each path's scheme, and reads table files with the storage settings the catalog returns with each table. It also asks the catalog for vended credentials, by sending `X-Iceberg-Access-Delegation: vended-credentials`, as PyIceberg does by default. When the catalog vends nothing, opendal's default credential chain falls back to the user's own (for S3, AWS environment variables or profile); when both exist, vended credentials win. `--access-delegation none` (or BERGR_REST_ACCESS_DELEGATION=none) sends the spec's `none` value instead, for catalogs that shouldn't be asked. The option takes the spec's own values; `remote-signing` isn't offered, since iceberg-rust doesn't implement it. The Glue path is unchanged. Closes #41 --- README.md | 6 +++ src/cli.rs | 11 +++++ src/main.rs | 4 +- src/rest.rs | 139 +++++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 153 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c6d63f..ea930e3 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,12 @@ BERGR_REST_OAUTH2_CREDENTIAL="$(cat ~/.config/polaris-credential)" BERGR_REST_OA `bearer` and `oauth2` correspond to the Iceberg REST catalog properties `token`, `credential`, `scope` and `oauth2-server-uri`. `sigv4` currently only supports AWS Glue endpoints (`glue..amazonaws.com`). +#### Table files + +To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile. + +Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. AWS Glue's table files are always read with your AWS credentials. + ### Direct access via metadata file location ```bash diff --git a/src/cli.rs b/src/cli.rs index f14a0e9..0b9b1c1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,3 +1,4 @@ +use crate::rest::AccessDelegation; use crate::rest_auth::AuthType; use clap::{Parser, Subcommand}; @@ -42,6 +43,16 @@ pub enum Commands { #[arg(long, value_enum, env = "BERGR_REST_AUTH", default_value_t = AuthType::Auto)] auth: AuthType, + /// Whether to ask the catalog for credentials to read table files with. Ignored for + /// AWS Glue, whose table files are read with your AWS credentials. + #[arg( + long, + value_enum, + env = "BERGR_REST_ACCESS_DELEGATION", + default_value_t = AccessDelegation::VendedCredentials + )] + access_delegation: AccessDelegation, + #[command(subcommand)] command: CatalogCommands, }, diff --git a/src/main.rs b/src/main.rs index f9bb78c..00b43cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -78,6 +78,7 @@ async fn run(command: Commands) -> Result<()> { uri, warehouse, auth, + access_delegation, command, } => { let mut output = TerminalOutput::new(); @@ -94,7 +95,8 @@ async fn run(command: Commands) -> Result<()> { handle_catalog_command(&catalog, command, &mut output, Some(&s3)).await?; } RestAuth::Http(auth) => { - let catalog = rest_catalog(&uri, warehouse.as_deref(), &auth).await?; + let catalog = + rest_catalog(&uri, warehouse.as_deref(), &auth, access_delegation).await?; handle_catalog_command(&catalog, command, &mut output, None).await?; } } diff --git a/src/rest.rs b/src/rest.rs index ef4f5a9..f229a9e 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -3,12 +3,39 @@ use crate::basic_auth::BasicAuthManager; use crate::rest_auth::HttpAuth; use anyhow::Result; +use clap::ValueEnum; use iceberg::CatalogBuilder; use iceberg_catalog_rest::{ AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, REST_CATALOG_PROP_AUTH_TYPE, REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder, }; +use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use std::collections::HashMap; +use std::sync::Arc; + +/// iceberg-catalog-rest sends `header.*` properties as request headers +const ACCESS_DELEGATION_HEADER_PROP: &str = "header.X-Iceberg-Access-Delegation"; + +/// Whether to ask the catalog for credentials to read table files with (the Iceberg REST +/// `X-Iceberg-Access-Delegation` header) +#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AccessDelegation { + /// Ask the catalog to vend short-lived storage credentials with each table + #[default] + VendedCredentials, + /// Tell the catalog that bergr doesn't need delegated access, and read table files with your + /// own storage credentials (e.g. AWS environment variables or profile) + None, +} + +impl AccessDelegation { + fn header_value(self) -> &'static str { + match self { + AccessDelegation::VendedCredentials => "vended-credentials", + AccessDelegation::None => "none", + } + } +} /// Catalog properties for a REST catalog with the given URI and optional warehouse pub fn rest_catalog_props(uri: &str, warehouse: Option<&str>) -> HashMap { @@ -56,11 +83,15 @@ fn auth_props(auth: &HttpAuth) -> HashMap { /// Create a REST catalog with the given URI, optional warehouse location, and authentication /// +/// Table files are read with the storage settings, and any credentials, that the catalog returns +/// for each table. +/// /// # Arguments /// /// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181") /// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse") /// * `auth` - How to authenticate requests to the catalog +/// * `access_delegation` - Whether to ask the catalog for vended storage credentials /// /// # Returns /// @@ -69,15 +100,22 @@ pub async fn rest_catalog( uri: &str, warehouse: Option<&str>, auth: &HttpAuth, + access_delegation: AccessDelegation, ) -> Result { let mut props = rest_catalog_props(uri, warehouse); props.extend(auth_props(auth)); + props.insert( + ACCESS_DELEGATION_HEADER_PROP.to_string(), + access_delegation.header_value().to_string(), + ); + let builder = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalResolvingStorageFactory::new())); let builder = match auth { HttpAuth::Basic { user, password } => { - RestCatalogBuilder::default().with_auth_manager(BasicAuthManager::new(user, password)?) + builder.with_auth_manager(BasicAuthManager::new(user, password)?) } - _ => RestCatalogBuilder::default(), + _ => builder, }; Ok(builder.load("rest", props).await?) @@ -99,7 +137,8 @@ mod tests { let warehouse = Some("s3://test-warehouse"); // This should succeed - the catalog is created lazily - let result = rest_catalog(uri, warehouse, &HttpAuth::None).await; + let result = + rest_catalog(uri, warehouse, &HttpAuth::None, AccessDelegation::default()).await; // The catalog should be created successfully assert!( @@ -116,7 +155,7 @@ mod tests { let uri = "http://localhost:8181"; // Test that it works without an explicit warehouse (uses default) - let result = rest_catalog(uri, None, &HttpAuth::None).await; + let result = rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await; // Should succeed with default warehouse assert!( @@ -138,7 +177,8 @@ mod tests { ]; for uri in test_cases { - let result = rest_catalog(uri, None, &HttpAuth::None).await; + let result = + rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await; assert!( result.is_ok(), "Catalog creation should succeed for URI {}: {:?}", @@ -171,7 +211,7 @@ mod tests { .create_async() .await; - let catalog = rest_catalog(&server.url(), None, auth).await?; + let catalog = rest_catalog(&server.url(), None, auth, AccessDelegation::default()).await?; catalog.list_namespaces(None).await?; config.assert_async().await; @@ -231,4 +271,91 @@ mod tests { tokens.assert_async().await; Ok(()) } + + const TABLE_METADATA: &str = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/db/events", + "last-sequence-number": 0, + "last-updated-ms": 1700000000000, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}] + }"#; + + #[tokio::test] + async fn load_table_requests_and_uses_vended_credentials() -> Result<()> { + let mut server = Server::new_async().await; + server + .mock("GET", "/v1/config") + .match_query(Matcher::Any) + .with_body(r#"{"defaults": {}, "overrides": {}}"#) + .create_async() + .await; + let load_table = server + .mock("GET", "/v1/namespaces/db/tables/events") + .match_header("x-iceberg-access-delegation", "vended-credentials") + .with_body(format!( + r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json", + "metadata": {TABLE_METADATA}, + "config": {{"s3.access-key-id": "vended-key", "s3.secret-access-key": "vended-secret"}}}}"# + )) + .create_async() + .await; + + let catalog = rest_catalog( + &server.url(), + None, + &HttpAuth::None, + AccessDelegation::default(), + ) + .await?; + let table = catalog + .load_table(&iceberg::TableIdent::from_strs(["db", "events"])?) + .await?; + + load_table.assert_async().await; + assert_eq!( + table.file_io().config().props().get("s3.access-key-id"), + Some(&"vended-key".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn access_delegation_none_is_sent_as_none() -> Result<()> { + let mut server = Server::new_async().await; + let config = server + .mock("GET", "/v1/config") + .match_query(Matcher::Any) + .match_header("x-iceberg-access-delegation", "none") + .with_body(r#"{"defaults": {}, "overrides": {}}"#) + .create_async() + .await; + let load_table = server + .mock("GET", "/v1/namespaces/db/tables/events") + .match_header("x-iceberg-access-delegation", "none") + .with_body(format!( + r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json", + "metadata": {TABLE_METADATA}}}"# + )) + .create_async() + .await; + + let catalog = + rest_catalog(&server.url(), None, &HttpAuth::None, AccessDelegation::None).await?; + catalog + .load_table(&iceberg::TableIdent::from_strs(["db", "events"])?) + .await?; + + config.assert_async().await; + load_table.assert_async().await; + Ok(()) + } }