diff --git a/README.md b/README.md index 1c6d63f..ea930e3 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,12 @@ BERGR_REST_OAUTH2_CREDENTIAL="$(cat ~/.config/polaris-credential)" BERGR_REST_OA `bearer` and `oauth2` correspond to the Iceberg REST catalog properties `token`, `credential`, `scope` and `oauth2-server-uri`. `sigv4` currently only supports AWS Glue endpoints (`glue..amazonaws.com`). +#### Table files + +To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile. + +Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. AWS Glue's table files are always read with your AWS credentials. + ### Direct access via metadata file location ```bash diff --git a/src/cli.rs b/src/cli.rs index f14a0e9..0b9b1c1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,3 +1,4 @@ +use crate::rest::AccessDelegation; use crate::rest_auth::AuthType; use clap::{Parser, Subcommand}; @@ -42,6 +43,16 @@ pub enum Commands { #[arg(long, value_enum, env = "BERGR_REST_AUTH", default_value_t = AuthType::Auto)] auth: AuthType, + /// Whether to ask the catalog for credentials to read table files with. Ignored for + /// AWS Glue, whose table files are read with your AWS credentials. + #[arg( + long, + value_enum, + env = "BERGR_REST_ACCESS_DELEGATION", + default_value_t = AccessDelegation::VendedCredentials + )] + access_delegation: AccessDelegation, + #[command(subcommand)] command: CatalogCommands, }, diff --git a/src/main.rs b/src/main.rs index f9bb78c..00b43cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -78,6 +78,7 @@ async fn run(command: Commands) -> Result<()> { uri, warehouse, auth, + access_delegation, command, } => { let mut output = TerminalOutput::new(); @@ -94,7 +95,8 @@ async fn run(command: Commands) -> Result<()> { handle_catalog_command(&catalog, command, &mut output, Some(&s3)).await?; } RestAuth::Http(auth) => { - let catalog = rest_catalog(&uri, warehouse.as_deref(), &auth).await?; + let catalog = + rest_catalog(&uri, warehouse.as_deref(), &auth, access_delegation).await?; handle_catalog_command(&catalog, command, &mut output, None).await?; } } diff --git a/src/rest.rs b/src/rest.rs index ef4f5a9..f229a9e 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -3,12 +3,39 @@ use crate::basic_auth::BasicAuthManager; use crate::rest_auth::HttpAuth; use anyhow::Result; +use clap::ValueEnum; use iceberg::CatalogBuilder; use iceberg_catalog_rest::{ AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, REST_CATALOG_PROP_AUTH_TYPE, REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder, }; +use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use std::collections::HashMap; +use std::sync::Arc; + +/// iceberg-catalog-rest sends `header.*` properties as request headers +const ACCESS_DELEGATION_HEADER_PROP: &str = "header.X-Iceberg-Access-Delegation"; + +/// Whether to ask the catalog for credentials to read table files with (the Iceberg REST +/// `X-Iceberg-Access-Delegation` header) +#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AccessDelegation { + /// Ask the catalog to vend short-lived storage credentials with each table + #[default] + VendedCredentials, + /// Tell the catalog that bergr doesn't need delegated access, and read table files with your + /// own storage credentials (e.g. AWS environment variables or profile) + None, +} + +impl AccessDelegation { + fn header_value(self) -> &'static str { + match self { + AccessDelegation::VendedCredentials => "vended-credentials", + AccessDelegation::None => "none", + } + } +} /// Catalog properties for a REST catalog with the given URI and optional warehouse pub fn rest_catalog_props(uri: &str, warehouse: Option<&str>) -> HashMap { @@ -56,11 +83,15 @@ fn auth_props(auth: &HttpAuth) -> HashMap { /// Create a REST catalog with the given URI, optional warehouse location, and authentication /// +/// Table files are read with the storage settings, and any credentials, that the catalog returns +/// for each table. +/// /// # Arguments /// /// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181") /// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse") /// * `auth` - How to authenticate requests to the catalog +/// * `access_delegation` - Whether to ask the catalog for vended storage credentials /// /// # Returns /// @@ -69,15 +100,22 @@ pub async fn rest_catalog( uri: &str, warehouse: Option<&str>, auth: &HttpAuth, + access_delegation: AccessDelegation, ) -> Result { let mut props = rest_catalog_props(uri, warehouse); props.extend(auth_props(auth)); + props.insert( + ACCESS_DELEGATION_HEADER_PROP.to_string(), + access_delegation.header_value().to_string(), + ); + let builder = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalResolvingStorageFactory::new())); let builder = match auth { HttpAuth::Basic { user, password } => { - RestCatalogBuilder::default().with_auth_manager(BasicAuthManager::new(user, password)?) + builder.with_auth_manager(BasicAuthManager::new(user, password)?) } - _ => RestCatalogBuilder::default(), + _ => builder, }; Ok(builder.load("rest", props).await?) @@ -99,7 +137,8 @@ mod tests { let warehouse = Some("s3://test-warehouse"); // This should succeed - the catalog is created lazily - let result = rest_catalog(uri, warehouse, &HttpAuth::None).await; + let result = + rest_catalog(uri, warehouse, &HttpAuth::None, AccessDelegation::default()).await; // The catalog should be created successfully assert!( @@ -116,7 +155,7 @@ mod tests { let uri = "http://localhost:8181"; // Test that it works without an explicit warehouse (uses default) - let result = rest_catalog(uri, None, &HttpAuth::None).await; + let result = rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await; // Should succeed with default warehouse assert!( @@ -138,7 +177,8 @@ mod tests { ]; for uri in test_cases { - let result = rest_catalog(uri, None, &HttpAuth::None).await; + let result = + rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await; assert!( result.is_ok(), "Catalog creation should succeed for URI {}: {:?}", @@ -171,7 +211,7 @@ mod tests { .create_async() .await; - let catalog = rest_catalog(&server.url(), None, auth).await?; + let catalog = rest_catalog(&server.url(), None, auth, AccessDelegation::default()).await?; catalog.list_namespaces(None).await?; config.assert_async().await; @@ -231,4 +271,91 @@ mod tests { tokens.assert_async().await; Ok(()) } + + const TABLE_METADATA: &str = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/db/events", + "last-sequence-number": 0, + "last-updated-ms": 1700000000000, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, + "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}] + }"#; + + #[tokio::test] + async fn load_table_requests_and_uses_vended_credentials() -> Result<()> { + let mut server = Server::new_async().await; + server + .mock("GET", "/v1/config") + .match_query(Matcher::Any) + .with_body(r#"{"defaults": {}, "overrides": {}}"#) + .create_async() + .await; + let load_table = server + .mock("GET", "/v1/namespaces/db/tables/events") + .match_header("x-iceberg-access-delegation", "vended-credentials") + .with_body(format!( + r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json", + "metadata": {TABLE_METADATA}, + "config": {{"s3.access-key-id": "vended-key", "s3.secret-access-key": "vended-secret"}}}}"# + )) + .create_async() + .await; + + let catalog = rest_catalog( + &server.url(), + None, + &HttpAuth::None, + AccessDelegation::default(), + ) + .await?; + let table = catalog + .load_table(&iceberg::TableIdent::from_strs(["db", "events"])?) + .await?; + + load_table.assert_async().await; + assert_eq!( + table.file_io().config().props().get("s3.access-key-id"), + Some(&"vended-key".to_string()) + ); + Ok(()) + } + + #[tokio::test] + async fn access_delegation_none_is_sent_as_none() -> Result<()> { + let mut server = Server::new_async().await; + let config = server + .mock("GET", "/v1/config") + .match_query(Matcher::Any) + .match_header("x-iceberg-access-delegation", "none") + .with_body(r#"{"defaults": {}, "overrides": {}}"#) + .create_async() + .await; + let load_table = server + .mock("GET", "/v1/namespaces/db/tables/events") + .match_header("x-iceberg-access-delegation", "none") + .with_body(format!( + r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json", + "metadata": {TABLE_METADATA}}}"# + )) + .create_async() + .await; + + let catalog = + rest_catalog(&server.url(), None, &HttpAuth::None, AccessDelegation::None).await?; + catalog + .load_table(&iceberg::TableIdent::from_strs(["db", "events"])?) + .await?; + + config.assert_async().await; + load_table.assert_async().await; + Ok(()) + } }