Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ BERGR_REST_OAUTH2_CREDENTIAL="$(cat ~/.config/polaris-credential)" BERGR_REST_OA

`bearer` and `oauth2` correspond to the Iceberg REST catalog properties `token`, `credential`, `scope` and `oauth2-server-uri`. `sigv4` currently only supports AWS Glue endpoints (`glue.<region>.amazonaws.com`).

#### Table files

To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile.

Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. AWS Glue's table files are always read with your AWS credentials.

### Direct access via metadata file location

```bash
Expand Down
11 changes: 11 additions & 0 deletions src/cli.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::rest::AccessDelegation;
use crate::rest_auth::AuthType;
use clap::{Parser, Subcommand};

Expand Down Expand Up @@ -42,6 +43,16 @@ pub enum Commands {
#[arg(long, value_enum, env = "BERGR_REST_AUTH", default_value_t = AuthType::Auto)]
auth: AuthType,

/// Whether to ask the catalog for credentials to read table files with. Ignored for
/// AWS Glue, whose table files are read with your AWS credentials.
#[arg(
long,
value_enum,
env = "BERGR_REST_ACCESS_DELEGATION",
default_value_t = AccessDelegation::VendedCredentials
)]
access_delegation: AccessDelegation,

#[command(subcommand)]
command: CatalogCommands,
},
Expand Down
4 changes: 3 additions & 1 deletion src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ async fn run(command: Commands) -> Result<()> {
uri,
warehouse,
auth,
access_delegation,
command,
} => {
let mut output = TerminalOutput::new();
Expand All @@ -94,7 +95,8 @@ async fn run(command: Commands) -> Result<()> {
handle_catalog_command(&catalog, command, &mut output, Some(&s3)).await?;
}
RestAuth::Http(auth) => {
let catalog = rest_catalog(&uri, warehouse.as_deref(), &auth).await?;
let catalog =
rest_catalog(&uri, warehouse.as_deref(), &auth, access_delegation).await?;
handle_catalog_command(&catalog, command, &mut output, None).await?;
}
}
Expand Down
139 changes: 133 additions & 6 deletions src/rest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,39 @@
use crate::basic_auth::BasicAuthManager;
use crate::rest_auth::HttpAuth;
use anyhow::Result;
use clap::ValueEnum;
use iceberg::CatalogBuilder;
use iceberg_catalog_rest::{
AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, REST_CATALOG_PROP_AUTH_TYPE, REST_CATALOG_PROP_URI,
REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder,
};
use iceberg_storage_opendal::OpenDalResolvingStorageFactory;
use std::collections::HashMap;
use std::sync::Arc;

/// iceberg-catalog-rest sends `header.*` properties as request headers
const ACCESS_DELEGATION_HEADER_PROP: &str = "header.X-Iceberg-Access-Delegation";

/// Whether to ask the catalog for credentials to read table files with (the Iceberg REST
/// `X-Iceberg-Access-Delegation` header)
#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AccessDelegation {
/// Ask the catalog to vend short-lived storage credentials with each table
#[default]
VendedCredentials,
/// Tell the catalog that bergr doesn't need delegated access, and read table files with your
/// own storage credentials (e.g. AWS environment variables or profile)
None,
}

impl AccessDelegation {
fn header_value(self) -> &'static str {
match self {
AccessDelegation::VendedCredentials => "vended-credentials",
AccessDelegation::None => "none",
}
}
}

/// Catalog properties for a REST catalog with the given URI and optional warehouse
pub fn rest_catalog_props(uri: &str, warehouse: Option<&str>) -> HashMap<String, String> {
Expand Down Expand Up @@ -56,11 +83,15 @@ fn auth_props(auth: &HttpAuth) -> HashMap<String, String> {

/// Create a REST catalog with the given URI, optional warehouse location, and authentication
///
/// Table files are read with the storage settings, and any credentials, that the catalog returns
/// for each table.
///
/// # Arguments
///
/// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181")
/// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse")
/// * `auth` - How to authenticate requests to the catalog
/// * `access_delegation` - Whether to ask the catalog for vended storage credentials
///
/// # Returns
///
Expand All @@ -69,15 +100,22 @@ pub async fn rest_catalog(
uri: &str,
warehouse: Option<&str>,
auth: &HttpAuth,
access_delegation: AccessDelegation,
) -> Result<RestCatalog> {
let mut props = rest_catalog_props(uri, warehouse);
props.extend(auth_props(auth));
props.insert(
ACCESS_DELEGATION_HEADER_PROP.to_string(),
access_delegation.header_value().to_string(),
);

let builder = RestCatalogBuilder::default()
.with_storage_factory(Arc::new(OpenDalResolvingStorageFactory::new()));
let builder = match auth {
HttpAuth::Basic { user, password } => {
RestCatalogBuilder::default().with_auth_manager(BasicAuthManager::new(user, password)?)
builder.with_auth_manager(BasicAuthManager::new(user, password)?)
}
_ => RestCatalogBuilder::default(),
_ => builder,
};

Ok(builder.load("rest", props).await?)
Expand All @@ -99,7 +137,8 @@ mod tests {
let warehouse = Some("s3://test-warehouse");

// This should succeed - the catalog is created lazily
let result = rest_catalog(uri, warehouse, &HttpAuth::None).await;
let result =
rest_catalog(uri, warehouse, &HttpAuth::None, AccessDelegation::default()).await;

// The catalog should be created successfully
assert!(
Expand All @@ -116,7 +155,7 @@ mod tests {
let uri = "http://localhost:8181";

// Test that it works without an explicit warehouse (uses default)
let result = rest_catalog(uri, None, &HttpAuth::None).await;
let result = rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await;

// Should succeed with default warehouse
assert!(
Expand All @@ -138,7 +177,8 @@ mod tests {
];

for uri in test_cases {
let result = rest_catalog(uri, None, &HttpAuth::None).await;
let result =
rest_catalog(uri, None, &HttpAuth::None, AccessDelegation::default()).await;
assert!(
result.is_ok(),
"Catalog creation should succeed for URI {}: {:?}",
Expand Down Expand Up @@ -171,7 +211,7 @@ mod tests {
.create_async()
.await;

let catalog = rest_catalog(&server.url(), None, auth).await?;
let catalog = rest_catalog(&server.url(), None, auth, AccessDelegation::default()).await?;
catalog.list_namespaces(None).await?;

config.assert_async().await;
Expand Down Expand Up @@ -231,4 +271,91 @@ mod tests {
tokens.assert_async().await;
Ok(())
}

const TABLE_METADATA: &str = r#"{
"format-version": 2,
"table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
"location": "s3://bucket/db/events",
"last-sequence-number": 0,
"last-updated-ms": 1700000000000,
"last-column-id": 1,
"current-schema-id": 0,
"schemas": [{"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]}],
"default-spec-id": 0,
"partition-specs": [{"spec-id": 0, "fields": []}],
"last-partition-id": 999,
"default-sort-order-id": 0,
"sort-orders": [{"order-id": 0, "fields": []}]
}"#;

#[tokio::test]
async fn load_table_requests_and_uses_vended_credentials() -> Result<()> {
let mut server = Server::new_async().await;
server
.mock("GET", "/v1/config")
.match_query(Matcher::Any)
.with_body(r#"{"defaults": {}, "overrides": {}}"#)
.create_async()
.await;
let load_table = server
.mock("GET", "/v1/namespaces/db/tables/events")
.match_header("x-iceberg-access-delegation", "vended-credentials")
.with_body(format!(
r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json",
"metadata": {TABLE_METADATA},
"config": {{"s3.access-key-id": "vended-key", "s3.secret-access-key": "vended-secret"}}}}"#
))
.create_async()
.await;

let catalog = rest_catalog(
&server.url(),
None,
&HttpAuth::None,
AccessDelegation::default(),
)
.await?;
let table = catalog
.load_table(&iceberg::TableIdent::from_strs(["db", "events"])?)
.await?;

load_table.assert_async().await;
assert_eq!(
table.file_io().config().props().get("s3.access-key-id"),
Some(&"vended-key".to_string())
);
Ok(())
}

#[tokio::test]
async fn access_delegation_none_is_sent_as_none() -> Result<()> {
let mut server = Server::new_async().await;
let config = server
.mock("GET", "/v1/config")
.match_query(Matcher::Any)
.match_header("x-iceberg-access-delegation", "none")
.with_body(r#"{"defaults": {}, "overrides": {}}"#)
.create_async()
.await;
let load_table = server
.mock("GET", "/v1/namespaces/db/tables/events")
.match_header("x-iceberg-access-delegation", "none")
.with_body(format!(
r#"{{"metadata-location": "s3://bucket/db/events/metadata/00000.metadata.json",
"metadata": {TABLE_METADATA}}}"#
))
.create_async()
.await;

let catalog =
rest_catalog(&server.url(), None, &HttpAuth::None, AccessDelegation::None).await?;
catalog
.load_table(&iceberg::TableIdent::from_strs(["db", "events"])?)
.await?;

config.assert_async().await;
load_table.assert_async().await;
Ok(())
}
}
Loading