Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ async-trait = "0.1"
aws-config = "1.8.11"
aws-credential-types = "1.2.10"
aws-sdk-s3 = "1.115.0"
aws-sigv4 = "1.6.3"
aws-smithy-runtime-api = "1.19.1"
bytes = "1.11.0"
clap = { version = "4.5.53", features = ["derive"] }
futures = "0.3.31"
Expand All @@ -24,6 +26,7 @@ iceberg-catalog-glue = "0.11.0"
iceberg-catalog-rest = "0.11.0"
iceberg-storage-opendal = "0.11.0"
reqsign-core = "3.3.0"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
tokio = { version = "1.48.0", features = ["full"] }
Expand Down
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ bergr rest http://localhost:8181 namespace my_namespace info
bergr rest http://localhost:8181 namespace my_namespace tables
```

AWS Glue's Iceberg REST endpoint is detected from the URI, and requests to it are signed with SigV4 using your AWS credentials. Pass the catalog ID as the warehouse: the AWS account ID, for the default catalog.

```bash
bergr rest https://glue.us-west-2.amazonaws.com/iceberg --warehouse 123456789012 namespaces
```

Glue's REST endpoint needs `glue:GetCatalog` and `lakeformation:GetDataAccess` permissions, in addition to those the `glue` command needs.

### Direct access via metadata file location

```bash
Expand Down
28 changes: 27 additions & 1 deletion src/aws.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
//! AWS integration utilities for credential loading

use anyhow::Result;
use crate::rest::rest_catalog_props;
use crate::sigv4::{SigV4AuthManager, SigningScope};
use anyhow::{Context as _, Result};
use aws_config::BehaviorVersion;
use aws_config::meta::credentials::CredentialsProviderChain;
use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider};
Expand All @@ -10,6 +12,7 @@ use iceberg_catalog_glue::{
AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN,
GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder,
};
use iceberg_catalog_rest::{RestCatalog, RestCatalogBuilder};
use iceberg_storage_opendal::{
AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential,
};
Expand Down Expand Up @@ -140,6 +143,29 @@ pub async fn glue_catalog(aws_config: &aws_config::SdkConfig) -> Result<GlueCata
Ok(catalog)
}

/// Create a REST catalog hosted by AWS Glue, with SigV4-signed requests
pub async fn aws_rest_catalog(
aws_config: &aws_config::SdkConfig,
scope: SigningScope,
uri: &str,
warehouse: Option<&str>,
) -> Result<RestCatalog> {
let credentials = aws_config
.credentials_provider()
.context("AWS credentials not configured")?;

let mut props = rest_catalog_props(uri, warehouse);
props.insert(S3_REGION.to_string(), scope.region.clone());

let catalog = RestCatalogBuilder::default()
.with_auth_manager(SigV4AuthManager::new(scope, credentials))
.with_storage_factory(s3_storage_factory(aws_config))
.load("rest", props)
.await?;

Ok(catalog)
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ pub mod cli;
pub mod error;
pub mod file_existence;
pub mod rest;
pub mod sigv4;
pub mod table_commands;
pub mod terminal_output;
23 changes: 20 additions & 3 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
use anyhow::Result;
use bergr::aws::{get_aws_config, glue_catalog, s3_file_io};
use aws_config::Region;
use bergr::aws::{aws_rest_catalog, get_aws_config, glue_catalog, s3_file_io};
use bergr::catalog_commands::handle_catalog_command;
use bergr::cli::{Cli, Commands};
use bergr::error::ExpectedError;
use bergr::rest::rest_catalog;
use bergr::sigv4::SigningScope;
use bergr::table_commands::{handle_table_command, load_table};
use bergr::terminal_output::TerminalOutput;
use clap::Parser;
Expand Down Expand Up @@ -77,9 +79,24 @@ async fn run(command: Commands) -> Result<()> {
warehouse,
command,
} => {
let catalog = rest_catalog(&uri, warehouse.as_deref()).await?;
let mut output = TerminalOutput::new();
handle_catalog_command(&catalog, command, &mut output, None).await?;
match SigningScope::from_uri(&uri) {
Some(scope) => {
let aws_config = get_aws_config()
.await
.to_builder()
.region(Region::new(scope.region.clone()))
.build();
let catalog =
aws_rest_catalog(&aws_config, scope, &uri, warehouse.as_deref()).await?;
let s3 = aws_sdk_s3::Client::new(&aws_config);
handle_catalog_command(&catalog, command, &mut output, Some(&s3)).await?;
}
None => {
let catalog = rest_catalog(&uri, warehouse.as_deref()).await?;
handle_catalog_command(&catalog, command, &mut output, None).await?;
}
}
}
}

Expand Down
31 changes: 19 additions & 12 deletions src/rest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,8 @@ use iceberg_catalog_rest::{
};
use std::collections::HashMap;

/// Create a REST catalog with the given URI and optional warehouse location
///
/// # Arguments
///
/// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181")
/// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse")
///
/// # Returns
///
/// A configured `RestCatalog` instance
pub async fn rest_catalog(uri: &str, warehouse: Option<&str>) -> Result<RestCatalog> {
/// Catalog properties for a REST catalog with the given URI and optional warehouse
pub fn rest_catalog_props(uri: &str, warehouse: Option<&str>) -> HashMap<String, String> {
let mut props = HashMap::new();

// Required: REST catalog URI
Expand All @@ -31,7 +22,23 @@ pub async fn rest_catalog(uri: &str, warehouse: Option<&str>) -> Result<RestCata
warehouse_value.to_string(),
);

let catalog = RestCatalogBuilder::default().load("rest", props).await?;
props
}

/// Create a REST catalog with the given URI and optional warehouse location
///
/// # Arguments
///
/// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181")
/// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse")
///
/// # Returns
///
/// A configured `RestCatalog` instance
pub async fn rest_catalog(uri: &str, warehouse: Option<&str>) -> Result<RestCatalog> {
let catalog = RestCatalogBuilder::default()
.load("rest", rest_catalog_props(uri, warehouse))
.await?;

Ok(catalog)
}
Expand Down
Loading
Loading