diff --git a/Cargo.lock b/Cargo.lock index 7dd460b..97cdd45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -970,6 +970,8 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-sigv4", + "aws-smithy-runtime-api", "bytes", "clap", "futures", @@ -979,6 +981,7 @@ dependencies = [ "iceberg-catalog-rest", "iceberg-storage-opendal", "reqsign-core", + "reqwest 0.12.28", "serde", "serde_json", "tokio", @@ -3747,16 +3750,22 @@ dependencies = [ "http-body 1.1.0", "http-body-util", "hyper 1.12.0", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.46", + "rustls-native-certs", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls 0.26.6", "tower", "tower-http", "tower-service", @@ -3896,6 +3905,7 @@ checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.103.16", "subtle", diff --git a/Cargo.toml b/Cargo.toml index e4079c2..1ad3959 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ async-trait = "0.1" aws-config = "1.8.11" aws-credential-types = "1.2.10" aws-sdk-s3 = "1.115.0" +aws-sigv4 = "1.6.3" +aws-smithy-runtime-api = "1.19.1" bytes = "1.11.0" clap = { version = "4.5.53", features = ["derive"] } futures = "0.3.31" @@ -24,6 +26,7 @@ iceberg-catalog-glue = "0.11.0" iceberg-catalog-rest = "0.11.0" iceberg-storage-opendal = "0.11.0" reqsign-core = "3.3.0" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" tokio = { version = "1.48.0", features = ["full"] } diff --git a/README.md b/README.md index 672e1be..a4fcd87 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,14 @@ bergr rest http://localhost:8181 namespace my_namespace info bergr rest http://localhost:8181 namespace my_namespace tables ``` +AWS Glue's Iceberg REST endpoint is detected from the URI, and requests to it are signed with SigV4 using your AWS credentials. Pass the catalog ID as the warehouse: the AWS account ID, for the default catalog. + +```bash +bergr rest https://glue.us-west-2.amazonaws.com/iceberg --warehouse 123456789012 namespaces +``` + +Glue's REST endpoint needs `glue:GetCatalog` and `lakeformation:GetDataAccess` permissions, in addition to those the `glue` command needs. + ### Direct access via metadata file location ```bash diff --git a/src/aws.rs b/src/aws.rs index 8d11440..3e2659a 100644 --- a/src/aws.rs +++ b/src/aws.rs @@ -1,6 +1,8 @@ //! AWS integration utilities for credential loading -use anyhow::Result; +use crate::rest::rest_catalog_props; +use crate::sigv4::{SigV4AuthManager, SigningScope}; +use anyhow::{Context as _, Result}; use aws_config::BehaviorVersion; use aws_config::meta::credentials::CredentialsProviderChain; use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider}; @@ -10,6 +12,7 @@ use iceberg_catalog_glue::{ AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, }; +use iceberg_catalog_rest::{RestCatalog, RestCatalogBuilder}; use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; @@ -140,6 +143,29 @@ pub async fn glue_catalog(aws_config: &aws_config::SdkConfig) -> Result, +) -> Result { + let credentials = aws_config + .credentials_provider() + .context("AWS credentials not configured")?; + + let mut props = rest_catalog_props(uri, warehouse); + props.insert(S3_REGION.to_string(), scope.region.clone()); + + let catalog = RestCatalogBuilder::default() + .with_auth_manager(SigV4AuthManager::new(scope, credentials)) + .with_storage_factory(s3_storage_factory(aws_config)) + .load("rest", props) + .await?; + + Ok(catalog) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 6ad5776..4a1e41d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,5 +4,6 @@ pub mod cli; pub mod error; pub mod file_existence; pub mod rest; +pub mod sigv4; pub mod table_commands; pub mod terminal_output; diff --git a/src/main.rs b/src/main.rs index fef99e7..30d9876 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,11 @@ use anyhow::Result; -use bergr::aws::{get_aws_config, glue_catalog, s3_file_io}; +use aws_config::Region; +use bergr::aws::{aws_rest_catalog, get_aws_config, glue_catalog, s3_file_io}; use bergr::catalog_commands::handle_catalog_command; use bergr::cli::{Cli, Commands}; use bergr::error::ExpectedError; use bergr::rest::rest_catalog; +use bergr::sigv4::SigningScope; use bergr::table_commands::{handle_table_command, load_table}; use bergr::terminal_output::TerminalOutput; use clap::Parser; @@ -77,9 +79,24 @@ async fn run(command: Commands) -> Result<()> { warehouse, command, } => { - let catalog = rest_catalog(&uri, warehouse.as_deref()).await?; let mut output = TerminalOutput::new(); - handle_catalog_command(&catalog, command, &mut output, None).await?; + match SigningScope::from_uri(&uri) { + Some(scope) => { + let aws_config = get_aws_config() + .await + .to_builder() + .region(Region::new(scope.region.clone())) + .build(); + let catalog = + aws_rest_catalog(&aws_config, scope, &uri, warehouse.as_deref()).await?; + let s3 = aws_sdk_s3::Client::new(&aws_config); + handle_catalog_command(&catalog, command, &mut output, Some(&s3)).await?; + } + None => { + let catalog = rest_catalog(&uri, warehouse.as_deref()).await?; + handle_catalog_command(&catalog, command, &mut output, None).await?; + } + } } } diff --git a/src/rest.rs b/src/rest.rs index 8590503..bd16587 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -7,17 +7,8 @@ use iceberg_catalog_rest::{ }; use std::collections::HashMap; -/// Create a REST catalog with the given URI and optional warehouse location -/// -/// # Arguments -/// -/// * `uri` - The REST catalog endpoint URL (e.g., "http://localhost:8181") -/// * `warehouse` - Optional warehouse location (e.g., "s3://my-bucket/warehouse") -/// -/// # Returns -/// -/// A configured `RestCatalog` instance -pub async fn rest_catalog(uri: &str, warehouse: Option<&str>) -> Result { +/// Catalog properties for a REST catalog with the given URI and optional warehouse +pub fn rest_catalog_props(uri: &str, warehouse: Option<&str>) -> HashMap { let mut props = HashMap::new(); // Required: REST catalog URI @@ -31,7 +22,23 @@ pub async fn rest_catalog(uri: &str, warehouse: Option<&str>) -> Result) -> Result { + let catalog = RestCatalogBuilder::default() + .load("rest", rest_catalog_props(uri, warehouse)) + .await?; Ok(catalog) } diff --git a/src/sigv4.rs b/src/sigv4.rs new file mode 100644 index 0000000..4b931fa --- /dev/null +++ b/src/sigv4.rs @@ -0,0 +1,247 @@ +//! AWS SigV4 request signing for REST catalogs hosted by AWS Glue +//! +//! A stopgap until iceberg-rust ships its own SigV4 support for REST catalogs. This is a +//! minimal signer: unlike upstream's, it doesn't reproduce Iceberg Java's signing quirks, +//! and has only been tested against Glue. +//! +//! TODO: replace with iceberg-catalog-rest's `SigV4AuthManager` once it's released +//! (https://github.com/apache/iceberg-rust/pull/3092). + +use async_trait::async_trait; +use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider}; +use aws_sigv4::http_request::{SignableBody, SignableRequest, SigningSettings, sign}; +use aws_sigv4::sign::v4; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_catalog_rest::{AuthManager, AuthSession, HttpClient, HttpRequest}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::SystemTime; +use url::Url; + +/// Where (and as which service) requests to an AWS-hosted REST catalog are signed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SigningScope { + pub service: String, + pub region: String, +} + +impl SigningScope { + /// Derive the scope from an AWS Glue endpoint URI, e.g. + /// `https://glue.us-west-2.amazonaws.com/iceberg` -> (`glue`, `us-west-2`). + /// + /// Returns `None` for URIs that aren't AWS Glue Iceberg REST endpoints. + pub fn from_uri(uri: &str) -> Option { + let url = Url::parse(uri).ok()?; + let host = url.host_str()?; + let labels: Vec<&str> = host.split('.').collect(); + match labels.as_slice() { + ["glue", region, "amazonaws", "com"] => Some(Self { + service: "glue".to_string(), + region: region.to_string(), + }), + _ => None, + } + } +} + +/// An [`AuthManager`] that signs every request with AWS SigV4. +#[derive(Debug, Clone)] +pub struct SigV4AuthManager { + session: Arc, +} + +impl SigV4AuthManager { + pub fn new(scope: SigningScope, credentials: SharedCredentialsProvider) -> Self { + Self { + session: Arc::new(SigV4Session { scope, credentials }), + } + } +} + +#[async_trait] +impl AuthManager for SigV4AuthManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(self.session.as_ref().clone())) + } + + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(self.session.clone()) + } +} + +#[derive(Debug, Clone)] +struct SigV4Session { + scope: SigningScope, + credentials: SharedCredentialsProvider, +} + +fn unexpected(message: &str, source: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::new(ErrorKind::Unexpected, message.to_string()).with_source(source) +} + +impl SigV4Session { + fn sign_with( + &self, + request: &mut HttpRequest, + identity: &aws_smithy_runtime_api::client::identity::Identity, + time: SystemTime, + ) -> Result<()> { + let body = request.body().as_bytes().ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + "cannot SigV4-sign a streaming request body", + ) + })?; + + let headers = request + .headers() + .iter() + .map(|(name, value)| { + value + .to_str() + .map(|v| (name.as_str(), v)) + .map_err(|e| unexpected("non-UTF-8 header value", e)) + }) + .collect::>>()?; + + let params = v4::SigningParams::builder() + .identity(identity) + .region(&self.scope.region) + .name(&self.scope.service) + .time(time) + .settings(SigningSettings::default()) + .build() + .map_err(|e| unexpected("invalid SigV4 signing params", e))? + .into(); + + let signable = SignableRequest::new( + request.method().as_str(), + request.url_str(), + headers.into_iter(), + SignableBody::Bytes(body), + ) + .map_err(|e| unexpected("request is not signable", e))?; + + let (instructions, _signature) = sign(signable, ¶ms) + .map_err(|e| unexpected("SigV4 signing failed", e))? + .into_parts(); + let (signed_headers, _query_params) = instructions.into_parts(); + + signed_headers.into_iter().try_for_each(|header| { + let name: http::HeaderName = header + .name() + .parse() + .map_err(|e| unexpected("invalid signed header name", e))?; + let value: http::HeaderValue = header + .value() + .parse() + .map_err(|e| unexpected("invalid signed header value", e))?; + request.headers_mut().insert(name, value); + Ok(()) + }) + } +} + +#[async_trait] +impl AuthSession for SigV4Session { + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + let credentials = self + .credentials + .provide_credentials() + .await + .map_err(|e| unexpected("failed to load AWS credentials", e))?; + self.sign_with(request, &credentials.into(), SystemTime::now()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use aws_credential_types::Credentials; + use std::time::{Duration, UNIX_EPOCH}; + + #[test] + fn scope_from_glue_uri() { + assert_eq!( + SigningScope::from_uri("https://glue.us-west-2.amazonaws.com/iceberg"), + Some(SigningScope { + service: "glue".into(), + region: "us-west-2".into() + }) + ); + } + + #[test] + fn scope_from_non_aws_uri() { + assert_eq!(SigningScope::from_uri("http://localhost:8181"), None); + assert_eq!( + SigningScope::from_uri("https://s3.us-west-2.amazonaws.com"), + None + ); + } + + fn session() -> SigV4Session { + SigV4Session { + scope: SigningScope { + service: "glue".into(), + region: "us-west-2".into(), + }, + credentials: SharedCredentialsProvider::new(Credentials::new( + "AKIDEXAMPLE", + "secret", + Some("token".into()), + None, + "test", + )), + } + } + + fn request(method: reqwest::Method, url: &str, body: Option<&'static str>) -> HttpRequest { + let builder = reqwest::Client::new().request(method, url); + let builder = match body { + Some(b) => builder.body(b), + None => builder, + }; + HttpRequest::new(builder.build().unwrap()) + } + + #[tokio::test] + async fn adds_sigv4_headers() { + let mut req = request( + reqwest::Method::GET, + "https://glue.us-west-2.amazonaws.com/iceberg/v1/config?warehouse=123456789012", + None, + ); + session().authenticate(&mut req).await.unwrap(); + + let headers = req.headers(); + let auth = headers["authorization"].to_str().unwrap(); + assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/")); + assert!(auth.contains("/us-west-2/glue/aws4_request")); + assert!(headers.contains_key("x-amz-date")); + assert_eq!(headers["x-amz-security-token"], "token"); + } + + #[test] + fn signature_covers_body() { + let identity = Credentials::new("AKIDEXAMPLE", "secret", None, None, "test").into(); + let time = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let url = "https://glue.us-west-2.amazonaws.com/iceberg/v1/catalogs/1/namespaces"; + let sign = |body| { + let mut req = request(reqwest::Method::POST, url, Some(body)); + session().sign_with(&mut req, &identity, time).unwrap(); + req.headers()["authorization"].clone() + }; + + assert_eq!(sign("{}"), sign("{}")); + assert_ne!(sign("{}"), sign("{\"x\":1}")); + } +}