diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index a0c6fd03ce..d4232d8ff2 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -1,5 +1,6 @@ pub mod iceberg_catalog_rest pub use iceberg_catalog_rest::Credentials +pub use iceberg_catalog_rest::SharedCredentialsProvider pub enum iceberg_catalog_rest::HttpRequestBody<'a> pub iceberg_catalog_rest::HttpRequestBody::Buffered(&'a [u8]) pub iceberg_catalog_rest::HttpRequestBody::Empty @@ -235,6 +236,7 @@ impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::signs_requests(&self) -> bool pub struct iceberg_catalog_rest::OAuth2Manager impl iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::new(token_endpoint: impl core::convert::Into) -> Self @@ -248,6 +250,7 @@ impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::signs_requests(&self) -> bool pub struct iceberg_catalog_rest::RegisterTableRequest pub iceberg_catalog_rest::RegisterTableRequest::metadata_location: alloc::string::String pub iceberg_catalog_rest::RegisterTableRequest::name: alloc::string::String @@ -346,6 +349,17 @@ impl core::default::Default for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::default() -> Self impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::SigV4AuthManager +impl iceberg_catalog_rest::SigV4AuthManager +pub fn iceberg_catalog_rest::SigV4AuthManager::from_properties(delegate: alloc::sync::Arc, props: &std::collections::hash::map::HashMap) -> iceberg::error::Result +pub fn iceberg_catalog_rest::SigV4AuthManager::new(delegate: alloc::sync::Arc, signer: iceberg_catalog_rest::SigV4Signer, credentials: aws_credential_types::provider::credentials::SharedCredentialsProvider) -> Self +impl core::fmt::Debug for iceberg_catalog_rest::SigV4AuthManager +pub fn iceberg_catalog_rest::SigV4AuthManager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::SigV4AuthManager +pub fn iceberg_catalog_rest::SigV4AuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::signs_requests(&self) -> bool pub struct iceberg_catalog_rest::SigV4Signer impl iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> @@ -403,24 +417,41 @@ impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::UpdateNames pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub const iceberg_catalog_rest::AUTH_TYPE_NONE: &str pub const iceberg_catalog_rest::AUTH_TYPE_OAUTH2: &str +pub const iceberg_catalog_rest::AUTH_TYPE_SIGV4: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_ACCESS_KEY_ID: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_AUTH_TYPE: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_PAGE_SIZE: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SECRET_ACCESS_KEY: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SESSION_TOKEN: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_NAME: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGNING_REGION: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_SIGV4_ENABLED: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_TABLE_CACHE_MAX_ENTRIES: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_URI: &str pub const iceberg_catalog_rest::REST_CATALOG_PROP_WAREHOUSE: &str +pub const iceberg_catalog_rest::SIGNING_NAME_DEFAULT: &str pub trait iceberg_catalog_rest::AuthManager: core::fmt::Debug + core::marker::Send + core::marker::Sync pub fn iceberg_catalog_rest::AuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::AuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::AuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::AuthManager::signs_requests(&self) -> bool impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::signs_requests(&self) -> bool impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::signs_requests(&self) -> bool +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::SigV4AuthManager +pub fn iceberg_catalog_rest::SigV4AuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::SigV4AuthManager::signs_requests(&self) -> bool pub trait iceberg_catalog_rest::AuthSession: core::fmt::Debug + core::marker::Send + core::marker::Sync pub fn iceberg_catalog_rest::AuthSession::authenticate<'life0, 'life1, 'async_trait>(&'life0 self, request: &'life1 mut iceberg_catalog_rest::HttpRequest) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index a3aab4df90..8e451e1d32 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -29,10 +29,16 @@ use std::sync::Arc; use async_trait::async_trait; #[cfg(feature = "sigv4")] pub use aws_credential_types::Credentials; +#[cfg(feature = "sigv4")] +pub use aws_credential_types::provider::SharedCredentialsProvider; use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; #[cfg(feature = "sigv4")] -pub use sigv4::{PayloadHashMode, SigV4Signer}; +pub use sigv4::{ + PayloadHashMode, REST_CATALOG_PROP_ACCESS_KEY_ID, REST_CATALOG_PROP_SECRET_ACCESS_KEY, + REST_CATALOG_PROP_SESSION_TOKEN, REST_CATALOG_PROP_SIGNING_NAME, + REST_CATALOG_PROP_SIGNING_REGION, SIGNING_NAME_DEFAULT, SigV4AuthManager, SigV4Signer, +}; use crate::client::HttpClient; use crate::request::HttpRequest; @@ -41,6 +47,8 @@ use crate::request::HttpRequest; pub const AUTH_TYPE_NONE: &str = "none"; /// `rest.auth.type` value selecting OAuth2 token authentication. pub const AUTH_TYPE_OAUTH2: &str = "oauth2"; +/// `rest.auth.type` value selecting AWS SigV4 request signing. +pub const AUTH_TYPE_SIGV4: &str = "sigv4"; /// Creates the [`AuthSession`]s used to authenticate REST catalog requests. /// @@ -108,12 +116,21 @@ pub trait AuthManager: Debug + Send + Sync { let _ = context; Ok(catalog_session) } + + /// Whether this manager's sessions sign requests. The catalog's default + /// client then does not follow redirects, which would replay a signature + /// at another URL. + fn signs_requests(&self) -> bool { + false + } } /// Authenticates outgoing REST catalog requests. #[async_trait] pub trait AuthSession: Debug + Send + Sync { /// Applies authentication to the request (adds headers, signs, ...). + /// + /// The request already carries the configured `header.*` values. async fn authenticate(&self, request: &mut HttpRequest) -> Result<()>; } diff --git a/crates/catalog/rest/src/auth/oauth2.rs b/crates/catalog/rest/src/auth/oauth2.rs index aad91996a8..4411eb7e90 100644 --- a/crates/catalog/rest/src/auth/oauth2.rs +++ b/crates/catalog/rest/src/auth/oauth2.rs @@ -219,7 +219,10 @@ fn attach_bearer(req: &mut HttpRequest, token: &SensitiveString) -> Result<()> { .with_source(e) })?; value.set_sensitive(true); - req.headers_mut().insert(http::header::AUTHORIZATION, value); + // As in Java's `OAuth2Util.AuthSession`, a configured header wins. + if !req.headers().contains_key(http::header::AUTHORIZATION) { + req.headers_mut().insert(http::header::AUTHORIZATION, value); + } Ok(()) } diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index b7b4ae5bf1..203599f615 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -17,12 +17,19 @@ //! AWS SigV4 request signing for the REST catalog. +use std::collections::HashMap; +use std::sync::{Arc, OnceLock}; + +use async_trait::async_trait; +use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider}; use chrono::{DateTime, Utc}; -use iceberg::{Error, ErrorKind, Result}; +use iceberg::sensitive::SensitiveString; +use iceberg::{Error, ErrorKind, Result, SessionContext}; use sha2::{Digest, Sha256}; use typed_builder::TypedBuilder; -use super::Credentials; +use super::{AuthManager, AuthSession, Credentials}; +use crate::client::HttpClient; /// How the payload hash is encoded in the `x-amz-content-sha256` header. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -364,8 +371,359 @@ fn relocate_conflicting( } } +/// SigV4 signing region. +pub const REST_CATALOG_PROP_SIGNING_REGION: &str = "rest.signing-region"; +/// SigV4 signing name; defaults to [`SIGNING_NAME_DEFAULT`]. +pub const REST_CATALOG_PROP_SIGNING_NAME: &str = "rest.signing-name"; +/// Static SigV4 access key id. Unlike Java, there is no fallback to the AWS +/// default provider chain. +pub const REST_CATALOG_PROP_ACCESS_KEY_ID: &str = "rest.access-key-id"; +/// Static SigV4 secret access key (see [`REST_CATALOG_PROP_ACCESS_KEY_ID`]). +pub const REST_CATALOG_PROP_SECRET_ACCESS_KEY: &str = "rest.secret-access-key"; +/// Static SigV4 session token (see [`REST_CATALOG_PROP_ACCESS_KEY_ID`]). +pub const REST_CATALOG_PROP_SESSION_TOKEN: &str = "rest.session-token"; + +/// Java's `REST_SIGNING_NAME_DEFAULT`: API Gateway and Lambda. +pub const SIGNING_NAME_DEFAULT: &str = "execute-api"; + +const CREDENTIAL_PROPERTIES: [&str; 3] = [ + REST_CATALOG_PROP_ACCESS_KEY_ID, + REST_CATALOG_PROP_SECRET_ACCESS_KEY, + REST_CATALOG_PROP_SESSION_TOKEN, +]; +const SIGNING_PROPERTIES: [&str; 5] = [ + REST_CATALOG_PROP_SIGNING_REGION, + REST_CATALOG_PROP_SIGNING_NAME, + REST_CATALOG_PROP_ACCESS_KEY_ID, + REST_CATALOG_PROP_SECRET_ACCESS_KEY, + REST_CATALOG_PROP_SESSION_TOKEN, +]; + +/// `base` with `overrides` on top. Overriding any credential replaces the +/// base's credentials whole: fields from two sources never form a valid +/// credential. +fn overlay_signing( + base: &HashMap, + overrides: HashMap, +) -> HashMap { + let replaces_credentials = CREDENTIAL_PROPERTIES + .iter() + .any(|key| overrides.contains_key(*key)); + let mut props: HashMap<_, _> = base + .iter() + .filter(|(k, _)| !(replaces_credentials && CREDENTIAL_PROPERTIES.contains(&k.as_str()))) + .map(|(k, v)| (k.clone(), v.expose().to_string())) + .collect(); + props.extend(overrides); + props +} + +/// The non-blank [`SIGNING_PROPERTIES`] of `props`. +fn signing_overrides(props: &HashMap) -> HashMap { + SIGNING_PROPERTIES + .iter() + .filter_map(|key| Some((key.to_string(), non_blank(props, key)?))) + .collect() +} + +fn sensitive(props: HashMap) -> HashMap { + props.into_iter().map(|(k, v)| (k, v.into())).collect() +} + +/// Trims a property, treating blank as absent. +fn non_blank(props: &HashMap, key: &str) -> Option { + props + .get(key) + .map(|v| v.trim()) + .filter(|v| !v.is_empty()) + .map(str::to_string) +} + +/// Builds a signer and static credentials from Java's `AwsProperties` names. +fn from_props(props: &HashMap) -> Result<(SigV4Signer, SharedCredentialsProvider)> { + let region = non_blank(props, REST_CATALOG_PROP_SIGNING_REGION).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("'{REST_CATALOG_PROP_SIGNING_REGION}' is required for SigV4 signing"), + ) + })?; + let name = non_blank(props, REST_CATALOG_PROP_SIGNING_NAME) + .unwrap_or_else(|| SIGNING_NAME_DEFAULT.into()); + + let credentials = credentials_from_props(props)?; + Ok(( + SigV4Signer::builder() + .region(region) + .service(name) + .mode(PayloadHashMode::IcebergRest) + .build(), + credentials, + )) +} + +/// Like Java, branches on the access key id alone; a lone secret is an error. +fn credentials_from_props(props: &HashMap) -> Result { + let Some(access_key_id) = non_blank(props, REST_CATALOG_PROP_ACCESS_KEY_ID) else { + if non_blank(props, REST_CATALOG_PROP_SECRET_ACCESS_KEY).is_some() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "'{REST_CATALOG_PROP_SECRET_ACCESS_KEY}' is set without '{REST_CATALOG_PROP_ACCESS_KEY_ID}'" + ), + )); + } + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "no SigV4 credentials: set '{REST_CATALOG_PROP_ACCESS_KEY_ID}' and \ + '{REST_CATALOG_PROP_SECRET_ACCESS_KEY}', or build the catalog with a \ + `SigV4AuthManager` carrying your own credentials provider" + ), + )); + }; + let secret_access_key = non_blank(props, REST_CATALOG_PROP_SECRET_ACCESS_KEY).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("'{REST_CATALOG_PROP_ACCESS_KEY_ID}' is set without '{REST_CATALOG_PROP_SECRET_ACCESS_KEY}'"), + ) + })?; + Ok(SharedCredentialsProvider::new(Credentials::new( + access_key_id, + secret_access_key, + non_blank(props, REST_CATALOG_PROP_SESSION_TOKEN), + None, + "iceberg-rest-properties", + ))) +} + +/// [`AuthManager`] that SigV4-signs every request on top of a delegate, as +/// Java's `RESTSigV4AuthManager` does. The delegate's `Authorization` moves to +/// `Original-Authorization` and is signed over. +/// +/// Unlike Java, properties configure static credentials only; pass a provider +/// to [`Self::new`] for anything else. Context values override catalog ones, +/// blank context values are ignored, and context credentials replace the +/// catalog's as a set. +/// +/// Configure signing-relevant headers as `header.*` properties: an injected +/// client's default headers are added after signing. +#[derive(Debug)] +pub struct SigV4AuthManager { + delegate: Arc, + signing: Signing, + catalog: OnceLock, +} + +#[derive(Debug)] +enum Signing { + Injected { + signer: SigV4Signer, + credentials: SharedCredentialsProvider, + }, + /// The constructor's signing properties, which each session method's + /// properties override, as Java rebuilds `AwsProperties` in each. + FromProperties(HashMap), +} + +#[derive(Debug)] +struct SigV4CatalogState { + session: Arc, + /// The signing properties the catalog session was built from. + signing_properties: HashMap, +} + +impl SigV4AuthManager { + /// Signs with `signer` and `credentials`, whatever the properties say. + pub fn new( + delegate: Arc, + signer: SigV4Signer, + credentials: SharedCredentialsProvider, + ) -> Self { + Self { + delegate, + signing: Signing::Injected { + signer, + credentials, + }, + catalog: OnceLock::new(), + } + } + + /// Signs as `props` describe. The properties each session method is given + /// override them, so `/v1/config` overrides apply. + /// + /// # Errors + /// + /// Returns [`ErrorKind::DataInvalid`] if `props` lack a signing region or + /// a complete static credential pair. + pub fn from_properties( + delegate: Arc, + props: &HashMap, + ) -> Result { + let base = signing_overrides(props); + from_props(&base)?; + Ok(Self { + delegate, + signing: Signing::FromProperties(sensitive(base)), + catalog: OnceLock::new(), + }) + } + + /// The signer and credentials for `props`, and the signing properties + /// they came from (empty for an injected signer). + fn signing_for( + &self, + props: &HashMap, + ) -> Result<( + SigV4Signer, + SharedCredentialsProvider, + HashMap, + )> { + match &self.signing { + Signing::Injected { + signer, + credentials, + } => Ok((signer.clone(), credentials.clone(), HashMap::new())), + Signing::FromProperties(base) => { + let props = overlay_signing(base, signing_overrides(props)); + let (signer, credentials) = from_props(&props)?; + Ok((signer, credentials, sensitive(props))) + } + } + } +} + +/// The non-blank [`SIGNING_PROPERTIES`] a context sets, its credentials over +/// its properties. +fn context_signing_overrides(context: &SessionContext) -> HashMap { + SIGNING_PROPERTIES + .iter() + .filter_map(|key| { + let credential = context.credentials().get(*key).map(|v| v.expose()); + let property = context.properties().get(*key).map(String::as_str); + let value = [credential, property] + .into_iter() + .flatten() + .find(|v| !v.trim().is_empty())?; + Some((key.to_string(), value.to_string())) + }) + .collect() +} + +#[async_trait] +impl AuthManager for SigV4AuthManager { + async fn init_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result> { + let (signer, credentials, _) = self.signing_for(props)?; + Ok(Box::new(SigV4Session { + delegate: Arc::from(self.delegate.init_session(client, props).await?), + signer, + credentials, + })) + } + + async fn catalog_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result> { + let (signer, credentials, signing_properties) = self.signing_for(props)?; + let session = Arc::new(SigV4Session { + delegate: self.delegate.catalog_session(client, props).await?, + signer, + credentials, + }); + self.catalog + .set(SigV4CatalogState { + session: session.clone(), + signing_properties, + }) + .map_err(|_| { + Error::new( + ErrorKind::Unexpected, + "SigV4 catalog session already initialized", + ) + })?; + Ok(session) + } + + async fn contextual_session( + &self, + context: &SessionContext, + catalog_session: Arc, + ) -> Result> { + let catalog = self.catalog.get().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "SigV4 catalog session is not initialized", + ) + })?; + let parent: Arc = catalog.session.clone(); + if !Arc::ptr_eq(&parent, &catalog_session) { + return Err(Error::new( + ErrorKind::DataInvalid, + "unexpected SigV4 catalog session", + )); + } + + let delegate = self + .delegate + .contextual_session(context, catalog.session.delegate.clone()) + .await?; + let overrides = match self.signing { + Signing::FromProperties(_) => context_signing_overrides(context), + Signing::Injected { .. } => HashMap::new(), + }; + if overrides.is_empty() && Arc::ptr_eq(&delegate, &catalog.session.delegate) { + return Ok(catalog_session); + } + let (signer, credentials) = if !overrides.is_empty() { + from_props(&overlay_signing(&catalog.signing_properties, overrides))? + } else { + ( + catalog.session.signer.clone(), + catalog.session.credentials.clone(), + ) + }; + Ok(Arc::new(SigV4Session { + delegate, + signer, + credentials, + })) + } + + fn signs_requests(&self) -> bool { + true + } +} + +/// [`AuthSession`] applying the delegate's auth, then SigV4-signing. +#[derive(Debug)] +struct SigV4Session { + delegate: Arc, + signer: SigV4Signer, + credentials: SharedCredentialsProvider, +} + +#[async_trait] +impl AuthSession for SigV4Session { + async fn authenticate(&self, request: &mut crate::HttpRequest) -> Result<()> { + self.delegate.authenticate(request).await?; + // Per request, as in Java; caching is the provider's job. + let credentials = self.credentials.provide_credentials().await.map_err(|e| { + Error::new(ErrorKind::Unexpected, "failed to resolve AWS credentials").with_source(e) + })?; + self.signer.sign(request, &credentials) + } +} + #[cfg(test)] mod tests { + use std::sync::Mutex; + use chrono::TimeZone; use super::*; @@ -514,7 +872,7 @@ mod tests { /// Records every event field. #[derive(Clone, Default)] - struct CapturedLog(std::sync::Arc>); + struct CapturedLog(Arc>); impl tracing::field::Visit for CapturedLog { fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { @@ -1222,4 +1580,750 @@ mod tests { "{date}" ); } + + fn test_credentials_provider() -> SharedCredentialsProvider { + SharedCredentialsProvider::new(test_credentials()) + } + + fn test_session() -> SigV4Session { + SigV4Session { + delegate: Arc::new(crate::auth::NoopSession), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: test_credentials_provider(), + } + } + + fn request_with(headers: &[(&'static str, &str)]) -> HttpRequest { + let mut builder = reqwest::Client::new().get("https://rest.example.com/v1/config"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + HttpRequest::new(builder.build().unwrap()) + } + + #[tokio::test] + async fn relocation_keeps_an_existing_original_authorization() { + let mut request = request_with(&[ + ("original-authorization", "credential-A"), + ("authorization", "credential-B"), + ]); + + test_session().authenticate(&mut request).await.unwrap(); + + let relocated: Vec<_> = request + .headers() + .get_all("original-authorization") + .iter() + .map(|v| v.to_str().unwrap().to_string()) + .collect(); + assert_eq!(relocated, ["credential-A", "credential-B"]); + assert!( + request + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap() + .starts_with("AWS4-HMAC-SHA256 ") + ); + } + + #[tokio::test] + async fn credentials_are_resolved_once_per_request() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let calls = Arc::new(AtomicUsize::new(0)); + let counted = { + let calls = calls.clone(); + aws_credential_types::credential_fn::provide_credentials_fn(move || { + let calls = calls.clone(); + async move { + let n = calls.fetch_add(1, Ordering::SeqCst); + Ok(Credentials::new( + format!("AKID{n}"), + "secret", + None::, + None, + "test", + )) + } + }) + }; + let session = SigV4Session { + delegate: Arc::new(crate::auth::NoopSession), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: SharedCredentialsProvider::new(counted), + }; + + let mut first = request_with(&[]); + session.authenticate(&mut first).await.unwrap(); + let mut second = request_with(&[]); + session.authenticate(&mut second).await.unwrap(); + + assert_eq!(calls.load(Ordering::SeqCst), 2); + let auth = |r: &HttpRequest| { + r.headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap() + .to_string() + }; + assert!(auth(&first).contains("AKID0/"), "{}", auth(&first)); + assert!(auth(&second).contains("AKID1/"), "{}", auth(&second)); + } + + #[tokio::test] + async fn the_delegate_authenticates_before_signing() { + #[derive(Debug)] + struct Bearer; + #[async_trait] + impl AuthSession for Bearer { + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + request + .headers_mut() + .insert("authorization", "Bearer delegate".parse().unwrap()); + Ok(()) + } + } + + let session = SigV4Session { + delegate: Arc::new(Bearer), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: test_credentials_provider(), + }; + let mut request = request_with(&[]); + session.authenticate(&mut request).await.unwrap(); + + assert_eq!( + request.headers().get("original-authorization").unwrap(), + "Bearer delegate" + ); + let auth = request.headers().get("authorization").unwrap(); + assert!(auth.to_str().unwrap().contains("original-authorization")); + } + + fn props(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[tokio::test] + async fn static_credentials_come_from_properties() { + let (signer, credentials) = from_props(&props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "cn-hangzhou"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "AKID"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "secret"), + (REST_CATALOG_PROP_SESSION_TOKEN, "token"), + ])) + .unwrap(); + + assert_eq!(signer.region, "cn-hangzhou"); + assert_eq!(signer.service, SIGNING_NAME_DEFAULT); + assert_eq!(signer.mode, PayloadHashMode::IcebergRest); + let resolved = credentials.provide_credentials().await.unwrap(); + assert_eq!(resolved.access_key_id(), "AKID"); + assert_eq!(resolved.session_token(), Some("token")); + } + + #[tokio::test] + async fn a_blank_property_counts_as_absent() { + let err = from_props(&props(&[(REST_CATALOG_PROP_SIGNING_REGION, " ")])).unwrap_err(); + assert!( + err.message().contains(REST_CATALOG_PROP_SIGNING_REGION), + "{err}" + ); + } + + #[tokio::test] + async fn half_a_credential_pair_is_rejected_either_way() { + for (present, missing) in [ + ( + REST_CATALOG_PROP_ACCESS_KEY_ID, + REST_CATALOG_PROP_SECRET_ACCESS_KEY, + ), + ( + REST_CATALOG_PROP_SECRET_ACCESS_KEY, + REST_CATALOG_PROP_ACCESS_KEY_ID, + ), + ] { + let err = from_props(&props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (present, "value"), + ])) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + let message = err.message(); + assert!(message.contains(present), "{err}"); + assert!(message.contains(missing), "{err}"); + assert!(!message.contains("credentials provider"), "{err}"); + } + } + + #[tokio::test] + async fn without_credentials_the_error_points_at_the_provider() { + let err = + from_props(&props(&[(REST_CATALOG_PROP_SIGNING_REGION, "us-east-1")])).unwrap_err(); + assert!(err.message().contains("credentials provider"), "{err}"); + } + + #[tokio::test] + async fn the_signing_name_property_overrides_the_default() { + let (signer, _) = from_props(&props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_SIGNING_NAME, "glue"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "AKID"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "secret"), + ])) + .unwrap(); + assert_eq!(signer.service, "glue"); + assert_ne!(signer.service, SIGNING_NAME_DEFAULT); + } + + #[tokio::test] + async fn a_padded_property_is_trimmed_not_just_accepted() { + let (signer, credentials) = from_props(&props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, " us-east-1 "), + (REST_CATALOG_PROP_ACCESS_KEY_ID, " AKID "), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "secret"), + ])) + .unwrap(); + assert_eq!(signer.region, "us-east-1"); + assert_eq!( + credentials + .provide_credentials() + .await + .unwrap() + .access_key_id(), + "AKID" + ); + } + + #[tokio::test] + async fn a_failing_delegate_stops_the_request_unsigned() { + #[derive(Debug)] + struct Failing; + #[async_trait] + impl AuthSession for Failing { + async fn authenticate(&self, _: &mut HttpRequest) -> Result<()> { + Err(Error::new(ErrorKind::Unexpected, "token refresh failed")) + } + } + + let session = SigV4Session { + delegate: Arc::new(Failing), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: test_credentials_provider(), + }; + let mut request = request_with(&[]); + let err = session.authenticate(&mut request).await.unwrap_err(); + + assert!(err.message().contains("token refresh failed"), "{err}"); + assert!(request.headers().get("authorization").is_none()); + } + + /// Records which delegate method ran and with which properties. + #[derive(Debug, Default)] + struct RecordingManager { + calls: Mutex)>>, + } + + #[async_trait] + impl AuthManager for RecordingManager { + async fn init_session( + &self, + _: &HttpClient, + props: &HashMap, + ) -> Result> { + self.calls.lock().unwrap().push(("init", props.clone())); + Ok(Box::new(crate::auth::NoopSession)) + } + + async fn catalog_session( + &self, + _: &HttpClient, + props: &HashMap, + ) -> Result> { + self.calls.lock().unwrap().push(("catalog", props.clone())); + Ok(Arc::new(crate::auth::NoopSession)) + } + } + + fn test_client() -> HttpClient { + HttpClient::new( + &crate::RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .build(), + ) + .unwrap() + } + + #[derive(Debug)] + struct ContextDelegate { + parent: Arc, + } + + #[derive(Debug)] + struct ContextBearer(String); + + #[async_trait] + impl AuthSession for ContextBearer { + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + request.headers_mut().insert( + "authorization", + format!("Bearer {}", self.0).parse().unwrap(), + ); + Ok(()) + } + } + + #[async_trait] + impl AuthManager for ContextDelegate { + async fn init_session( + &self, + _: &HttpClient, + _: &HashMap, + ) -> Result> { + Ok(Box::new(crate::auth::NoopSession)) + } + + async fn catalog_session( + &self, + _: &HttpClient, + _: &HashMap, + ) -> Result> { + Ok(self.parent.clone()) + } + + async fn contextual_session( + &self, + context: &SessionContext, + parent: Arc, + ) -> Result> { + assert!(Arc::ptr_eq(&parent, &self.parent)); + match context.identity() { + Some("fail") => Err(Error::new( + ErrorKind::DataInvalid, + "delegate context failed", + )), + Some(identity) => Ok(Arc::new(ContextBearer(identity.to_string()))), + None => Ok(parent), + } + } + } + + #[tokio::test] + async fn contextual_sigv4_sessions_use_the_delegate_and_context_credentials() { + let base = props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "CATALOG"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "catalog-secret"), + ]); + let manager = SigV4AuthManager::from_properties( + Arc::new(ContextDelegate { + parent: Arc::new(crate::auth::NoopSession), + }), + &base, + ) + .unwrap(); + let parent = manager + .catalog_session(&test_client(), &base) + .await + .unwrap(); + let context = SessionContext::builder() + .identity("alice".to_string()) + .properties(props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "eu-west-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "PROPERTY-KEY"), + ])) + .credentials( + props(&[ + (REST_CATALOG_PROP_ACCESS_KEY_ID, "CONTEXT-KEY"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "context-secret"), + (REST_CATALOG_PROP_SESSION_TOKEN, "context-token"), + ]) + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + ) + .build(); + let session = manager + .contextual_session(&context, parent.clone()) + .await + .unwrap(); + let mut request = request_with(&[]); + session.authenticate(&mut request).await.unwrap(); + let authorization = request + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!( + authorization.contains("Credential=CONTEXT-KEY/"), + "{authorization}" + ); + assert!( + authorization.contains("/eu-west-1/execute-api/"), + "{authorization}" + ); + assert_eq!( + request.headers().get("original-authorization").unwrap(), + "Bearer alice" + ); + assert_eq!( + request.headers().get("x-amz-security-token").unwrap(), + "context-token" + ); + + let date = request + .headers() + .get("x-amz-date") + .unwrap() + .to_str() + .unwrap(); + let now = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") + .unwrap() + .and_utc(); + let mut expected = request_with(&[("authorization", "Bearer alice")]); + SigV4Signer::builder() + .region("eu-west-1") + .service("execute-api") + .mode(PayloadHashMode::IcebergRest) + .build() + .sign_at( + &mut expected, + &Credentials::new( + "CONTEXT-KEY", + "context-secret", + Some("context-token".into()), + None, + "test", + ), + now, + ) + .unwrap(); + assert_eq!( + request.headers().get("authorization"), + expected.headers().get("authorization") + ); + + let unchanged = manager + .contextual_session(&SessionContext::empty(), parent.clone()) + .await + .unwrap(); + assert!(Arc::ptr_eq(&parent, &unchanged)); + assert!(scope_of(parent.as_ref()).await.starts_with("CATALOG/")); + + let failed = SessionContext::builder().identity("fail".into()).build(); + let err = manager + .contextual_session(&failed, parent) + .await + .unwrap_err(); + assert_eq!(err.message(), "delegate context failed"); + assert!(!format!("{manager:?}").contains("catalog-secret")); + } + + #[tokio::test] + async fn context_credentials_replace_the_catalog_credentials_as_a_set() { + let base = props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "CATALOG"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "catalog-secret"), + (REST_CATALOG_PROP_SESSION_TOKEN, "catalog-token"), + ]); + let manager = + SigV4AuthManager::from_properties(Arc::new(RecordingManager::default()), &base) + .unwrap(); + let parent = manager + .catalog_session(&test_client(), &base) + .await + .unwrap(); + let context = SessionContext::builder() + .properties(props(&[(REST_CATALOG_PROP_SIGNING_REGION, " ")])) + .credentials( + props(&[ + (REST_CATALOG_PROP_ACCESS_KEY_ID, "CONTEXT"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "context-secret"), + ]) + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + ) + .build(); + + let session = manager.contextual_session(&context, parent).await.unwrap(); + + let mut request = request_with(&[]); + session.authenticate(&mut request).await.unwrap(); + let mut expected = request_with(&[]); + let date = request + .headers() + .get("x-amz-date") + .unwrap() + .to_str() + .unwrap(); + let now = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") + .unwrap() + .and_utc(); + test_signer(PayloadHashMode::IcebergRest) + .sign_at( + &mut expected, + &Credentials::new("CONTEXT", "context-secret", None, None, "test"), + now, + ) + .unwrap(); + assert_eq!(header_list(&request), header_list(&expected)); + } + + #[tokio::test] + async fn context_session_token_alone_does_not_reuse_catalog_keys() { + let base = props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "CATALOG"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "catalog-secret"), + ]); + let manager = + SigV4AuthManager::from_properties(Arc::new(RecordingManager::default()), &base) + .unwrap(); + let parent = manager + .catalog_session(&test_client(), &base) + .await + .unwrap(); + let context = SessionContext::builder() + .credentials( + props(&[(REST_CATALOG_PROP_SESSION_TOKEN, "context-token")]) + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + ) + .build(); + + let err = manager + .contextual_session(&context, parent) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + assert!(err.message().contains("credentials provider"), "{err}"); + } + + #[tokio::test] + async fn the_manager_forwards_to_the_matching_delegate_method() { + let delegate = Arc::new(RecordingManager::default()); + let manager = SigV4AuthManager::new( + delegate.clone(), + test_signer(PayloadHashMode::IcebergRest), + test_credentials_provider(), + ); + let init_props = props(&[("a", "1")]); + let catalog_props = props(&[("b", "2")]); + + manager + .init_session(&test_client(), &init_props) + .await + .unwrap(); + manager + .catalog_session(&test_client(), &catalog_props) + .await + .unwrap(); + + let calls = delegate.calls.lock().unwrap().clone(); + assert_eq!(calls, vec![ + ("init", init_props), + ("catalog", catalog_props) + ]); + } + + /// The credential scope a session signs with. + async fn scope_of(session: &dyn AuthSession) -> String { + let mut request = request_with(&[]); + session.authenticate(&mut request).await.unwrap(); + let auth = request + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap() + .to_string(); + let scope = auth.split("Credential=").nth(1).unwrap(); + scope.split(',').next().unwrap().to_string() + } + + #[tokio::test] + async fn a_property_built_manager_rebuilds_from_the_merged_properties() { + let base = props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "AKID"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "secret"), + ]); + let manager = + SigV4AuthManager::from_properties(Arc::new(RecordingManager::default()), &base) + .unwrap(); + + let mut merged = base.clone(); + merged.insert(REST_CATALOG_PROP_SIGNING_REGION.into(), "eu-west-1".into()); + merged.insert(REST_CATALOG_PROP_SIGNING_NAME.into(), "glue".into()); + merged.insert(REST_CATALOG_PROP_ACCESS_KEY_ID.into(), "ROTATED".into()); + + for session in [ + scope_of( + manager + .catalog_session(&test_client(), &merged) + .await + .unwrap() + .as_ref(), + ) + .await, + scope_of( + manager + .init_session(&test_client(), &merged) + .await + .unwrap() + .as_ref(), + ) + .await, + ] { + assert!(session.contains("/eu-west-1/glue/"), "{session}"); + assert!(!session.contains("us-east-1"), "{session}"); + assert!(session.starts_with("ROTATED/"), "{session}"); + } + } + + #[tokio::test] + async fn constructor_properties_apply_when_session_properties_omit_them() { + let manager = SigV4AuthManager::from_properties( + Arc::new(RecordingManager::default()), + &props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "us-east-1"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "AKID"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "secret"), + (REST_CATALOG_PROP_SESSION_TOKEN, "constructor-token"), + ]), + ) + .unwrap(); + let catalog_props = props(&[ + ("uri", "http://localhost"), + (REST_CATALOG_PROP_SIGNING_NAME, "glue"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "ROTATED"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "rotated-secret"), + ]); + + let init = manager + .init_session(&test_client(), &props(&[("uri", "http://localhost")])) + .await + .unwrap(); + let catalog = manager + .catalog_session(&test_client(), &catalog_props) + .await + .unwrap(); + + // The scope's date comes from the live clock. + let key_region_service = |scope: String| { + let parts: Vec<_> = scope.split('/').collect(); + format!("{}/{}/{}", parts[0], parts[2], parts[3]) + }; + let mut request = request_with(&[]); + catalog.authenticate(&mut request).await.unwrap(); + assert_eq!( + ( + key_region_service(scope_of(init.as_ref()).await), + key_region_service(scope_of(catalog.as_ref()).await), + request.headers().get("x-amz-security-token"), + ), + ( + "AKID/us-east-1/execute-api".to_string(), + "ROTATED/us-east-1/glue".to_string(), + None + ) + ); + } + + #[tokio::test] + async fn an_injected_signer_survives_catalog_and_context_properties() { + let manager = SigV4AuthManager::new( + Arc::new(RecordingManager::default()), + SigV4Signer::builder() + .region("ap-south-1") + .service("custom") + .mode(PayloadHashMode::StandardAws) + .build(), + test_credentials_provider(), + ); + let overriding = props(&[ + (REST_CATALOG_PROP_SIGNING_REGION, "eu-west-1"), + (REST_CATALOG_PROP_SIGNING_NAME, "glue"), + (REST_CATALOG_PROP_ACCESS_KEY_ID, "OTHER"), + (REST_CATALOG_PROP_SECRET_ACCESS_KEY, "other"), + ]); + + let parent = manager + .catalog_session(&test_client(), &overriding) + .await + .unwrap(); + let context = SessionContext::builder() + .properties(overriding.clone()) + .credentials(overriding.into_iter().map(|(k, v)| (k, v.into())).collect()) + .build(); + let session = manager + .contextual_session(&context, parent.clone()) + .await + .unwrap(); + assert!(Arc::ptr_eq(&parent, &session)); + let scope = scope_of(session.as_ref()).await; + assert!(scope.starts_with("ak/"), "{scope}"); + assert!(scope.contains("/ap-south-1/custom/"), "{scope}"); + assert!(!scope.contains("eu-west-1"), "{scope}"); + } + + #[tokio::test] + async fn a_request_the_signer_rejects_fails_the_session() { + let session = SigV4Session { + delegate: Arc::new(crate::auth::NoopSession), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: test_credentials_provider(), + }; + let mut request = HttpRequest::new( + reqwest::Client::new() + .post("https://rest.example.com/v1/namespaces") + .body(reqwest::Body::wrap_stream(futures::stream::once(async { + Ok::<_, std::io::Error>(bytes::Bytes::from_static(b"chunk")) + }))) + .build() + .unwrap(), + ); + + let err = session.authenticate(&mut request).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + assert!(request.headers().get("authorization").is_none()); + } + + #[tokio::test] + async fn a_failing_credentials_provider_fails_the_session() { + #[derive(Debug)] + struct Failing; + impl ProvideCredentials for Failing { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where Self: 'a { + aws_credential_types::provider::future::ProvideCredentials::ready(Err( + aws_credential_types::provider::error::CredentialsError::not_loaded("no creds"), + )) + } + } + + let session = SigV4Session { + delegate: Arc::new(crate::auth::NoopSession), + signer: test_signer(PayloadHashMode::IcebergRest), + credentials: SharedCredentialsProvider::new(Failing), + }; + let mut request = request_with(&[]); + + let err = session.authenticate(&mut request).await.unwrap_err(); + assert!(err.message().contains("resolve AWS credentials"), "{err}"); + assert!(request.headers().get("authorization").is_none()); + } } diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index f8e7c4179b..571b012969 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -43,8 +43,11 @@ use reqwest::{Client, Method, StatusCode, Url}; use tokio::sync::OnceCell; use typed_builder::TypedBuilder; +#[cfg(feature = "sigv4")] +use crate::auth::SigV4AuthManager; use crate::auth::{ - AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AuthManager, AuthSession, NoopAuthManager, OAuth2Manager, + AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AUTH_TYPE_SIGV4, AuthManager, AuthSession, NoopAuthManager, + OAuth2Manager, }; use crate::client::{ HttpClient, deserialize_catalog_response, deserialize_unexpected_catalog_error, @@ -68,9 +71,9 @@ pub const REST_CATALOG_PROP_PAGE_SIZE: &str = "rest-page-size"; /// Disable header redaction in error logs and `Debug` output (defaults to /// false for security) pub const REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str = "disable-header-redaction"; -/// Authentication scheme: `none` or `oauth2`. When unset, `oauth2` is used -/// if a `token`, `credential` or `oauth2-server-uri` is configured, `none` -/// otherwise. +/// Authentication scheme: `none`, `oauth2` or `sigv4` (requires the `sigv4` +/// feature). When unset, `oauth2` is used if a `token`, `credential` or +/// `oauth2-server-uri` is configured, `none` otherwise. pub const REST_CATALOG_PROP_AUTH_TYPE: &str = "rest.auth.type"; /// Maximum number of loaded tables kept for freshness-aware loading (defaults /// to 100; 0 disables the cache). The cache is keyed by session id, so it only @@ -99,6 +102,11 @@ fn is_short_lived_credential(key: &str) -> bool { key == S3_SESSION_TOKEN || key == GCS_TOKEN || key.starts_with(ADLS_SAS_TOKEN) } +/// Deprecated switch enabling SigV4, taking precedence over `rest.auth.type`. +pub const REST_CATALOG_PROP_SIGV4_ENABLED: &str = "rest.sigv4-enabled"; +/// Auth scheme wrapped by SigV4: `oauth2` (the default) or `none`. +pub const REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE: &str = "rest.auth.sigv4.delegate-auth-type"; + const ICEBERG_REST_SPEC_VERSION: &str = "0.14.1"; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); const PATH_V1: &str = "v1"; @@ -156,6 +164,8 @@ impl RestCatalogBuilder { } /// Configures the catalog with a custom HTTP client. + /// + /// With SigV4 signing, the client must not follow redirects. pub fn with_client(mut self, client: Client) -> Self { self.inner = self.inner.with_client(client); self @@ -325,6 +335,19 @@ impl RestCatalogConfig { .unwrap_or_else(|| self.default_client.get_or_init(Client::default).clone()) } + /// Builds the shared default client without following redirects. An + /// injected client is left as is. + pub(crate) fn disable_client_redirects(&self) { + if self.client.is_none() { + self.default_client.get_or_init(|| { + Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("failed to build default HTTP client") + }); + } + } + /// Get the token from the config. /// /// The client can use this token to send requests. @@ -898,6 +921,17 @@ impl RestSessionCatalog { /// `credential` or `oauth2-server-uri` is configured (preserving /// pre-`rest.auth.type` setups), `none` when none is. fn auth_type(config: &RestCatalogConfig) -> String { + if config + .props + .get(REST_CATALOG_PROP_SIGV4_ENABLED) + .is_some_and(|value| value.eq_ignore_ascii_case("true")) + { + tracing::warn!( + "'{REST_CATALOG_PROP_SIGV4_ENABLED}' is deprecated; set \ + '{REST_CATALOG_PROP_AUTH_TYPE}={AUTH_TYPE_SIGV4}' instead" + ); + return AUTH_TYPE_SIGV4.to_string(); + } config .props .get(REST_CATALOG_PROP_AUTH_TYPE) @@ -935,6 +969,35 @@ impl RestSessionCatalog { match auth_type.as_str() { AUTH_TYPE_NONE => Ok(Arc::new(NoopAuthManager)), AUTH_TYPE_OAUTH2 => Ok(Arc::new(OAuth2Manager::from_config(config)?)), + #[cfg(feature = "sigv4")] + AUTH_TYPE_SIGV4 => { + let delegate_type = config + .props + .get(REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE) + .map(|value| value.to_ascii_lowercase()); + // OAuth2 even without a token: `/v1/config` may supply one. + let delegate: Arc = match delegate_type.as_deref() { + Some(AUTH_TYPE_NONE) => Arc::new(NoopAuthManager), + Some(AUTH_TYPE_OAUTH2) | None => Arc::new(OAuth2Manager::from_config(config)?), + Some(other) => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "unknown '{REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE}': {other}" + ), + )); + } + }; + Ok(Arc::new(SigV4AuthManager::from_properties( + delegate, + &config.props, + )?)) + } + #[cfg(not(feature = "sigv4"))] + AUTH_TYPE_SIGV4 => Err(Error::new( + ErrorKind::FeatureUnsupported, + "SigV4 authentication requires the `sigv4` feature", + )), other => Err(Error::new( ErrorKind::DataInvalid, format!( @@ -950,7 +1013,11 @@ impl RestSessionCatalog { async fn client(&self) -> Result<&RestCatalogClient> { self.client .get_or_try_init(|| async { - RestCatalogClient::init(&self.user_config, self.resolve_auth_manager()?).await + let auth_manager = self.resolve_auth_manager()?; + if auth_manager.signs_requests() { + self.user_config.disable_client_redirects(); + } + RestCatalogClient::init(&self.user_config, auth_manager).await }) .await } @@ -1735,6 +1802,8 @@ impl RestSessionCatalogBuilder { } /// Configures the catalog with a custom HTTP client. + /// + /// With SigV4 signing, the client must not follow redirects. pub fn with_client(mut self, client: Client) -> Self { self.config.client = Some(client); self @@ -1970,6 +2039,29 @@ mod tests { RestSessionCatalog::new(config, None, None, Runtime::current(), None) } + #[cfg(feature = "sigv4")] + fn test_sigv4_manager() -> SigV4AuthManager { + use aws_credential_types::provider::SharedCredentialsProvider; + + use crate::auth::{PayloadHashMode, SigV4AuthManager, SigV4Signer}; + + SigV4AuthManager::new( + Arc::new(NoopAuthManager), + SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(PayloadHashMode::IcebergRest) + .build(), + SharedCredentialsProvider::new(aws_credential_types::Credentials::new( + "ak", + "sk", + None::, + None, + "test", + )), + ) + } + fn test_catalog_with(config: RestCatalogConfig, auth_manager: M) -> RestSessionCatalog where M: AuthManager + 'static { RestSessionCatalog::new( @@ -3616,6 +3708,343 @@ mod tests { assert!(err.message().contains(REST_CATALOG_PROP_AUTH_TYPE)); } + #[cfg(feature = "sigv4")] + fn sigv4_props() -> HashMap { + HashMap::from([ + (REST_CATALOG_PROP_AUTH_TYPE.into(), "SiGv4".into()), + ( + crate::REST_CATALOG_PROP_SIGNING_REGION.into(), + "us-east-1".into(), + ), + ( + crate::REST_CATALOG_PROP_ACCESS_KEY_ID.into(), + "CATALOG".into(), + ), + ( + crate::REST_CATALOG_PROP_SECRET_ACCESS_KEY.into(), + "catalog-secret".into(), + ), + ]) + } + + #[cfg(feature = "sigv4")] + #[tokio::test] + async fn test_configured_sigv4_signs_handshake_catalog_and_context_requests() { + use mockito::Matcher; + + let mut server = Server::new_async().await; + let config = server + .mock("GET", "/v1/config") + .match_header( + "authorization", + Matcher::Regex( + r"^AWS4-HMAC-SHA256 Credential=CATALOG/\d{8}/us-east-1/execute-api/".into(), + ), + ) + .match_header("original-authorization", Matcher::Missing) + .with_status(200) + .with_body( + r#"{"defaults":{},"overrides":{ + "rest.signing-region":"eu-west-1", + "rest.access-key-id":"SERVER", + "rest.secret-access-key":"server-secret", + "token":"server-token" + }}"#, + ) + .create_async() + .await; + let catalog_request = server + .mock("GET", "/v1/namespaces") + .match_header( + "authorization", + Matcher::Regex( + r"^AWS4-HMAC-SHA256 Credential=SERVER/\d{8}/eu-west-1/execute-api/".into(), + ), + ) + .match_header("original-authorization", "Bearer server-token") + .with_status(200) + .with_body(r#"{"namespaces":[]}"#) + .create_async() + .await; + let context_request = server + .mock("GET", "/v1/namespaces") + .match_header( + "authorization", + Matcher::Regex( + r"^AWS4-HMAC-SHA256 Credential=CONTEXT/\d{8}/ap-south-1/execute-api/".into(), + ), + ) + .match_header("original-authorization", "Bearer server-token") + .match_header("x-amz-security-token", "context-token") + .with_status(200) + .with_body(r#"{"namespaces":[]}"#) + .create_async() + .await; + let catalog = test_catalog( + RestCatalogConfig::builder() + .uri(server.url()) + .props(sigv4_props()) + .build(), + ); + catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); + let context = SessionContext::builder() + .properties(HashMap::from([( + crate::REST_CATALOG_PROP_SIGNING_REGION.into(), + "ap-south-1".into(), + )])) + .credentials(HashMap::from([ + ( + crate::REST_CATALOG_PROP_ACCESS_KEY_ID.into(), + "CONTEXT".to_string().into(), + ), + ( + crate::REST_CATALOG_PROP_SECRET_ACCESS_KEY.into(), + "context-secret".to_string().into(), + ), + ( + crate::REST_CATALOG_PROP_SESSION_TOKEN.into(), + "context-token".to_string().into(), + ), + ])) + .build(); + catalog.list_namespaces(&context, None).await.unwrap(); + config.assert_async().await; + catalog_request.assert_async().await; + context_request.assert_async().await; + } + + #[cfg(feature = "sigv4")] + #[tokio::test] + async fn test_configured_sigv4_delegate_modes() { + for delegate in [None, Some("OAuth2"), Some("none")] { + let mut props = sigv4_props(); + props.insert("token".into(), "delegate-token".into()); + if let Some(delegate) = delegate { + props.insert( + REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE.into(), + delegate.into(), + ); + } + let manager = test_catalog( + RestCatalogConfig::builder() + .uri("http://localhost".into()) + .props(props.clone()) + .build(), + ) + .resolve_auth_manager() + .unwrap(); + let session = manager.init_session(&test_client(), &props).await.unwrap(); + let mut request = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut request).await.unwrap(); + assert!( + request + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap() + .starts_with("AWS4-HMAC-SHA256 ") + ); + if delegate == Some("none") { + assert!(!request.headers().contains_key("original-authorization")); + } else { + assert_eq!( + request.headers().get("original-authorization").unwrap(), + "Bearer delegate-token" + ); + } + } + let mut props = sigv4_props(); + props.insert( + REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE.into(), + "sigv4".into(), + ); + let err = test_catalog( + RestCatalogConfig::builder() + .uri("http://localhost".into()) + .props(props) + .build(), + ) + .resolve_auth_manager() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains(REST_CATALOG_PROP_SIGV4_DELEGATE_AUTH_TYPE) + ); + } + + #[tokio::test] + async fn test_legacy_sigv4_switch_and_injected_manager_precedence() { + let config = RestCatalogConfig::builder() + .uri("http://localhost".into()) + .props(HashMap::from([ + (REST_CATALOG_PROP_AUTH_TYPE.into(), AUTH_TYPE_NONE.into()), + (REST_CATALOG_PROP_SIGV4_ENABLED.into(), "TRUE".into()), + ])) + .build(); + assert_eq!(RestSessionCatalog::auth_type(&config), AUTH_TYPE_SIGV4); + let manager = test_catalog_with(config, NoopAuthManager) + .resolve_auth_manager() + .unwrap(); + assert!(!manager.signs_requests()); + } + + #[cfg(not(feature = "sigv4"))] + #[tokio::test] + async fn test_configured_sigv4_requires_the_feature() { + for (key, value) in [ + (REST_CATALOG_PROP_AUTH_TYPE, AUTH_TYPE_SIGV4), + (REST_CATALOG_PROP_SIGV4_ENABLED, "true"), + ] { + let config = RestCatalogConfig::builder() + .uri("http://localhost".into()) + .props(HashMap::from([(key.into(), value.into())])) + .build(); + let err = test_catalog(config).resolve_auth_manager().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!(err.message().contains("`sigv4` feature")); + } + } + + #[cfg(feature = "sigv4")] + #[tokio::test] + async fn test_configured_sigv4_does_not_follow_redirects() { + for legacy in [false, true] { + let mut server = Server::new_async().await; + let redirect = server + .mock("GET", "/v1/config") + .with_status(307) + .with_header("location", "/redirected") + .create_async() + .await; + let target = server + .mock("GET", "/redirected") + .expect(0) + .create_async() + .await; + let mut props = sigv4_props(); + if legacy { + props.insert(REST_CATALOG_PROP_AUTH_TYPE.into(), AUTH_TYPE_NONE.into()); + props.insert(REST_CATALOG_PROP_SIGV4_ENABLED.into(), "true".into()); + } + let catalog = test_catalog( + RestCatalogConfig::builder() + .uri(server.url()) + .props(props) + .build(), + ); + let err = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap_err(); + assert!(err.to_string().contains("307"), "{err}"); + redirect.assert_async().await; + target.assert_async().await; + } + } + + #[cfg(feature = "sigv4")] + #[tokio::test] + async fn test_a_signing_manager_does_not_follow_redirects() { + let mut server = Server::new_async().await; + let redirect = server + .mock("GET", "/v1/config") + .with_status(307) + .with_header("location", "https://elsewhere.example.com/v1/config") + .create_async() + .await; + + let config = RestCatalogConfig::builder().uri(server.url()).build(); + let err = test_catalog_with(config, test_sigv4_manager()) + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap_err(); + + assert!(err.to_string().contains("307"), "{err}"); + redirect.assert_async().await; + } + + #[tokio::test] + async fn test_a_non_signing_manager_still_follows_redirects() { + let mut server = Server::new_async().await; + let redirect = server + .mock("GET", "/v1/config") + .with_status(307) + .with_header("location", "/v1/config-final") + .create_async() + .await; + let final_config = server + .mock("GET", "/v1/config-final") + .with_status(200) + .with_body(r#"{"overrides": {}, "defaults": {}}"#) + .create_async() + .await; + let namespaces = server + .mock("GET", "/v1/namespaces") + .with_status(200) + .with_body(r#"{"namespaces": []}"#) + .create_async() + .await; + + let config = RestCatalogConfig::builder().uri(server.url()).build(); + test_catalog_with(config, NoopAuthManager) + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); + + redirect.assert_async().await; + final_config.assert_async().await; + namespaces.assert_async().await; + } + + #[cfg(feature = "sigv4")] + #[tokio::test] + async fn test_configured_authorization_header_does_not_clobber_the_signature() { + use mockito::{Matcher, Mock}; + + let mut server = Server::new_async().await; + let signed = |m: Mock| { + m.match_header("authorization", Matcher::Regex("^AWS4-HMAC-SHA256 ".into())) + .match_header("original-authorization", "Bearer configured") + }; + let config_mock = signed(server.mock("GET", "/v1/config")) + .with_status(200) + .with_body(r#"{"overrides": {}, "defaults": {}}"#) + .create_async() + .await; + let namespaces_mock = signed(server.mock("GET", "/v1/namespaces")) + .with_status(200) + .with_body(r#"{"namespaces": []}"#) + .create_async() + .await; + + let config = RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.authorization".to_string(), + "Bearer configured".to_string(), + )])) + .build(); + let catalog = test_catalog_with(config, test_sigv4_manager()); + + catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); + + config_mock.assert_async().await; + namespaces_mock.assert_async().await; + } + #[tokio::test] async fn test_with_auth_manager_overrides_config() { // A custom auth manager takes precedence over `rest.auth.type`. diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index f81f86574b..62be06ae08 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -151,12 +151,10 @@ impl HttpClient { auth_session: &dyn AuthSession, mut request: HttpRequest, ) -> Result { - // Authenticate first, then apply extra headers, so a configured - // `header.authorization` keeps overriding a token (unchanged behavior). + // `request()` already set the configured headers; applying them again + // here would override what authentication set, such as a signature. auth_session.authenticate(&mut request).await?; - - let mut request = request.into_inner(); - request.headers_mut().extend(self.extra_headers.clone()); + let request = request.into_inner(); HttpResponse::read(self.client.execute(request).await?).await }