diff --git a/Cargo.lock b/Cargo.lock index 8368191..e1cf7aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -999,6 +999,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "typetag", "url", ] diff --git a/Cargo.toml b/Cargo.toml index c6ab8e3..5bd5f8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ serde_json = "1.0.145" tokio = { version = "1.48.0", features = ["full"] } tracing = "0.1.41" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } +typetag = "0.2.23" url = "2.5.7" [dev-dependencies] diff --git a/README.md b/README.md index ea930e3..9872f13 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,11 @@ BERGR_REST_OAUTH2_CREDENTIAL="$(cat ~/.config/polaris-credential)" BERGR_REST_OA #### Table files -To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile. +To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile. `snapshot … files --verify` lists the table's data files with the same credentials where it can, and otherwise checks each file individually, which is slower. -Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. AWS Glue's table files are always read with your AWS credentials. +Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. + +AWS Glue vends credentials whatever the setting, through Lake Formation, so its table files are read with Lake Formation's permissions rather than your own access to S3. ### Direct access via metadata file location diff --git a/src/aws.rs b/src/aws.rs index b27fbac..7351117 100644 --- a/src/aws.rs +++ b/src/aws.rs @@ -4,11 +4,15 @@ use crate::error::ExpectedError; use crate::rest::rest_catalog_props; use crate::sigv4::{SigV4AuthManager, SigningScope}; use anyhow::{Context as _, Result}; -use aws_config::BehaviorVersion; use aws_config::meta::credentials::CredentialsProviderChain; +use aws_config::{BehaviorVersion, Region}; +use aws_credential_types::Credentials; use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider}; use iceberg::CatalogBuilder; -use iceberg::io::{FileIO, FileIOBuilder, S3_REGION}; +use iceberg::io::{ + CLIENT_REGION, FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, + S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, Storage, StorageConfig, StorageFactory, +}; use iceberg_catalog_glue::{ AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, @@ -18,6 +22,7 @@ use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; use reqsign_core::{Context, Error as ReqsignError, ErrorKind, Result as ReqsignResult}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; @@ -79,6 +84,71 @@ fn s3_storage_factory(aws_config: &aws_config::SdkConfig) -> Arc, +} + +#[typetag::serde(name = "BergrVendedFirstS3StorageFactory")] +impl StorageFactory for VendedFirstS3StorageFactory { + fn build(&self, config: &StorageConfig) -> iceberg::Result> { + OpenDalStorageFactory::S3 { + customized_credential_load: storage_credential_loader( + config.props(), + self.fallback.as_ref(), + ), + } + .build(config) + } +} + +/// A custom loader replaces opendal's whole credential chain, including any keys in the table's +/// config, so leave it out when the catalog has vended keys. +fn storage_credential_loader( + props: &HashMap, + fallback: Option<&CustomAwsCredentialLoader>, +) -> Option { + fallback + .cloned() + .filter(|_| !props.contains_key(S3_ACCESS_KEY_ID)) +} + +/// The S3 credentials a catalog vended for a table, if any, from the table's storage properties. +fn vended_credentials(props: &HashMap) -> Option { + Some(Credentials::new( + props.get(S3_ACCESS_KEY_ID)?, + props.get(S3_SECRET_ACCESS_KEY)?, + props.get(S3_SESSION_TOKEN).cloned(), + None, + "catalog", + )) +} + +/// An S3 client that uses the keys a catalog vended for a table, if it vended any, configured from +/// the table's storage properties the way opendal reads them. +pub fn vended_s3_client(props: &HashMap) -> Option { + let credentials = vended_credentials(props)?; + let region = props + .get(CLIENT_REGION) + .or_else(|| props.get(S3_REGION)) + .map(|region| Region::new(region.clone())); + let path_style = props + .get(S3_PATH_STYLE_ACCESS) + .is_some_and(|value| ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str())); + + let mut config = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .credentials_provider(credentials) + .region(region) + .force_path_style(path_style); + config.set_endpoint_url(props.get(S3_ENDPOINT).cloned()); + Some(aws_sdk_s3::Client::from_conf(config.build())) +} + pub async fn get_aws_config() -> Result { let config = aws_config::defaults(BehaviorVersion::latest()) .credentials_provider(build_credentials_provider()) @@ -171,7 +241,9 @@ pub async fn aws_rest_catalog( let catalog = RestCatalogBuilder::default() .with_auth_manager(SigV4AuthManager::new(scope, credentials)) - .with_storage_factory(s3_storage_factory(aws_config)) + .with_storage_factory(Arc::new(VendedFirstS3StorageFactory { + fallback: credential_loader(aws_config), + })) .load("rest", props) .await?; @@ -243,6 +315,59 @@ mod tests { assert!(check_credentials(&aws_config).await.is_ok()); } + fn vended_props() -> HashMap { + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "vended-key".to_string()), + ( + S3_SECRET_ACCESS_KEY.to_string(), + "vended-secret".to_string(), + ), + (S3_SESSION_TOKEN.to_string(), "vended-token".to_string()), + ]) + } + + #[tokio::test] + async fn storage_uses_no_custom_loader_when_the_catalog_vends_keys() { + let fallback = credential_loader(&test_aws_config().await); + + assert!(storage_credential_loader(&vended_props(), fallback.as_ref()).is_none()); + } + + #[tokio::test] + async fn storage_falls_back_to_bergr_credentials_without_vended_keys() { + let fallback = credential_loader(&test_aws_config().await); + + assert!(storage_credential_loader(&HashMap::new(), fallback.as_ref()).is_some()); + } + + #[test] + fn no_vended_s3_client_without_vended_keys() { + assert!(vended_s3_client(&HashMap::new()).is_none()); + } + + #[test] + fn vended_credentials_come_from_the_table_props() { + let credentials = vended_credentials(&vended_props()).unwrap(); + + assert_eq!(credentials.access_key_id(), "vended-key"); + assert_eq!(credentials.secret_access_key(), "vended-secret"); + assert_eq!(credentials.session_token(), Some("vended-token")); + } + + #[test] + fn vended_s3_client_prefers_client_region_as_opendal_does() { + let mut props = vended_props(); + props.insert(S3_REGION.to_string(), "us-east-1".to_string()); + props.insert(CLIENT_REGION.to_string(), "us-west-2".to_string()); + + let client = vended_s3_client(&props).unwrap(); + + assert_eq!( + client.config().region().map(|r| r.as_ref()), + Some("us-west-2") + ); + } + #[tokio::test] async fn test_s3_file_io_sets_region() { let aws_config = test_aws_config().await; diff --git a/src/file_existence.rs b/src/file_existence.rs index 9af0035..6475d77 100644 --- a/src/file_existence.rs +++ b/src/file_existence.rs @@ -9,7 +9,7 @@ use aws_sdk_s3::Client; use iceberg::io::FileIO; use std::collections::HashSet; use tracing::debug; -use tracing::info; +use tracing::{info, warn}; /// Checks whether files exist at given locations. #[async_trait] @@ -65,7 +65,9 @@ impl FileExistenceChecker for PreloadedExistenceChecker { /// If the data prefix is on S3 and an S3 client is provided, uses a bulk /// `ListObjectsV2` call and returns a `PreloadedExistenceChecker`. /// -/// Otherwise, returns a `FileIOExistenceChecker` that delegates to per-file checks. +/// Otherwise, or if the listing fails, returns a `FileIOExistenceChecker` that delegates to +/// per-file checks. Listing is only an optimisation, and credentials that can read a table's files +/// can't always list them; Polaris's vended keys, for example, can't. pub async fn create_existence_checker( file_io: FileIO, data_prefix: &str, @@ -77,13 +79,17 @@ pub async fn create_existence_checker( if let Some((bucket, prefix)) = parse_s3_url(data_prefix) && let Some(client) = s3_client { - let base_url = format!("s3://{}/{}", bucket, prefix); - let suffixes = list_object_suffixes(client, bucket, prefix).await?; - debug!( - file_count = suffixes.len(), - "Using preloaded S3 existence checker" - ); - return Ok(Box::new(PreloadedExistenceChecker::new(base_url, suffixes))); + match list_object_suffixes(client, bucket, prefix).await { + Ok(suffixes) => { + debug!( + file_count = suffixes.len(), + "Using preloaded S3 existence checker" + ); + let base_url = format!("s3://{}/{}", bucket, prefix); + return Ok(Box::new(PreloadedExistenceChecker::new(base_url, suffixes))); + } + Err(err) => warn!(error = format!("{err:#}"), "Couldn't list S3 objects"), + } } debug!("Using FileIO existence checker"); @@ -151,6 +157,60 @@ mod tests { assert_eq!(parse_s3_url("not-a-url"), None); } + /// An S3 client whose every request is denied, as with vended keys that can read objects but + /// not list them. + fn access_denied_s3_client(server: &mockito::Server) -> Client { + let config = aws_sdk_s3::Config::builder() + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + "key", "secret", None, None, "test", + )) + .region(aws_sdk_s3::config::Region::new("us-east-1")) + .endpoint_url(server.url()) + .force_path_style(true) + .build(); + Client::from_conf(config) + } + + #[tokio::test] + async fn falls_back_to_file_io_when_listing_is_denied() { + let mut server = mockito::Server::new_async().await; + let _denied = server + .mock("GET", mockito::Matcher::Any) + .with_status(403) + .with_body("AccessDeniedAccess Denied") + .create_async() + .await; + let file_io = FileIO::new_with_memory(); + file_io + .new_output("s3://bucket/data/file1.parquet") + .unwrap() + .write(bytes::Bytes::from("data")) + .await + .unwrap(); + + let checker = create_existence_checker( + file_io, + "s3://bucket/data/", + Some(&access_denied_s3_client(&server)), + ) + .await + .unwrap(); + + assert!( + checker + .exists("s3://bucket/data/file1.parquet") + .await + .unwrap() + ); + assert!( + !checker + .exists("s3://bucket/data/file2.parquet") + .await + .unwrap() + ); + } + #[tokio::test] async fn test_preloaded_checker() { let mut suffixes = HashSet::new(); diff --git a/src/table_commands.rs b/src/table_commands.rs index a57991a..13ee167 100644 --- a/src/table_commands.rs +++ b/src/table_commands.rs @@ -1,3 +1,4 @@ +use crate::aws::vended_s3_client; use crate::cli::{SnapshotCmd, TableCommands}; use crate::error::ExpectedError; use crate::file_existence::{FileExistenceChecker, create_existence_checker}; @@ -257,6 +258,8 @@ async fn handle_snapshot_files( ) -> Result<()> { let existence_checker: Option> = if verify { let prefix = data_file_prefix(table.metadata())?; + let vended_s3_client = vended_s3_client(table.file_io().config().props()); + let s3_client = vended_s3_client.as_ref().or(s3_client); Some(create_existence_checker(table.file_io().clone(), &prefix, s3_client).await?) } else { None