Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,6 @@ url = "2.5.7"

[dev-dependencies]
mockito = "1"

[lints.clippy]
unwrap_used = "warn"
2 changes: 2 additions & 0 deletions clippy.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Lints configured in Cargo.toml [lints.clippy]
allow-unwrap-in-tests = true
73 changes: 64 additions & 9 deletions src/aws.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! AWS integration utilities for credential loading

use crate::error::ExpectedError;
use crate::rest::rest_catalog_props;
use crate::sigv4::{SigV4AuthManager, SigningScope};
use anyhow::{Context as _, Result};
Expand Down Expand Up @@ -78,21 +79,32 @@ fn s3_storage_factory(aws_config: &aws_config::SdkConfig) -> Arc<OpenDalStorageF
})
}

pub async fn get_aws_config() -> aws_config::SdkConfig {
pub async fn get_aws_config() -> Result<aws_config::SdkConfig> {
let config = aws_config::defaults(BehaviorVersion::latest())
.credentials_provider(build_credentials_provider())
.load()
.await;

// Try to resolve credentials to detect configuration errors early
if let Some(creds_provider) = config.credentials_provider() {
creds_provider
.provide_credentials()
.await
.expect("AWS credentials not configured");
}
check_credentials(&config).await?;
Ok(config)
}

config
/// Resolve credentials now, so a missing or broken configuration fails early with a clear error.
pub async fn check_credentials(aws_config: &aws_config::SdkConfig) -> Result<()> {
let Some(provider) = aws_config.credentials_provider() else {
return Ok(());
};
provider
.provide_credentials()
.await
.map(|_| ())
.map_err(|err| {
ExpectedError::UserInput(format!(
"AWS credentials not configured: {:#}",
anyhow::Error::new(err)
))
.into()
})
}

pub fn s3_file_io(aws_config: &aws_config::SdkConfig) -> FileIO {
Expand Down Expand Up @@ -188,6 +200,49 @@ mod tests {
.await
}

#[derive(Debug)]
struct NoCredentials;

impl ProvideCredentials for NoCredentials {
fn provide_credentials<'a>(
&'a self,
) -> aws_credential_types::provider::future::ProvideCredentials<'a>
where
Self: 'a,
{
aws_credential_types::provider::future::ProvideCredentials::ready(Err(
aws_credential_types::provider::error::CredentialsError::not_loaded(
"no credentials here",
),
))
}
}

#[tokio::test]
async fn check_credentials_reports_missing_credentials_as_expected_error() {
let aws_config = aws_config::defaults(BehaviorVersion::latest())
.credentials_provider(NoCredentials)
.load()
.await;

let err = check_credentials(&aws_config).await.unwrap_err();

let expected = err
.downcast_ref::<ExpectedError>()
.expect("should be an ExpectedError");
assert!(
expected.to_string().contains("AWS credentials"),
"unexpected message: {expected}"
);
}

#[tokio::test]
async fn check_credentials_accepts_working_credentials() {
let aws_config = test_aws_config().await;

assert!(check_credentials(&aws_config).await.is_ok());
}

#[tokio::test]
async fn test_s3_file_io_sets_region() {
let aws_config = test_aws_config().await;
Expand Down
31 changes: 18 additions & 13 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,28 @@ use clap::Parser;
use iceberg::io::FileIO;
use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};

/// Logged at debug level with `--debug`: bergr itself, plus the AWS SDK's Glue, S3 and HTTP operations.
const DEBUG_TARGETS: [&str; 4] = ["bergr", "aws_sdk_glue", "aws_sdk_s3", "aws_smithy_runtime"];
#[tokio::main]
async fn main() {
let cli = Cli::parse();

if cli.debug {
let filter = DEBUG_TARGETS.iter().fold(
EnvFilter::builder()
.with_default_directive(tracing::Level::INFO.into())
.from_env_lossy(),
|filter, target| {
filter.add_directive(
format!("{target}=debug")
.parse()
.expect("debug targets are valid directives"),
)
},
);
tracing_subscriber::registry()
.with(fmt::layer().with_writer(std::io::stderr))
.with(
EnvFilter::builder()
.with_default_directive(tracing::Level::INFO.into())
.from_env_lossy()
.add_directive("bergr=debug".parse().unwrap())
// AWS SDK tracing for Glue, S3, and HTTP operations
.add_directive("aws_sdk_glue=debug".parse().unwrap())
.add_directive("aws_sdk_s3=debug".parse().unwrap())
.add_directive("aws_smithy_runtime=debug".parse().unwrap()),
)
.with(filter)
.init();
}

Expand All @@ -55,7 +60,7 @@ async fn run(command: Commands) -> Result<()> {
Commands::From { location, command } => {
let is_s3 = location.starts_with("s3://") || location.starts_with("s3a://");
let (file_io, s3) = if is_s3 {
let aws_config = get_aws_config().await;
let aws_config = get_aws_config().await?;
(
s3_file_io(&aws_config),
Some(aws_sdk_s3::Client::new(&aws_config)),
Expand All @@ -68,7 +73,7 @@ async fn run(command: Commands) -> Result<()> {
handle_table_command(&table, command, &mut output, s3.as_ref()).await?;
}
Commands::Glue { command } => {
let aws_config = get_aws_config().await;
let aws_config = get_aws_config().await?;
let catalog = glue_catalog(&aws_config).await?;
let s3 = aws_sdk_s3::Client::new(&aws_config);
let mut output = TerminalOutput::new();
Expand All @@ -85,7 +90,7 @@ async fn run(command: Commands) -> Result<()> {
match RestAuth::resolve(auth, &uri, |name| std::env::var(name).ok())? {
RestAuth::SigV4(scope) => {
let aws_config = get_aws_config()
.await
.await?
.to_builder()
.region(Region::new(scope.region.clone()))
.build();
Expand Down
Loading