Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ serde_json = "1.0.145"
tokio = { version = "1.48.0", features = ["full"] }
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.20", features = ["env-filter"] }
typetag = "0.2.23"
url = "2.5.7"

[dev-dependencies]
Expand Down
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,11 @@ BERGR_REST_OAUTH2_CREDENTIAL="$(cat ~/.config/polaris-credential)" BERGR_REST_OA

#### Table files

To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile.
To read a table's files, `bergr rest` asks the catalog for short-lived storage credentials ("vended credentials"), and uses the storage settings the catalog returns with each table. If the catalog doesn't vend credentials, bergr falls back to your own: for S3, the standard AWS credential variables and profile. `snapshot … files --verify` lists the table's data files with the same credentials where it can, and otherwise checks each file individually, which is slower.

Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`. AWS Glue's table files are always read with your AWS credentials.
Set `--access-delegation none` (or `BERGR_REST_ACCESS_DELEGATION=none`) to tell the catalog that bergr doesn't need vended credentials. This sends `X-Iceberg-Access-Delegation: none` instead of `vended-credentials`.

AWS Glue vends credentials whatever the setting, through Lake Formation, so its table files are read with Lake Formation's permissions rather than your own access to S3.

### Direct access via metadata file location

Expand Down
131 changes: 128 additions & 3 deletions src/aws.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,15 @@ use crate::error::ExpectedError;
use crate::rest::rest_catalog_props;
use crate::sigv4::{SigV4AuthManager, SigningScope};
use anyhow::{Context as _, Result};
use aws_config::BehaviorVersion;
use aws_config::meta::credentials::CredentialsProviderChain;
use aws_config::{BehaviorVersion, Region};
use aws_credential_types::Credentials;
use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider};
use iceberg::CatalogBuilder;
use iceberg::io::{FileIO, FileIOBuilder, S3_REGION};
use iceberg::io::{
CLIENT_REGION, FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS,
S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, Storage, StorageConfig, StorageFactory,
};
use iceberg_catalog_glue::{
AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN,
GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder,
Expand All @@ -18,6 +22,7 @@ use iceberg_storage_opendal::{
AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential,
};
use reqsign_core::{Context, Error as ReqsignError, ErrorKind, Result as ReqsignResult};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;

Expand Down Expand Up @@ -79,6 +84,71 @@ fn s3_storage_factory(aws_config: &aws_config::SdkConfig) -> Arc<OpenDalStorageF
})
}

/// S3 storage that reads a table's files with credentials vended by the catalog, when the table's
/// config has them, and with bergr's own AWS credentials otherwise.
#[derive(Debug, Serialize, Deserialize)]
struct VendedFirstS3StorageFactory {
/// Process-local, so not serialised; a deserialised factory has no fallback.
#[serde(skip)]
fallback: Option<CustomAwsCredentialLoader>,
}

#[typetag::serde(name = "BergrVendedFirstS3StorageFactory")]
impl StorageFactory for VendedFirstS3StorageFactory {
fn build(&self, config: &StorageConfig) -> iceberg::Result<Arc<dyn Storage>> {
OpenDalStorageFactory::S3 {
customized_credential_load: storage_credential_loader(
config.props(),
self.fallback.as_ref(),
),
}
.build(config)
}
}

/// A custom loader replaces opendal's whole credential chain, including any keys in the table's
/// config, so leave it out when the catalog has vended keys.
fn storage_credential_loader(
props: &HashMap<String, String>,
fallback: Option<&CustomAwsCredentialLoader>,
) -> Option<CustomAwsCredentialLoader> {
fallback
.cloned()
.filter(|_| !props.contains_key(S3_ACCESS_KEY_ID))
}

/// The S3 credentials a catalog vended for a table, if any, from the table's storage properties.
fn vended_credentials(props: &HashMap<String, String>) -> Option<Credentials> {
Some(Credentials::new(
props.get(S3_ACCESS_KEY_ID)?,
props.get(S3_SECRET_ACCESS_KEY)?,
props.get(S3_SESSION_TOKEN).cloned(),
None,
"catalog",
))
}

/// An S3 client that uses the keys a catalog vended for a table, if it vended any, configured from
/// the table's storage properties the way opendal reads them.
pub fn vended_s3_client(props: &HashMap<String, String>) -> Option<aws_sdk_s3::Client> {
let credentials = vended_credentials(props)?;
let region = props
.get(CLIENT_REGION)
.or_else(|| props.get(S3_REGION))
.map(|region| Region::new(region.clone()));
let path_style = props
.get(S3_PATH_STYLE_ACCESS)
.is_some_and(|value| ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()));

let mut config = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.credentials_provider(credentials)
.region(region)
.force_path_style(path_style);
config.set_endpoint_url(props.get(S3_ENDPOINT).cloned());
Some(aws_sdk_s3::Client::from_conf(config.build()))
}

pub async fn get_aws_config() -> Result<aws_config::SdkConfig> {
let config = aws_config::defaults(BehaviorVersion::latest())
.credentials_provider(build_credentials_provider())
Expand Down Expand Up @@ -171,7 +241,9 @@ pub async fn aws_rest_catalog(

let catalog = RestCatalogBuilder::default()
.with_auth_manager(SigV4AuthManager::new(scope, credentials))
.with_storage_factory(s3_storage_factory(aws_config))
.with_storage_factory(Arc::new(VendedFirstS3StorageFactory {
fallback: credential_loader(aws_config),
}))
.load("rest", props)
.await?;

Expand Down Expand Up @@ -243,6 +315,59 @@ mod tests {
assert!(check_credentials(&aws_config).await.is_ok());
}

fn vended_props() -> HashMap<String, String> {
HashMap::from([
(S3_ACCESS_KEY_ID.to_string(), "vended-key".to_string()),
(
S3_SECRET_ACCESS_KEY.to_string(),
"vended-secret".to_string(),
),
(S3_SESSION_TOKEN.to_string(), "vended-token".to_string()),
])
}

#[tokio::test]
async fn storage_uses_no_custom_loader_when_the_catalog_vends_keys() {
let fallback = credential_loader(&test_aws_config().await);

assert!(storage_credential_loader(&vended_props(), fallback.as_ref()).is_none());
}

#[tokio::test]
async fn storage_falls_back_to_bergr_credentials_without_vended_keys() {
let fallback = credential_loader(&test_aws_config().await);

assert!(storage_credential_loader(&HashMap::new(), fallback.as_ref()).is_some());
}

#[test]
fn no_vended_s3_client_without_vended_keys() {
assert!(vended_s3_client(&HashMap::new()).is_none());
}

#[test]
fn vended_credentials_come_from_the_table_props() {
let credentials = vended_credentials(&vended_props()).unwrap();

assert_eq!(credentials.access_key_id(), "vended-key");
assert_eq!(credentials.secret_access_key(), "vended-secret");
assert_eq!(credentials.session_token(), Some("vended-token"));
}

#[test]
fn vended_s3_client_prefers_client_region_as_opendal_does() {
let mut props = vended_props();
props.insert(S3_REGION.to_string(), "us-east-1".to_string());
props.insert(CLIENT_REGION.to_string(), "us-west-2".to_string());

let client = vended_s3_client(&props).unwrap();

assert_eq!(
client.config().region().map(|r| r.as_ref()),
Some("us-west-2")
);
}

#[tokio::test]
async fn test_s3_file_io_sets_region() {
let aws_config = test_aws_config().await;
Expand Down
78 changes: 69 additions & 9 deletions src/file_existence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use aws_sdk_s3::Client;
use iceberg::io::FileIO;
use std::collections::HashSet;
use tracing::debug;
use tracing::info;
use tracing::{info, warn};

/// Checks whether files exist at given locations.
#[async_trait]
Expand Down Expand Up @@ -65,7 +65,9 @@ impl FileExistenceChecker for PreloadedExistenceChecker {
/// If the data prefix is on S3 and an S3 client is provided, uses a bulk
/// `ListObjectsV2` call and returns a `PreloadedExistenceChecker`.
///
/// Otherwise, returns a `FileIOExistenceChecker` that delegates to per-file checks.
/// Otherwise, or if the listing fails, returns a `FileIOExistenceChecker` that delegates to
/// per-file checks. Listing is only an optimisation, and credentials that can read a table's files
/// can't always list them; Polaris's vended keys, for example, can't.
pub async fn create_existence_checker(
file_io: FileIO,
data_prefix: &str,
Expand All @@ -77,13 +79,17 @@ pub async fn create_existence_checker(
if let Some((bucket, prefix)) = parse_s3_url(data_prefix)
&& let Some(client) = s3_client
{
let base_url = format!("s3://{}/{}", bucket, prefix);
let suffixes = list_object_suffixes(client, bucket, prefix).await?;
debug!(
file_count = suffixes.len(),
"Using preloaded S3 existence checker"
);
return Ok(Box::new(PreloadedExistenceChecker::new(base_url, suffixes)));
match list_object_suffixes(client, bucket, prefix).await {
Ok(suffixes) => {
debug!(
file_count = suffixes.len(),
"Using preloaded S3 existence checker"
);
let base_url = format!("s3://{}/{}", bucket, prefix);
return Ok(Box::new(PreloadedExistenceChecker::new(base_url, suffixes)));
}
Err(err) => warn!(error = format!("{err:#}"), "Couldn't list S3 objects"),
}
}

debug!("Using FileIO existence checker");
Expand Down Expand Up @@ -151,6 +157,60 @@ mod tests {
assert_eq!(parse_s3_url("not-a-url"), None);
}

/// An S3 client whose every request is denied, as with vended keys that can read objects but
/// not list them.
fn access_denied_s3_client(server: &mockito::Server) -> Client {
let config = aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.credentials_provider(aws_sdk_s3::config::Credentials::new(
"key", "secret", None, None, "test",
))
.region(aws_sdk_s3::config::Region::new("us-east-1"))
.endpoint_url(server.url())
.force_path_style(true)
.build();
Client::from_conf(config)
}

#[tokio::test]
async fn falls_back_to_file_io_when_listing_is_denied() {
let mut server = mockito::Server::new_async().await;
let _denied = server
.mock("GET", mockito::Matcher::Any)
.with_status(403)
.with_body("<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>")
.create_async()
.await;
let file_io = FileIO::new_with_memory();
file_io
.new_output("s3://bucket/data/file1.parquet")
.unwrap()
.write(bytes::Bytes::from("data"))
.await
.unwrap();

let checker = create_existence_checker(
file_io,
"s3://bucket/data/",
Some(&access_denied_s3_client(&server)),
)
.await
.unwrap();

assert!(
checker
.exists("s3://bucket/data/file1.parquet")
.await
.unwrap()
);
assert!(
!checker
.exists("s3://bucket/data/file2.parquet")
.await
.unwrap()
);
}

#[tokio::test]
async fn test_preloaded_checker() {
let mut suffixes = HashSet::new();
Expand Down
3 changes: 3 additions & 0 deletions src/table_commands.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::aws::vended_s3_client;
use crate::cli::{SnapshotCmd, TableCommands};
use crate::error::ExpectedError;
use crate::file_existence::{FileExistenceChecker, create_existence_checker};
Expand Down Expand Up @@ -257,6 +258,8 @@ async fn handle_snapshot_files<W: Write>(
) -> Result<()> {
let existence_checker: Option<Box<dyn FileExistenceChecker>> = if verify {
let prefix = data_file_prefix(table.metadata())?;
let vended_s3_client = vended_s3_client(table.file_io().config().props());
let s3_client = vended_s3_client.as_ref().or(s3_client);
Some(create_existence_checker(table.file_io().clone(), &prefix, s3_client).await?)
} else {
None
Expand Down
Loading