Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions crates/catalog/rest/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ repository = { workspace = true }

[dependencies]
async-trait = { workspace = true }
# AWS SigV4 request signing for AWS-native Iceberg REST endpoints
# (S3 Tables, Glue). Enabled per-catalog via the `rest.sigv4-enabled`
# property; otherwise these are inert.
aws-config = "1.8"
aws-credential-types = "1.2"
aws-sigv4 = { version = "1.2", features = ["http1"] }
aws-smithy-runtime-api = { version = "1", features = ["client"] }
chrono = { workspace = true }
http = { workspace = true }
iceberg = { workspace = true }
Expand Down
40 changes: 39 additions & 1 deletion crates/catalog/rest/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,37 @@ impl RestCatalogConfig {
self.client.clone()
}

/// AWS SigV4 signing config, read from props. Returns
/// `Some((region, signing_name))` when `rest.sigv4-enabled` is
/// `"true"`, else `None` (no signing).
///
/// Used for AWS-native Iceberg REST endpoints (S3 Tables signing
/// name `s3tables`, Glue `glue`) which require every request to be
/// SigV4-signed. Region falls back to `s3.region`; signing name
/// defaults to `execute-api`.
pub(crate) fn sigv4_config(&self) -> Option<(String, String)> {
let enabled = self
.props
.get("rest.sigv4-enabled")
.map(|v| v == "true")
.unwrap_or(false);
if !enabled {
return None;
}
let region = self
.props
.get("rest.signing-region")
.or_else(|| self.props.get("s3.region"))
.cloned()
.unwrap_or_else(|| "us-east-1".to_string());
let name = self
.props
.get("rest.signing-name")
.cloned()
.unwrap_or_else(|| "execute-api".to_string());
Some((region, name))
}

/// Get the token from the config.
///
/// The client can use this token to send requests.
Expand Down Expand Up @@ -848,6 +879,12 @@ impl Catalog for RestCatalog {
}
};

// Stash the response-side config map BEFORE merging it with
// the catalog-level user_config — `Table::properties()`
// surfaces *only* the per-table response props so downstream
// consumers can spot vended credentials without re-issuing
// the request.
let response_config = response.config.clone();
let config = response
.config
.into_iter()
Expand All @@ -862,7 +899,8 @@ impl Catalog for RestCatalog {
.identifier(table_ident.clone())
.file_io(file_io)
.metadata(response.metadata)
.runtime(self.runtime.clone());
.runtime(self.runtime.clone())
.properties(response_config);

if let Some(metadata_location) = response.metadata_location {
table_builder.metadata_location(metadata_location).build()
Expand Down
137 changes: 135 additions & 2 deletions crates/catalog/rest/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,137 @@

use std::collections::HashMap;
use std::fmt::{Debug, Formatter};
use std::str::FromStr;

use http::StatusCode;
use iceberg::{Error, ErrorKind, Result};
use reqwest::header::HeaderMap;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use reqwest::{Client, IntoUrl, Method, Request, RequestBuilder, Response};
use serde::de::DeserializeOwned;
use tokio::sync::Mutex;
use tokio::sync::{Mutex, OnceCell};

use crate::RestCatalogConfig;
use crate::types::{ErrorResponse, TokenResponse};

/// AWS SigV4 request signer for AWS-native Iceberg REST endpoints
/// (S3 Tables, Glue). Credentials come from the standard AWS chain
/// (env, ECS/EKS task role, EC2 IMDS, shared config), loaded lazily on
/// first use and cached for the process lifetime.
struct SigV4Signer {
region: String,
name: String,
provider: OnceCell<aws_credential_types::provider::SharedCredentialsProvider>,
}

impl SigV4Signer {
fn new(region: String, name: String) -> Self {
Self {
region,
name,
provider: OnceCell::new(),
}
}

async fn provider(&self) -> Result<&aws_credential_types::provider::SharedCredentialsProvider> {
self.provider
.get_or_try_init(|| async {
let cfg = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
cfg.credentials_provider().ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"no AWS credentials provider for SigV4 signing",
)
})
})
.await
}

/// Sign `request` in place: adds `authorization`, `x-amz-date`, and
/// `x-amz-content-sha256` headers computed over the request as it
/// currently stands. Must be called after all other headers/body
/// are set and before sending.
async fn sign(&self, request: &mut Request) -> Result<()> {
use aws_credential_types::provider::ProvideCredentials;
use aws_sigv4::http_request::{
PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, sign,
};
use aws_sigv4::sign::v4;

let creds = self
.provider()
.await?
.provide_credentials()
.await
.map_err(|e| {
Error::new(ErrorKind::Unexpected, "load AWS credentials").with_source(e)
})?;
let identity: aws_smithy_runtime_api::client::identity::Identity = creds.into();

let mut settings = SigningSettings::default();
// AWS-native catalogs (S3 Tables/Glue) expect the payload hash.
settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256;

let params = v4::SigningParams::builder()
.identity(&identity)
.region(&self.region)
.name(&self.name)
.time(std::time::SystemTime::now())
.settings(settings)
.build()
.map_err(|e| Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e))?
.into();

let body_bytes = request
.body()
.and_then(|b| b.as_bytes())
.unwrap_or(&[])
.to_vec();
let uri = request.url().as_str().to_string();
let method = request.method().as_str().to_string();
let headers: Vec<(String, String)> = request
.headers()
.iter()
.map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string()))
.collect();

let signable = SignableRequest::new(
&method,
&uri,
headers.iter().map(|(k, v)| (k.as_str(), v.as_str())),
SignableBody::Bytes(&body_bytes),
)
.map_err(|e| Error::new(ErrorKind::Unexpected, "build signable request").with_source(e))?;

let (instructions, _signature) = sign(signable, &params)
.map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 sign").with_source(e))?
.into_parts();

// Apply the signing headers onto a throwaway http::Request that
// mirrors the outgoing one, then copy the resulting headers back.
let mut http_req = http::Request::builder()
.method(request.method().clone())
.uri(uri.as_str());
for (k, v) in request.headers().iter() {
http_req = http_req.header(k, v);
}
let mut http_req = http_req.body(()).map_err(|e| {
Error::new(ErrorKind::Unexpected, "build http request for signing").with_source(e)
})?;
instructions.apply_to_request_http1x(&mut http_req);

for (k, v) in http_req.headers().iter() {
let name = HeaderName::from_str(k.as_str()).map_err(|e| {
Error::new(ErrorKind::Unexpected, "signed header name").with_source(e)
})?;
let val = HeaderValue::from_bytes(v.as_bytes()).map_err(|e| {
Error::new(ErrorKind::Unexpected, "signed header value").with_source(e)
})?;
request.headers_mut().insert(name, val);
}
Ok(())
}
}

pub(crate) struct HttpClient {
client: Client,

Expand All @@ -45,6 +165,8 @@ pub(crate) struct HttpClient {
extra_oauth_params: HashMap<String, String>,
/// Whether to disable header redaction in error logs (defaults to false for security).
disable_header_redaction: bool,
/// AWS SigV4 signer, set when `rest.sigv4-enabled` is configured.
sigv4: Option<SigV4Signer>,
}

impl Debug for HttpClient {
Expand All @@ -68,6 +190,9 @@ impl HttpClient {
extra_headers,
extra_oauth_params: cfg.extra_oauth_params(),
disable_header_redaction: cfg.disable_header_redaction(),
sigv4: cfg
.sigv4_config()
.map(|(region, name)| SigV4Signer::new(region, name)),
})
}

Expand Down Expand Up @@ -96,6 +221,10 @@ impl HttpClient {
self.extra_oauth_params
},
disable_header_redaction: cfg.disable_header_redaction(),
sigv4: cfg
.sigv4_config()
.map(|(region, name)| SigV4Signer::new(region, name))
.or(self.sigv4),
})
}

Expand Down Expand Up @@ -253,6 +382,10 @@ impl HttpClient {
/// Executes the given `Request` and returns a `Response`.
pub async fn execute(&self, mut request: Request) -> Result<Response> {
request.headers_mut().extend(self.extra_headers.clone());
// SigV4 must sign last, over the final headers + body.
if let Some(signer) = &self.sigv4 {
signer.sign(&mut request).await?;
}
Ok(self.client.execute(request).await?)
}

Expand Down
7 changes: 6 additions & 1 deletion crates/iceberg/src/spec/snapshot_summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,9 +334,14 @@ pub(crate) fn update_snapshot_summaries(
previous_summary: Option<&Summary>,
truncate_full_table: bool,
) -> Result<Summary> {
// Validate that the operation is supported
// Validate that the operation is supported. `Replace` is treated like
// `Overwrite` for summary-update purposes — both swap files in/out
// without changing logical contents — but we keep them distinct in
// the snapshot summary so MoR tooling can distinguish compaction
// (`replace`) from logical overwrites (`overwrite`).
if summary.operation != Operation::Append
&& summary.operation != Operation::Overwrite
&& summary.operation != Operation::Replace
&& summary.operation != Operation::Delete
{
return Err(Error::new(
Expand Down
32 changes: 32 additions & 0 deletions crates/iceberg/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

//! Table API for Apache Iceberg

use std::collections::HashMap;
use std::sync::Arc;

use crate::arrow::ArrowReaderBuilder;
Expand All @@ -41,6 +42,7 @@ pub struct TableBuilder {
disable_cache: bool,
cache_size_bytes: Option<u64>,
runtime: Option<Runtime>,
properties: HashMap<String, String>,
}

impl TableBuilder {
Expand All @@ -55,6 +57,7 @@ impl TableBuilder {
disable_cache: false,
cache_size_bytes: None,
runtime: None,
properties: HashMap::new(),
}
}

Expand Down Expand Up @@ -118,6 +121,15 @@ impl TableBuilder {
self
}

/// optional - per-table properties returned by the catalog (e.g. the
/// `config` map from a REST `loadTable` response, which carries
/// vended S3 credentials when the catalog is asked for delegated
/// access).
pub fn properties(mut self, properties: HashMap<String, String>) -> Self {
self.properties = properties;
self
}

/// build the Table
pub fn build(self) -> Result<Table> {
let Self {
Expand All @@ -130,6 +142,7 @@ impl TableBuilder {
disable_cache,
cache_size_bytes,
runtime,
properties,
} = self;

let Some(file_io) = file_io else {
Expand Down Expand Up @@ -190,6 +203,7 @@ impl TableBuilder {
object_cache,
runtime,
encryption_manager,
properties,
})
}
}
Expand All @@ -205,6 +219,13 @@ pub struct Table {
object_cache: Arc<ObjectCache>,
runtime: Runtime,
encryption_manager: Option<Arc<EncryptionManager>>,
/// Per-table properties returned by the catalog. For REST catalogs,
/// this is the `config` map in the `loadTable` response — empty for
/// catalogs that don't surface table-scoped config (file, glue,
/// memory). Useful for downstream consumers that need to extract
/// vended credentials (`s3.access-key-id`, `s3.session-token`,
/// etc.) without re-issuing the REST request.
properties: HashMap<String, String>,
}

impl Table {
Expand Down Expand Up @@ -260,6 +281,17 @@ impl Table {
&self.file_io
}

/// Returns per-table properties returned by the catalog. For REST
/// catalogs, this is the `config` map from `loadTable`. Empty for
/// catalogs that don't surface table-scoped config.
///
/// Polynya patch: lets downstream consumers (e.g. pg2iceberg's
/// vended-S3 router) extract per-table S3 credentials without
/// duplicating the REST request.
pub fn properties(&self) -> &HashMap<String, String> {
&self.properties
}

/// Returns this table's object cache
pub(crate) fn object_cache(&self) -> Arc<ObjectCache> {
self.object_cache.clone()
Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/transaction/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ use crate::transaction::Transaction;
use crate::{Result, TableRequirement, TableUpdate};

/// A boxed, thread-safe reference to a `TransactionAction`.
pub(crate) type BoxedTransactionAction = Arc<dyn TransactionAction>;
pub type BoxedTransactionAction = Arc<dyn TransactionAction>;

/// A trait representing an atomic action that can be part of a transaction.
///
/// Implementors of this trait define how a specific action is committed to a table.
/// Each action is responsible for generating the updates and requirements needed
/// to modify the table metadata.
#[async_trait]
pub(crate) trait TransactionAction: AsAny + Sync + Send {
pub trait TransactionAction: AsAny + Sync + Send {
/// Commits this action against the provided table and returns the resulting updates.
/// NOTE: This function is intended for internal use only and should not be called directly by users.
///
Expand Down
Loading
Loading