From b3e291e2c94f7f7023ec00128056cd0c716a25cf Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 27 Aug 2026 01:13:42 -0400 Subject: [PATCH 01/30] feat(rest): add a SigV4 request signer --- Cargo.lock | 5 + Cargo.toml | 7 + crates/catalog/rest/Cargo.toml | 5 + crates/catalog/rest/public-api.txt | 26 + crates/catalog/rest/src/auth/mod.rs | 2 + crates/catalog/rest/src/auth/sigv4/mod.rs | 22 + crates/catalog/rest/src/auth/sigv4/signer.rs | 978 +++++++++++++++++++ 7 files changed, 1045 insertions(+) create mode 100644 crates/catalog/rest/src/auth/sigv4/mod.rs create mode 100644 crates/catalog/rest/src/auth/sigv4/signer.rs diff --git a/Cargo.lock b/Cargo.lock index e2900acb2e..416153a369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2830,9 +2830,13 @@ name = "iceberg-catalog-rest" version = "0.10.1" dependencies = [ "async-trait", + "aws-credential-types", + "aws-sigv4", + "base64 0.22.1", "bytes", "chrono", "futures", + "hmac 0.12.1", "http 1.5.0", "iceberg", "iceberg_test_utils", @@ -2842,6 +2846,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sha2 0.10.9", "tokio", "tracing", "typed-builder", diff --git a/Cargo.toml b/Cargo.toml index 661c56b094..5cf55d2a76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" +aws-credential-types = "1.2" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", @@ -68,6 +69,10 @@ aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-sigv4 = { version = "1.4", default-features = false, features = [ + "sign-http", + "http1", +] } backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" @@ -87,6 +92,7 @@ fnv = "1.0.7" form_urlencoded = "1.2.2" futures = "0.3" hive_metastore = "0.2.0" +hmac = "0.12" http = "1.2" iceberg = { version = "0.10.0", path = "./crates/iceberg" } iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" } @@ -124,6 +130,7 @@ serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" +sha2 = "0.10" sqlx = { version = "0.8.1", default-features = false } strum = "0.27.2" syn = "2" diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 8dc9a86d7f..06477503e2 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -31,6 +31,9 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } +aws-credential-types = { workspace = true } +aws-sigv4 = { workspace = true } +base64 = { workspace = true } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } @@ -39,6 +42,7 @@ reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } typed-builder = { workspace = true } @@ -47,6 +51,7 @@ uuid = { workspace = true, features = ["v4"] } [dev-dependencies] bytes = { workspace = true } futures = { workspace = true } +hmac = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } mockito = { workspace = true } # `stream` lets tests build a streaming body to exercise HttpRequestBody::Streaming. diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 47393b5738..0cf8a25fee 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -14,6 +14,24 @@ impl<'a> core::fmt::Debug for iceberg_catalog_rest::HttpRequestBody<'a> pub fn iceberg_catalog_rest::HttpRequestBody<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl<'a> core::marker::Copy for iceberg_catalog_rest::HttpRequestBody<'a> impl<'a> core::marker::StructuralPartialEq for iceberg_catalog_rest::HttpRequestBody<'a> +pub enum iceberg_catalog_rest::PayloadHashMode +pub iceberg_catalog_rest::PayloadHashMode::IcebergRest +pub iceberg_catalog_rest::PayloadHashMode::StandardAws +impl core::clone::Clone for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::clone(&self) -> iceberg_catalog_rest::PayloadHashMode +impl core::cmp::Eq for iceberg_catalog_rest::PayloadHashMode +impl core::cmp::PartialEq for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::eq(&self, other: &iceberg_catalog_rest::PayloadHashMode) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg_catalog_rest::PayloadHashMode +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::PayloadHashMode +pub struct iceberg_catalog_rest::AwsCredentials +pub iceberg_catalog_rest::AwsCredentials::access_key_id: alloc::string::String +pub iceberg_catalog_rest::AwsCredentials::secret_access_key: iceberg::sensitive::SensitiveString +pub iceberg_catalog_rest::AwsCredentials::session_token: core::option::Option +impl core::clone::Clone for iceberg_catalog_rest::AwsCredentials +pub fn iceberg_catalog_rest::AwsCredentials::clone(&self) -> iceberg_catalog_rest::AwsCredentials pub struct iceberg_catalog_rest::CommitTableRequest pub iceberg_catalog_rest::CommitTableRequest::identifier: core::option::Option pub iceberg_catalog_rest::CommitTableRequest::requirements: alloc::vec::Vec @@ -331,6 +349,14 @@ impl core::default::Default for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::default() -> Self impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::SigV4Signer +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::new(credentials: iceberg_catalog_rest::AwsCredentials, region: alloc::string::String, service: alloc::string::String, mode: iceberg_catalog_rest::PayloadHashMode) -> Self +pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut reqwest::async_impl::request::Request) -> iceberg::error::Result<()> +impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer +impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_catalog_rest::StorageCredential pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index b4c0cf0be2..b601a4066e 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -19,6 +19,7 @@ //! `AuthManager`/`AuthSession` API. mod oauth2; +mod sigv4; use std::collections::HashMap; use std::fmt::Debug; @@ -27,6 +28,7 @@ use std::sync::Arc; use async_trait::async_trait; use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; +pub use sigv4::{AwsCredentials, PayloadHashMode, SigV4Signer}; use crate::client::HttpClient; use crate::request::HttpRequest; diff --git a/crates/catalog/rest/src/auth/sigv4/mod.rs b/crates/catalog/rest/src/auth/sigv4/mod.rs new file mode 100644 index 0000000000..6f1e9dea30 --- /dev/null +++ b/crates/catalog/rest/src/auth/sigv4/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +mod signer; + +pub use signer::{AwsCredentials, PayloadHashMode, SigV4Signer}; diff --git a/crates/catalog/rest/src/auth/sigv4/signer.rs b/crates/catalog/rest/src/auth/sigv4/signer.rs new file mode 100644 index 0000000000..de4c811186 --- /dev/null +++ b/crates/catalog/rest/src/auth/sigv4/signer.rs @@ -0,0 +1,978 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use chrono::{DateTime, Utc}; +#[cfg(test)] +use hmac::{Hmac, Mac}; +use iceberg::sensitive::SensitiveString; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; + +/// Hex SHA-256 of the empty string. +const EMPTY_BODY_HEX_SHA256: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PayloadHashMode { + /// Iceberg Java's RESTSigV4 style: base64 header for non-empty bodies, hex + /// for empty; the canonical request always uses hex. + IcebergRest, + /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). + StandardAws, +} + +/// Derives the AWS SigV4 signing key. +#[cfg(test)] +fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { + let mut mac = as Mac>::new_from_slice(key).expect("HMAC takes a key of any size"); + mac.update(data); + mac.finalize().into_bytes().to_vec() +} + +fn hex_sha256(data: &[u8]) -> String { + encode_hex(&Sha256::digest(data)) +} + +#[cfg(test)] +fn hex_hmac_sha256(key: &[u8], data: &[u8]) -> String { + encode_hex(&hmac_sha256(key, data)) +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +#[cfg(test)] +fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec { + let k_date = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); + let k_region = hmac_sha256(&k_date, region.as_bytes()); + let k_service = hmac_sha256(&k_region, service.as_bytes()); + hmac_sha256(&k_service, b"aws4_request") +} + +/// Computes the value of the `x-amz-content-sha256` header. +fn content_sha256_header(body: &[u8], mode: PayloadHashMode) -> String { + match mode { + PayloadHashMode::StandardAws => hex_sha256(body), + PayloadHashMode::IcebergRest => { + if body.is_empty() { + EMPTY_BODY_HEX_SHA256.to_string() + } else { + base64_encode(&Sha256::digest(body)) + } + } + } +} + +/// Static AWS-style credentials used for SigV4 signing of catalog requests. +#[derive(Clone)] +pub struct AwsCredentials { + /// AWS access key id. + pub access_key_id: String, + /// AWS secret access key. + pub secret_access_key: SensitiveString, + /// Optional STS session token. + pub session_token: Option, +} + +/// AWS SigV4 signer following Iceberg Java's `RESTSigV4AuthSession`: it adds the +/// required amz headers and signs all request headers except a small blacklist. +#[derive(Clone)] +pub struct SigV4Signer { + credentials: AwsCredentials, + region: String, + service: String, + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Creates a new SigV4 signer. + pub fn new( + credentials: AwsCredentials, + region: String, + service: String, + mode: PayloadHashMode, + ) -> Self { + Self { + credentials, + region, + service, + mode, + } + } + + /// Signs `request` in place, rewriting it where signing requires it: an + /// existing `Authorization` moves to `Original-Authorization` and is signed + /// over, userinfo is dropped from the URL, and a `+` in the query becomes + /// `%20`. + /// + /// That last one means a `+` is taken to be an encoded space, which is what + /// `RequestBuilder::query` writes; encode a literal plus as `%2B`. AWS + /// requires spaces as `%20` in a signed URL either way. + /// + /// Note that `aws_sigv4` traces the request it is given, so `RUST_LOG` at + /// trace level will print these headers. + pub fn sign(&self, request: &mut reqwest::Request) -> Result<()> { + self.sign_at(request, Utc::now()) + } + + fn sign_at(&self, request: &mut reqwest::Request, now: DateTime) -> Result<()> { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SignableBody, SignableRequest, + SigningSettings, UriPathNormalizationMode, sign, + }; + use aws_sigv4::sign::v4; + + let body: Vec = match request.body() { + None => Vec::new(), + Some(b) => b + .as_bytes() + .ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + ) + })? + .to_vec(), + }; + let content_header = content_sha256_header(&body, self.mode); + + // Renamed before signing, as Java's `convertHeaders` does, so a + // delegate's bearer token travels along and is signed over. + let displaced_authorization: Vec<_> = request + .headers() + .get_all(reqwest::header::AUTHORIZATION) + .iter() + .cloned() + .collect(); + if !displaced_authorization.is_empty() { + request.headers_mut().remove(reqwest::header::AUTHORIZATION); + for mut value in displaced_authorization { + value.set_sensitive(true); + request.headers_mut().append(RELOCATED_AUTHORIZATION, value); + } + } + + // Relocated after signing, as Java does, so the `Original-` copy is + // not itself signed. + let displaced_content_hash: Vec<_> = request + .headers() + .get_all("x-amz-content-sha256") + .iter() + .filter(|v| v.as_bytes() != content_header.as_bytes()) + .cloned() + .collect(); + request + .headers_mut() + .insert("x-amz-content-sha256", content_header.parse().unwrap()); + + // The wire Host never carries userinfo, so signing it would mismatch + // and would feed the password to the HMAC. `RequestBuilder` strips it, + // a hand-built request does not. + if !request.url().username().is_empty() || request.url().password().is_some() { + let url = request.url_mut(); + let _ = url.set_username(""); + let _ = url.set_password(None); + } + + // Verifiers disagree on `+` in a query — a literal under RFC 3986, a + // space under form encoding — and reqwest writes spaces as `+`. Rewrite + // it so the wire and the canonical request agree either way. + if let Some(query) = request.url().query().filter(|q| q.contains('+')) { + let unambiguous = query.replace('+', "%20"); + request.url_mut().set_query(Some(&unambiguous)); + } + + let mut settings = SigningSettings::default(); + // `Aws4Signer` defaults: normalize the path and double-encode it. + settings.percent_encoding_mode = PercentEncodingMode::Double; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; + // The header is ours to set: IcebergRest mode puts base64 there, + // while the signer hashes the body in hex for the canonical request. + settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; + // Java's `AbstractAws4Signer` ignores these too; the crate's defaults + // do not. + let mut excluded = settings.excluded_headers.take().unwrap_or_default(); + excluded.extend([ + "expect".into(), + "connection".into(), + "x-forwarded-for".into(), + ]); + settings.excluded_headers = Some(excluded); + + let identity = aws_credential_types::Credentials::new( + self.credentials.access_key_id.clone(), + self.credentials.secret_access_key.expose().to_string(), + self.credentials + .session_token + .as_ref() + .map(|t| t.expose().to_string()), + None, + "iceberg-rest", + ) + .into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.service) + .time(now.into()) + .settings(settings) + .build() + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "failed to build SigV4 params").with_source(e) + })? + .into(); + + let headers: Vec<(&str, &str)> = request + .headers() + .iter() + .filter_map(|(n, v)| v.to_str().ok().map(|v| (n.as_str(), v))) + .collect(); + let signable = SignableRequest::new( + request.method().as_str(), + request.url().as_str(), + headers.into_iter(), + SignableBody::Bytes(&body), + ) + .map_err(|e| { + Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) + })?; + + let (instructions, _signature) = sign(signable, ¶ms) + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? + .into_parts(); + + let (signed_headers, _params) = instructions.into_parts(); + let h = request.headers_mut(); + for mut value in displaced_content_hash { + // The original may carry a credential. + value.set_sensitive(true); + h.append(RELOCATED_CONTENT_SHA256, value); + } + for header in signed_headers { + let name: reqwest::header::HeaderName = header.name().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header name").with_source(e) + })?; + // Moved aside rather than dropped, as Java does. + if let Some(relocated) = relocated_name(name.as_str()) { + relocate_conflicting(h, name.as_str(), header.value(), relocated); + } + let mut value: reqwest::header::HeaderValue = header.value().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header value").with_source(e) + })?; + if name == reqwest::header::AUTHORIZATION || name == "x-amz-security-token" { + value.set_sensitive(true); + } + h.insert(name, value); + } + Ok(()) + } +} + +const RELOCATED_AUTHORIZATION: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-authorization"); +const RELOCATED_AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-date"); +const RELOCATED_CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-content-sha256"); +const RELOCATED_SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-security-token"); + +/// The `Original-` counterpart of a header the signer generates. +fn relocated_name(name: &str) -> Option { + match name { + "x-amz-date" => Some(RELOCATED_AMZ_DATE), + "x-amz-content-sha256" => Some(RELOCATED_CONTENT_SHA256), + "x-amz-security-token" => Some(RELOCATED_SECURITY_TOKEN), + _ => None, + } +} + +/// Moves `name`'s current values to `Original-` when they differ from the +/// value about to be signed, so a caller's header is not silently dropped +/// (Java's `RESTSigV4AuthSession.updateRequestHeaders`). +fn relocate_conflicting( + headers: &mut reqwest::header::HeaderMap, + name: &str, + signed: &str, + relocated: reqwest::header::HeaderName, +) { + let conflicting: Vec<_> = headers + .get_all(name) + .iter() + .filter(|value| value.as_bytes() != signed.as_bytes()) + .cloned() + .collect(); + for mut value in conflicting { + // The original may carry a credential (e.g. a session token). + value.set_sensitive(true); + headers.append(relocated.clone(), value); + } +} + +impl std::fmt::Debug for SigV4Signer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SigV4Signer") + .field("region", &self.region) + .field("service", &self.service) + .field("mode", &self.mode) + .field("access_key_id", &self.credentials.access_key_id) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const EMPTY_HEX: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + #[test] + fn signing_rewrites_an_ambiguous_plus_out_of_the_query() { + use chrono::TimeZone; + + // reqwest writes a space as `+`, which verifiers read either as a + // literal plus or as a space. Signing rewrites it to `%20`. + let mut request = reqwest::Client::new() + .get("https://rest.example.com/v1/namespaces") + .query(&[("parent", "my ns")]) + .build() + .unwrap(); + assert!(request.url().query().unwrap().contains("my+ns")); + + let signer = SigV4Signer::new( + AwsCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string().into(), + session_token: None, + }, + "us-east-1".to_string(), + "execute-api".to_string(), + PayloadHashMode::StandardAws, + ); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + signer.sign_at(&mut request, now).unwrap(); + + // The request that goes out no longer carries the ambiguous form. + let query = request.url().query().unwrap(); + assert!(!query.contains('+'), "{query}"); + assert!(query.contains("my%20ns"), "{query}"); + assert_signature_is( + &request, + "b7bb5a323a1ce0ace18454171084deef2dac44c933c3949771fc70179d3cce2b", + ); + } + + #[test] + fn content_sha256_header_iceberg_mode() { + let v = content_sha256_header(b"hello", PayloadHashMode::IcebergRest); + assert_eq!(v, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="); + let e = content_sha256_header(b"", PayloadHashMode::IcebergRest); + assert_eq!(e, EMPTY_HEX); + } + + #[test] + fn content_sha256_header_standard_mode() { + let v = content_sha256_header(b"hello", PayloadHashMode::StandardAws); + assert_eq!( + v, + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + } + + /// The signature the previous hand-rolled signer produced for this case, + /// pinned so a change in canonicalization is caught. + fn assert_signature_is(req: &reqwest::Request, expected: &str) { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!(auth.ends_with(&format!("Signature={expected}")), "{auth}"); + } + + fn test_signer(mode: PayloadHashMode) -> SigV4Signer { + SigV4Signer::new( + AwsCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string().into(), + session_token: None, + }, + "us-east-1".to_string(), + "execute-api".to_string(), + mode, + ) + } + + #[test] + fn userinfo_is_stripped_before_signing() { + use chrono::TimeZone; + + // `HttpRequest::new` is public, so a hand-built request can carry + // userinfo that the wire Host never has. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config" + .parse() + .unwrap(), + ); + assert_eq!(req.url().username(), "user"); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + + assert_eq!(req.url().username(), ""); + assert_eq!(req.url().password(), None); + assert_signature_is( + &req, + "0f4a3487bcff9dd16bf0a42d06c24dc49b2366e8a928dc9666f8424cf5b306b3", + ); + } + + #[test] + fn a_doubled_slash_in_the_path_is_normalized() { + use chrono::TimeZone; + + // A catalog URI with a trailing slash produces `//v1/...`; the signed + // path has to collapse it the way `Aws4Signer` does. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com//v1//config".parse().unwrap(), + ); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + + assert_signature_is( + &req, + "0f4a3487bcff9dd16bf0a42d06c24dc49b2366e8a928dc9666f8424cf5b306b3", + ); + } + + #[test] + fn caller_headers_the_signer_overwrites_are_relocated() { + use chrono::TimeZone; + + // Java's `updateRequestHeaders` moves a conflicting caller value to + // `Original-` rather than dropping it, credentials included. + let signer = SigV4Signer::new( + AwsCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string().into(), + session_token: Some("signer-token".to_string().into()), + }, + "us-east-1".to_string(), + "execute-api".to_string(), + PayloadHashMode::StandardAws, + ); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("x-amz-date", "19700101T000000Z") + .header("x-amz-security-token", "caller-session") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(); + + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + let h = req.headers(); + assert_eq!( + h.get("original-authorization").unwrap(), + "Bearer caller-token" + ); + assert_eq!(h.get("original-x-amz-date").unwrap(), "19700101T000000Z"); + assert_eq!( + h.get("original-x-amz-content-sha256").unwrap(), + "caller-hash" + ); + let token = h.get("original-x-amz-security-token").unwrap(); + assert_eq!(token, "caller-session"); + // Relocated originals may be credentials themselves. + assert!(token.is_sensitive()); + assert!( + h.get("original-x-amz-content-sha256") + .unwrap() + .is_sensitive() + ); + // And the signer's own values took their place. + assert!( + h.get("authorization") + .unwrap() + .to_str() + .unwrap() + .starts_with("AWS4-HMAC-SHA256 ") + ); + assert_eq!(h.get("x-amz-security-token").unwrap(), "signer-token"); + } + + #[test] + fn an_existing_authorization_is_never_signed() { + use chrono::TimeZone; + + // `authorization` must stay out of `SignedHeaders`: the signer replaces + // it, so signing the caller's value would guarantee a mismatch. The + // crate's own defaults carry that exclusion. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("user-agent", "example/1.0") + .build() + .unwrap(); + + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth + .split("SignedHeaders=") + .nth(1) + .unwrap() + .split(',') + .next() + .unwrap(); + for excluded in ["authorization", "user-agent"] { + assert!(!signed.split(';').any(|h| h == excluded), "{signed}"); + } + } + + #[test] + fn hop_by_hop_headers_are_not_signed() { + use chrono::TimeZone; + + // A proxy or an HTTP/2 hop may drop or rewrite these, so signing them + // would make the request fail verification. Java's `AbstractAws4Signer` + // ignores them too. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("expect", "100-continue") + .header("connection", "keep-alive") + .header("x-forwarded-for", "203.0.113.7") + .header("x-tenant", "acme") + .build() + .unwrap(); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth + .split("SignedHeaders=") + .nth(1) + .unwrap() + .split(',') + .next() + .unwrap(); + for skipped in ["expect", "connection", "x-forwarded-for"] { + assert!(!signed.split(';').any(|h| h == skipped), "{signed}"); + } + // An ordinary caller header is still signed. + assert!(signed.split(';').any(|h| h == "x-tenant"), "{signed}"); + assert_signature_is( + &req, + "f938221412ed6b55cf3db380ce6ded476419ad3d7db4c76d932031c98465ce79", + ); + } + + #[test] + fn signed_credentials_are_marked_sensitive() { + use chrono::TimeZone; + + // Both carry a credential, so a `Debug`-formatted request must not + // print them. + let signer = SigV4Signer::new( + AwsCredentials { + access_key_id: "ak".to_string(), + secret_access_key: "sk".to_string().into(), + session_token: Some("session-token".to_string().into()), + }, + "us-east-1".to_string(), + "execute-api".to_string(), + PayloadHashMode::StandardAws, + ); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(); + + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + assert!(req.headers().get("authorization").unwrap().is_sensitive()); + assert!( + req.headers() + .get("x-amz-security-token") + .unwrap() + .is_sensitive() + ); + let debug = format!("{req:?}"); + assert!(!debug.contains("session-token"), "{debug}"); + } + + #[test] + fn a_caller_content_hash_is_relocated_not_dropped() { + use chrono::TimeZone; + + // The signer overwrites `x-amz-content-sha256`; the caller's value + // moves aside instead of vanishing, after signing as Java does. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("x-amz-content-sha256", "caller-supplied") + .build() + .unwrap(); + + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + assert_eq!( + req.headers().get("original-x-amz-content-sha256").unwrap(), + "caller-supplied" + ); + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + EMPTY_HEX + ); + } + + #[test] + fn signs_with_a_non_default_service_and_session_token() { + use chrono::TimeZone; + + // What a non-AWS S3-compatible catalog vends: its own signing name + // rather than `execute-api`, its own region, and STS credentials. + let signer = SigV4Signer::new( + AwsCredentials { + access_key_id: "STS.EXAMPLEACCESSKEYID".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: Some("example-session-token".to_string().into()), + }, + "us-east-1".into(), + "custom-service".into(), + PayloadHashMode::IcebergRest, + ); + let mut req = reqwest::Client::new() + .get("https://catalog.example.com/v1/config?warehouse=my-catalog") + .build() + .unwrap(); + let now = Utc.with_ymd_and_hms(2026, 8, 26, 12, 0, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + + assert_signature_is( + &req, + "6b7065e5f44da4f5c3654126b8d5fe29599905afb6b98f4de65b6ec6e1be783f", + ); + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!( + auth.contains("/us-east-1/custom-service/aws4_request"), + "{auth}" + ); + assert_eq!( + req.headers().get("x-amz-security-token").unwrap(), + "example-session-token" + ); + } + + #[test] + fn signing_key_and_signature_match_aws_vector() { + let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + let date = "20150830"; + let region = "us-east-1"; + let service = "service"; + let key = signing_key(secret, date, region, service); + + let string_to_sign = "AWS4-HMAC-SHA256\n\ +20150830T123600Z\n\ +20150830/us-east-1/service/aws4_request\n\ +bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; + let sig = hex_hmac_sha256(&key, string_to_sign.as_bytes()); + assert_eq!( + sig, + "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" + ); + } + + #[test] + fn signs_request_iceberg_mode() { + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: Some("SESSIONTOKEN".to_string().into()), + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "glue".into(), + PayloadHashMode::IcebergRest, + ); + let client = reqwest::Client::new(); + let mut req = client + .post("https://rest.example.com/v1/namespaces") + .body("{}") + .build() + .unwrap(); + + signer.sign(&mut req).unwrap(); + + let h = req.headers(); + assert!( + h.get("authorization") + .unwrap() + .to_str() + .unwrap() + .starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") + ); + assert!(h.contains_key("x-amz-date")); + assert_eq!(h.get("x-amz-security-token").unwrap(), "SESSIONTOKEN"); + let csha = h.get("x-amz-content-sha256").unwrap().to_str().unwrap(); + assert_eq!(csha, "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o="); + } + + /// Empty body uses the hex constant and existing headers are signed too + /// (mirrors Java's `TestRESTSigV4AuthSession::authenticateWithoutBody`). + #[test] + fn signs_empty_body_and_all_headers() { + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: None, + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "glue".into(), + PayloadHashMode::IcebergRest, + ); + let client = reqwest::Client::new(); + let mut req = client + .get("https://rest.example.com/v1/config") + .header("content-type", "application/json") + .header("content-encoding", "gzip") + .build() + .unwrap(); + + signer.sign(&mut req).unwrap(); + + let h = req.headers(); + assert_eq!(h.get("x-amz-content-sha256").unwrap(), EMPTY_HEX); + assert!(!h.contains_key("x-amz-security-token")); + let auth = h.get("authorization").unwrap().to_str().unwrap(); + assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/")); + assert!(auth.contains( + "SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date" + )); + } + + /// The signed `host` must include an explicit non-default port, matching + /// what reqwest/hyper put on the wire and what the AWS SDK signs. + #[test] + fn iceberg_mode_signs_the_hex_payload_hash_not_the_base64_header() { + // The IcebergRest split: `x-amz-content-sha256` carries base64, but the + // canonical request must hash in hex. A body is required to tell them + // apart — every other signing test uses an empty one, where the header + // is the hex constant and the two values coincide. + // + // Java has no counterpart: there the split lives inside the AWS SDK + // (`SignerChecksumParams` puts a base64 checksum in the header while + // `Aws4Signer` canonicalizes hex), so `TestRESTSigV4Signer` only checks + // that the header is present. Reimplementing the signer makes the + // invariant ours to keep. + use chrono::TimeZone; + + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: None, + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "execute-api".into(), + PayloadHashMode::IcebergRest, + ); + let body = br#"{"namespace":["ns"]}"#; + let mut req = reqwest::Client::new() + .post("https://rest.example.com/v1/namespaces") + .body(body.to_vec()) + .build() + .unwrap(); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + + // The header carries base64 while the pinned signature covers hex. + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + content_sha256_header(body, PayloadHashMode::IcebergRest).as_str() + ); + assert_ne!( + req.headers().get("x-amz-content-sha256").unwrap(), + hex_sha256(body).as_str() + ); + assert_signature_is( + &req, + "c68682c26cab6a781256f83b0076f50014f4922c3907f4ff09c204a74d61fc1d", + ); + } + + #[test] + fn signs_host_with_non_default_port() { + use chrono::TimeZone; + + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: None, + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "glue".into(), + PayloadHashMode::IcebergRest, + ); + let client = reqwest::Client::new(); + let mut req = client + .get("https://rest.example.com:8181/v1/config") + .build() + .unwrap(); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + assert_signature_is( + &req, + "f7801a4ecac5fe6dcc4ee385223428ec4833f21d0c2fc10d2fb00694b2c0def7", + ); + } + + /// AWS SDK v2 parity (`doubleUrlEncode`): the canonical URI encodes the + /// serialized path once more — literal `,` becomes `%2C`, an encoded + /// `%2C` becomes `%252C` — while plain paths stay byte-identical. + #[test] + fn canonical_uri_is_aws_double_encoded() { + use chrono::TimeZone; + + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: None, + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "glue".into(), + PayloadHashMode::IcebergRest, + ); + let client = reqwest::Client::new(); + let mut req = client + .get("https://rest.example.com/v1/namespaces/a%2Cb/tables/x,y") + .build() + .unwrap(); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + + signer.sign_at(&mut req, now).unwrap(); + assert_signature_is( + &req, + "4d966b6fc2dfb62be5a603e4e07e5dc85b2af1c7e6a181c5646bfbf38ddfa543", + ); + } + + #[test] + fn signs_request_standard_mode_uses_hex_header() { + let creds = AwsCredentials { + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + .to_string() + .into(), + session_token: None, + }; + let signer = SigV4Signer::new( + creds, + "us-east-1".into(), + "glue".into(), + PayloadHashMode::StandardAws, + ); + let client = reqwest::Client::new(); + let mut req = client + .post("https://rest.example.com/v1/namespaces") + .body("hello") + .build() + .unwrap(); + + signer.sign(&mut req).unwrap(); + + // StandardAws keeps the header in hex. + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + } +} From 1ad8e0ebb9e7fee1ea2fa0d9fdfa02b79858e5e8 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 27 Aug 2026 02:15:06 -0400 Subject: [PATCH 02/30] fix(rest): mute aws-sigv4 tracing and tighten payload hashing The crate traces the request it signs, and its redaction list covers `authorization` but not the `Original-` copy we make, so a delegate's bearer token could reach trace logs. Also reject non-UTF-8 headers rather than leave them unsigned, and hash a present-but-empty body as Java does. --- crates/catalog/rest/src/auth/sigv4/signer.rs | 224 ++++++++++++++++--- 1 file changed, 189 insertions(+), 35 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4/signer.rs b/crates/catalog/rest/src/auth/sigv4/signer.rs index de4c811186..31f3f06907 100644 --- a/crates/catalog/rest/src/auth/sigv4/signer.rs +++ b/crates/catalog/rest/src/auth/sigv4/signer.rs @@ -29,8 +29,8 @@ const EMPTY_BODY_HEX_SHA256: &str = /// How the payload hash is encoded in the `x-amz-content-sha256` header. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PayloadHashMode { - /// Iceberg Java's RESTSigV4 style: base64 header for non-empty bodies, hex - /// for empty; the canonical request always uses hex. + /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex + /// when there is none; the canonical request always uses hex. IcebergRest, /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). StandardAws, @@ -69,17 +69,15 @@ fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec hmac_sha256(&k_service, b"aws4_request") } -/// Computes the value of the `x-amz-content-sha256` header. -fn content_sha256_header(body: &[u8], mode: PayloadHashMode) -> String { +/// Computes the value of the `x-amz-content-sha256` header. `None` is a request +/// with no body at all, which the two modes disagree about. +fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { match mode { - PayloadHashMode::StandardAws => hex_sha256(body), - PayloadHashMode::IcebergRest => { - if body.is_empty() { - EMPTY_BODY_HEX_SHA256.to_string() - } else { - base64_encode(&Sha256::digest(body)) - } - } + PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), + PayloadHashMode::IcebergRest => match body { + None => EMPTY_BODY_HEX_SHA256.to_string(), + Some(body) => base64_encode(&Sha256::digest(body)), + }, } } @@ -129,8 +127,8 @@ impl SigV4Signer { /// `RequestBuilder::query` writes; encode a literal plus as `%2B`. AWS /// requires spaces as `%20` in a signed URL either way. /// - /// Note that `aws_sigv4` traces the request it is given, so `RUST_LOG` at - /// trace level will print these headers. + /// Fails rather than sign a request whose body is streaming or whose + /// headers are not UTF-8, since neither can be canonicalized faithfully. pub fn sign(&self, request: &mut reqwest::Request) -> Result<()> { self.sign_at(request, Utc::now()) } @@ -141,20 +139,24 @@ impl SigV4Signer { SigningSettings, UriPathNormalizationMode, sign, }; use aws_sigv4::sign::v4; - - let body: Vec = match request.body() { - None => Vec::new(), - Some(b) => b - .as_bytes() - .ok_or_else(|| { - Error::new( - ErrorKind::FeatureUnsupported, - "cannot sign a streaming request body", - ) - })? - .to_vec(), + use tracing::subscriber::NoSubscriber; + + // An absent body is not the same as an empty one: Java branches on + // `encodedBody() == null`, so a present-but-empty body is hashed. + let body: Option> = match request.body() { + None => None, + Some(b) => Some( + b.as_bytes() + .ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + ) + })? + .to_vec(), + ), }; - let content_header = content_sha256_header(&body, self.mode); + let content_header = content_sha256_header(body.as_deref(), self.mode); // Renamed before signing, as Java's `convertHeaders` does, so a // delegate's bearer token travels along and is signed over. @@ -242,22 +244,40 @@ impl SigV4Signer { })? .into(); + // Skipping a header that is not UTF-8 would leave it unsigned but still + // on the wire, which AWS rejects for `x-amz-*` and is hard to diagnose. let headers: Vec<(&str, &str)> = request .headers() .iter() - .filter_map(|(n, v)| v.to_str().ok().map(|v| (n.as_str(), v))) - .collect(); + .map(|(n, v)| { + let v = v.to_str().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("cannot sign non-UTF-8 header value for `{n}`"), + ) + .with_source(e) + })?; + Ok((n.as_str(), v)) + }) + .collect::>()?; let signable = SignableRequest::new( request.method().as_str(), request.url().as_str(), headers.into_iter(), - SignableBody::Bytes(&body), + SignableBody::Bytes(body.as_deref().unwrap_or_default()), ) .map_err(|e| { Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) })?; - let (instructions, _signature) = sign(signable, ¶ms) + // `aws_sigv4` traces the request it signs, and its redaction list covers + // `authorization` but not the `Original-` copy we just made, so a + // delegate's bearer token would be printed verbatim at trace level. + // Signing is synchronous and logs nothing else worth keeping, so mute + // it for the call. + let signed = + tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)); + let (instructions, _signature) = signed .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? .into_parts(); @@ -384,15 +404,27 @@ mod tests { #[test] fn content_sha256_header_iceberg_mode() { - let v = content_sha256_header(b"hello", PayloadHashMode::IcebergRest); + let v = content_sha256_header(Some(b"hello"), PayloadHashMode::IcebergRest); assert_eq!(v, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="); - let e = content_sha256_header(b"", PayloadHashMode::IcebergRest); + let e = content_sha256_header(None, PayloadHashMode::IcebergRest); assert_eq!(e, EMPTY_HEX); } + /// Java branches on `encodedBody() == null`, so a body that is present but + /// empty is hashed like any other rather than taking the absent-body path. + #[test] + fn content_sha256_header_separates_an_empty_body_from_an_absent_one() { + let empty = content_sha256_header(Some(b""), PayloadHashMode::IcebergRest); + assert_eq!(empty, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="); + assert_ne!( + empty, + content_sha256_header(None, PayloadHashMode::IcebergRest) + ); + } + #[test] fn content_sha256_header_standard_mode() { - let v = content_sha256_header(b"hello", PayloadHashMode::StandardAws); + let v = content_sha256_header(Some(b"hello"), PayloadHashMode::StandardAws); assert_eq!( v, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" @@ -424,6 +456,90 @@ mod tests { ) } + /// Collects every event field a subscriber would have been handed. + #[derive(Clone, Default)] + struct CapturedLog(std::sync::Arc>); + + impl tracing::field::Visit for CapturedLog { + fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.lock().unwrap().push_str(&format!("{value:?}")); + } + } + + impl tracing::Subscriber for CapturedLog { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::Id { + tracing::Id::from_u64(1) + } + fn record(&self, _: &tracing::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::Id, _: &tracing::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + event.record(&mut self.clone()); + } + fn enter(&self, _: &tracing::Id) {} + fn exit(&self, _: &tracing::Id) {} + } + + /// `aws_sigv4` traces the headers it is given, and its redaction list does + /// not cover the `Original-` copy of a relocated bearer token. + #[test] + fn signing_does_not_trace_a_relocated_bearer_token() { + use chrono::TimeZone; + + const TOKEN: &str = "Bearer topsecretdelegatetoken"; + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header(reqwest::header::AUTHORIZATION, TOKEN) + .build() + .unwrap(); + + let log = CapturedLog::default(); + tracing::subscriber::with_default(log.clone(), || { + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + // Without this the assertion below would pass even if nothing was + // ever captured. + tracing::trace!(canary = "subscriber-is-live"); + }); + + let captured = log.0.lock().unwrap().clone(); + assert!(captured.contains("subscriber-is-live"), "captured nothing"); + assert!(!captured.contains(TOKEN), "{captured}"); + // The token still travels, it is just not logged. + assert_eq!(req.headers().get(RELOCATED_AUTHORIZATION).unwrap(), TOKEN); + } + + #[test] + fn a_non_utf8_header_value_is_rejected_rather_than_left_unsigned() { + use chrono::TimeZone; + + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header( + "x-amz-meta-tenant", + reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), + ) + .build() + .unwrap(); + + let err = signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("x-amz-meta-tenant"), "{err}"); + } + #[test] fn userinfo_is_stripped_before_signing() { use chrono::TimeZone; @@ -571,6 +687,44 @@ mod tests { for excluded in ["authorization", "user-agent"] { assert!(!signed.split(';').any(|h| h == excluded), "{signed}"); } + // The relocated copy, on the other hand, is signed over — that is the + // point of renaming it before signing rather than after. + assert!( + signed.split(';').any(|h| h == "original-authorization"), + "{signed}" + ); + } + + /// Java groups all `Authorization` values under the relocated name, so + /// repeated credentials must survive together and stay redacted. + #[test] + fn every_repeated_authorization_is_relocated_and_kept_sensitive() { + use chrono::TimeZone; + + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer first") + .header("authorization", "Bearer second") + .build() + .unwrap(); + + signer + .sign_at( + &mut req, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + let relocated: Vec<_> = req + .headers() + .get_all(RELOCATED_AUTHORIZATION) + .iter() + .collect(); + assert_eq!(relocated.len(), 2, "{relocated:?}"); + assert_eq!(relocated[0], "Bearer first"); + assert_eq!(relocated[1], "Bearer second"); + assert!(relocated.iter().all(|v| v.is_sensitive()), "{relocated:?}"); } #[test] @@ -868,7 +1022,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; // The header carries base64 while the pinned signature covers hex. assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), - content_sha256_header(body, PayloadHashMode::IcebergRest).as_str() + content_sha256_header(Some(body), PayloadHashMode::IcebergRest).as_str() ); assert_ne!( req.headers().get("x-amz-content-sha256").unwrap(), From 53e5c903c174f381856ed151021d2cf128cae2a3 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 27 Aug 2026 23:55:53 -0400 Subject: [PATCH 03/30] refactor(rest): take credentials per call, as Java's signer does Java holds one `Aws4Signer` across sessions and resolves credentials from an `AwsCredentialsProvider` per request, so the signer itself carries no credential state. Follow that: drop `AwsCredentials`, take the AWS crate's `Credentials` as a `sign` argument, and leave the provider to the auth session. Also sign `HttpRequest` rather than the concrete request type, name the amz headers, split `convert_headers`/`update_request_headers` after their Java counterparts, collapse the one-file `sigv4` module, and put the AWS dependencies behind a `sigv4` feature. --- crates/catalog/rest/Cargo.toml | 12 +- crates/catalog/rest/public-api.txt | 10 +- crates/catalog/rest/src/auth/mod.rs | 4 +- .../src/auth/{sigv4/signer.rs => sigv4.rs} | 640 ++++++++++-------- crates/catalog/rest/src/auth/sigv4/mod.rs | 22 - crates/catalog/rest/src/request.rs | 15 + 6 files changed, 379 insertions(+), 324 deletions(-) rename crates/catalog/rest/src/auth/{sigv4/signer.rs => sigv4.rs} (69%) delete mode 100644 crates/catalog/rest/src/auth/sigv4/mod.rs diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 06477503e2..d395a9adf8 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -29,10 +29,16 @@ keywords = ["iceberg", "rest", "catalog"] license = { workspace = true } repository = { workspace = true } +[features] +default = [] +# SigV4 request signing, for catalogs that authenticate the AWS way. Off by +# default so the AWS crates are not pulled in unless they are used. +sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:sha2"] + [dependencies] async-trait = { workspace = true } -aws-credential-types = { workspace = true } -aws-sigv4 = { workspace = true } +aws-credential-types = { workspace = true, optional = true } +aws-sigv4 = { workspace = true, optional = true } base64 = { workspace = true } chrono = { workspace = true } http = { workspace = true } @@ -42,7 +48,7 @@ reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } -sha2 = { workspace = true } +sha2 = { workspace = true, optional = true } tokio = { workspace = true } tracing = { workspace = true } typed-builder = { workspace = true } diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 0cf8a25fee..98502cef9d 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -26,12 +26,6 @@ impl core::fmt::Debug for iceberg_catalog_rest::PayloadHashMode pub fn iceberg_catalog_rest::PayloadHashMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::Copy for iceberg_catalog_rest::PayloadHashMode impl core::marker::StructuralPartialEq for iceberg_catalog_rest::PayloadHashMode -pub struct iceberg_catalog_rest::AwsCredentials -pub iceberg_catalog_rest::AwsCredentials::access_key_id: alloc::string::String -pub iceberg_catalog_rest::AwsCredentials::secret_access_key: iceberg::sensitive::SensitiveString -pub iceberg_catalog_rest::AwsCredentials::session_token: core::option::Option -impl core::clone::Clone for iceberg_catalog_rest::AwsCredentials -pub fn iceberg_catalog_rest::AwsCredentials::clone(&self) -> iceberg_catalog_rest::AwsCredentials pub struct iceberg_catalog_rest::CommitTableRequest pub iceberg_catalog_rest::CommitTableRequest::identifier: core::option::Option pub iceberg_catalog_rest::CommitTableRequest::requirements: alloc::vec::Vec @@ -351,8 +345,8 @@ impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_catalog_rest::SigV4Signer impl iceberg_catalog_rest::SigV4Signer -pub fn iceberg_catalog_rest::SigV4Signer::new(credentials: iceberg_catalog_rest::AwsCredentials, region: alloc::string::String, service: alloc::string::String, mode: iceberg_catalog_rest::PayloadHashMode) -> Self -pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut reqwest::async_impl::request::Request) -> iceberg::error::Result<()> +pub fn iceberg_catalog_rest::SigV4Signer::new(region: alloc::string::String, service: alloc::string::String, mode: iceberg_catalog_rest::PayloadHashMode) -> Self +pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index b601a4066e..cc6910483a 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -19,6 +19,7 @@ //! `AuthManager`/`AuthSession` API. mod oauth2; +#[cfg(feature = "sigv4")] mod sigv4; use std::collections::HashMap; @@ -28,7 +29,8 @@ use std::sync::Arc; use async_trait::async_trait; use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; -pub use sigv4::{AwsCredentials, PayloadHashMode, SigV4Signer}; +#[cfg(feature = "sigv4")] +pub use sigv4::{PayloadHashMode, SigV4Signer}; use crate::client::HttpClient; use crate::request::HttpRequest; diff --git a/crates/catalog/rest/src/auth/sigv4/signer.rs b/crates/catalog/rest/src/auth/sigv4.rs similarity index 69% rename from crates/catalog/rest/src/auth/sigv4/signer.rs rename to crates/catalog/rest/src/auth/sigv4.rs index 31f3f06907..3a592ac7eb 100644 --- a/crates/catalog/rest/src/auth/sigv4/signer.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. +//! AWS SigV4 request signing for the REST catalog. + use chrono::{DateTime, Utc}; #[cfg(test)] use hmac::{Hmac, Mac}; -use iceberg::sensitive::SensitiveString; use iceberg::{Error, ErrorKind, Result}; use sha2::{Digest, Sha256}; @@ -81,22 +82,10 @@ fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { } } -/// Static AWS-style credentials used for SigV4 signing of catalog requests. -#[derive(Clone)] -pub struct AwsCredentials { - /// AWS access key id. - pub access_key_id: String, - /// AWS secret access key. - pub secret_access_key: SensitiveString, - /// Optional STS session token. - pub session_token: Option, -} - /// AWS SigV4 signer following Iceberg Java's `RESTSigV4AuthSession`: it adds the /// required amz headers and signs all request headers except a small blacklist. #[derive(Clone)] pub struct SigV4Signer { - credentials: AwsCredentials, region: String, service: String, mode: PayloadHashMode, @@ -104,14 +93,8 @@ pub struct SigV4Signer { impl SigV4Signer { /// Creates a new SigV4 signer. - pub fn new( - credentials: AwsCredentials, - region: String, - service: String, - mode: PayloadHashMode, - ) -> Self { + pub fn new(region: String, service: String, mode: PayloadHashMode) -> Self { Self { - credentials, region, service, mode, @@ -129,11 +112,20 @@ impl SigV4Signer { /// /// Fails rather than sign a request whose body is streaming or whose /// headers are not UTF-8, since neither can be canonicalized faithfully. - pub fn sign(&self, request: &mut reqwest::Request) -> Result<()> { - self.sign_at(request, Utc::now()) + pub fn sign( + &self, + request: &mut crate::HttpRequest, + credentials: &aws_credential_types::Credentials, + ) -> Result<()> { + self.sign_at(request, credentials, Utc::now()) } - fn sign_at(&self, request: &mut reqwest::Request, now: DateTime) -> Result<()> { + fn sign_at( + &self, + request: &mut crate::HttpRequest, + credentials: &aws_credential_types::Credentials, + now: DateTime, + ) -> Result<()> { use aws_sigv4::http_request::{ PayloadChecksumKind, PercentEncodingMode, SignableBody, SignableRequest, SigningSettings, UriPathNormalizationMode, sign, @@ -141,51 +133,34 @@ impl SigV4Signer { use aws_sigv4::sign::v4; use tracing::subscriber::NoSubscriber; - // An absent body is not the same as an empty one: Java branches on - // `encodedBody() == null`, so a present-but-empty body is hashed. + // `HttpRequestBody` already separates absent from empty, which Java + // relies on: it branches on `encodedBody() == null`. let body: Option> = match request.body() { - None => None, - Some(b) => Some( - b.as_bytes() - .ok_or_else(|| { - Error::new( - ErrorKind::FeatureUnsupported, - "cannot sign a streaming request body", - ) - })? - .to_vec(), - ), + crate::HttpRequestBody::Empty => None, + crate::HttpRequestBody::Buffered(bytes) => Some(bytes.to_vec()), + crate::HttpRequestBody::Streaming => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + )); + } }; let content_header = content_sha256_header(body.as_deref(), self.mode); - // Renamed before signing, as Java's `convertHeaders` does, so a - // delegate's bearer token travels along and is signed over. - let displaced_authorization: Vec<_> = request - .headers() - .get_all(reqwest::header::AUTHORIZATION) - .iter() - .cloned() - .collect(); - if !displaced_authorization.is_empty() { - request.headers_mut().remove(reqwest::header::AUTHORIZATION); - for mut value in displaced_authorization { - value.set_sensitive(true); - request.headers_mut().append(RELOCATED_AUTHORIZATION, value); - } - } + convert_headers(request); // Relocated after signing, as Java does, so the `Original-` copy is // not itself signed. let displaced_content_hash: Vec<_> = request .headers() - .get_all("x-amz-content-sha256") + .get_all(CONTENT_SHA256) .iter() .filter(|v| v.as_bytes() != content_header.as_bytes()) .cloned() .collect(); request .headers_mut() - .insert("x-amz-content-sha256", content_header.parse().unwrap()); + .insert(CONTENT_SHA256, content_header.parse().unwrap()); // The wire Host never carries userinfo, so signing it would mismatch // and would feed the password to the HMAC. `RequestBuilder` strips it, @@ -221,17 +196,7 @@ impl SigV4Signer { ]); settings.excluded_headers = Some(excluded); - let identity = aws_credential_types::Credentials::new( - self.credentials.access_key_id.clone(), - self.credentials.secret_access_key.expose().to_string(), - self.credentials - .session_token - .as_ref() - .map(|t| t.expose().to_string()), - None, - "iceberg-rest", - ) - .into(); + let identity = credentials.clone().into(); let params = v4::SigningParams::builder() .identity(&identity) .region(&self.region) @@ -262,7 +227,7 @@ impl SigV4Signer { .collect::>()?; let signable = SignableRequest::new( request.method().as_str(), - request.url().as_str(), + request.url_str(), headers.into_iter(), SignableBody::Bytes(body.as_deref().unwrap_or_default()), ) @@ -281,33 +246,71 @@ impl SigV4Signer { .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? .into_parts(); - let (signed_headers, _params) = instructions.into_parts(); - let h = request.headers_mut(); - for mut value in displaced_content_hash { - // The original may carry a credential. - value.set_sensitive(true); - h.append(RELOCATED_CONTENT_SHA256, value); + update_request_headers(request, instructions, displaced_content_hash) + } +} + +/// Renames `Authorization` to `Original-Authorization` so SigV4 can take the +/// name, as Java's `convertHeaders` does. Runs *before* signing, so the +/// relocated copy is itself signed over. +fn convert_headers(request: &mut crate::HttpRequest) { + let displaced: Vec<_> = request + .headers() + .get_all(reqwest::header::AUTHORIZATION) + .iter() + .cloned() + .collect(); + if displaced.is_empty() { + return; + } + request.headers_mut().remove(reqwest::header::AUTHORIZATION); + for mut value in displaced { + value.set_sensitive(true); + request.headers_mut().append(RELOCATED_AUTHORIZATION, value); + } +} + +/// Installs the signed headers, moving any conflicting caller value aside +/// rather than dropping it, as Java's `updateRequestHeaders` does. Runs +/// *after* signing, so these `Original-` copies are not themselves signed. +fn update_request_headers( + request: &mut crate::HttpRequest, + instructions: aws_sigv4::http_request::SigningInstructions, + displaced_content_hash: Vec, +) -> Result<()> { + let (signed_headers, _params) = instructions.into_parts(); + let h = request.headers_mut(); + for mut value in displaced_content_hash { + // The original may carry a credential. + value.set_sensitive(true); + h.append(RELOCATED_CONTENT_SHA256, value); + } + for header in signed_headers { + let name: reqwest::header::HeaderName = header.name().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header name").with_source(e) + })?; + if let Some(relocated) = relocated_name(name.as_str()) { + relocate_conflicting(h, name.as_str(), header.value(), relocated); } - for header in signed_headers { - let name: reqwest::header::HeaderName = header.name().parse().map_err(|e| { - Error::new(ErrorKind::Unexpected, "invalid signed header name").with_source(e) - })?; - // Moved aside rather than dropped, as Java does. - if let Some(relocated) = relocated_name(name.as_str()) { - relocate_conflicting(h, name.as_str(), header.value(), relocated); - } - let mut value: reqwest::header::HeaderValue = header.value().parse().map_err(|e| { - Error::new(ErrorKind::Unexpected, "invalid signed header value").with_source(e) - })?; - if name == reqwest::header::AUTHORIZATION || name == "x-amz-security-token" { - value.set_sensitive(true); - } - h.insert(name, value); + let mut value: reqwest::header::HeaderValue = header.value().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header value").with_source(e) + })?; + if name == reqwest::header::AUTHORIZATION || name == SECURITY_TOKEN { + value.set_sensitive(true); } - Ok(()) + h.insert(name, value); } + Ok(()) } +/// The payload-hash header SigV4 sets and Java relocates on conflict. +const CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-content-sha256"); +const AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-date"); +const SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-security-token"); + const RELOCATED_AUTHORIZATION: reqwest::header::HeaderName = reqwest::header::HeaderName::from_static("original-authorization"); const RELOCATED_AMZ_DATE: reqwest::header::HeaderName = @@ -320,9 +323,9 @@ const RELOCATED_SECURITY_TOKEN: reqwest::header::HeaderName = /// The `Original-` counterpart of a header the signer generates. fn relocated_name(name: &str) -> Option { match name { - "x-amz-date" => Some(RELOCATED_AMZ_DATE), - "x-amz-content-sha256" => Some(RELOCATED_CONTENT_SHA256), - "x-amz-security-token" => Some(RELOCATED_SECURITY_TOKEN), + n if n == AMZ_DATE => Some(RELOCATED_AMZ_DATE), + n if n == CONTENT_SHA256 => Some(RELOCATED_CONTENT_SHA256), + n if n == SECURITY_TOKEN => Some(RELOCATED_SECURITY_TOKEN), _ => None, } } @@ -355,7 +358,6 @@ impl std::fmt::Debug for SigV4Signer { .field("region", &self.region) .field("service", &self.service) .field("mode", &self.mode) - .field("access_key_id", &self.credentials.access_key_id) .finish_non_exhaustive() } } @@ -363,6 +365,7 @@ impl std::fmt::Debug for SigV4Signer { #[cfg(test)] mod tests { use super::*; + use crate::HttpRequest; const EMPTY_HEX: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; @@ -372,25 +375,24 @@ mod tests { // reqwest writes a space as `+`, which verifiers read either as a // literal plus or as a space. Signing rewrites it to `%20`. - let mut request = reqwest::Client::new() - .get("https://rest.example.com/v1/namespaces") - .query(&[("parent", "my ns")]) - .build() - .unwrap(); + let mut request = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/namespaces") + .query(&[("parent", "my ns")]) + .build() + .unwrap(), + ); assert!(request.url().query().unwrap().contains("my+ns")); let signer = SigV4Signer::new( - AwsCredentials { - access_key_id: "ak".to_string(), - secret_access_key: "sk".to_string().into(), - session_token: None, - }, "us-east-1".to_string(), "execute-api".to_string(), PayloadHashMode::StandardAws, ); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut request, now).unwrap(); + signer + .sign_at(&mut request, &test_credentials(), now) + .unwrap(); // The request that goes out no longer carries the ambiguous form. let query = request.url().query().unwrap(); @@ -422,6 +424,34 @@ mod tests { ); } + /// The same distinction, but through `sign_at`, so that collapsing the two + /// while reading the body off the request cannot go unnoticed. + #[test] + fn signing_separates_an_empty_body_from_an_absent_one() { + use chrono::TimeZone; + + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let hash_of = |builder: reqwest::RequestBuilder| { + let mut req = HttpRequest::new(builder.build().unwrap()); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + req.headers() + .get("x-amz-content-sha256") + .unwrap() + .to_str() + .unwrap() + .to_string() + }; + + let client = reqwest::Client::new(); + let url = "https://rest.example.com/v1/namespaces"; + assert_eq!( + hash_of(client.post(url).body("")), + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + ); + assert_eq!(hash_of(client.post(url)), EMPTY_HEX); + } + #[test] fn content_sha256_header_standard_mode() { let v = content_sha256_header(Some(b"hello"), PayloadHashMode::StandardAws); @@ -433,7 +463,7 @@ mod tests { /// The signature the previous hand-rolled signer produced for this case, /// pinned so a change in canonicalization is caught. - fn assert_signature_is(req: &reqwest::Request, expected: &str) { + fn assert_signature_is(req: &HttpRequest, expected: &str) { let auth = req .headers() .get("authorization") @@ -444,16 +474,11 @@ mod tests { } fn test_signer(mode: PayloadHashMode) -> SigV4Signer { - SigV4Signer::new( - AwsCredentials { - access_key_id: "ak".to_string(), - secret_access_key: "sk".to_string().into(), - session_token: None, - }, - "us-east-1".to_string(), - "execute-api".to_string(), - mode, - ) + SigV4Signer::new("us-east-1".to_string(), "execute-api".to_string(), mode) + } + + fn test_credentials() -> aws_credential_types::Credentials { + aws_credential_types::Credentials::new("ak", "sk", None::, None, "test") } /// Collects every event field a subscriber would have been handed. @@ -490,17 +515,20 @@ mod tests { const TOKEN: &str = "Bearer topsecretdelegatetoken"; let signer = test_signer(PayloadHashMode::IcebergRest); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header(reqwest::header::AUTHORIZATION, TOKEN) - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header(reqwest::header::AUTHORIZATION, TOKEN) + .build() + .unwrap(), + ); let log = CapturedLog::default(); tracing::subscriber::with_default(log.clone(), || { signer .sign_at( &mut req, + &test_credentials(), Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -521,18 +549,21 @@ mod tests { use chrono::TimeZone; let signer = test_signer(PayloadHashMode::IcebergRest); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header( - "x-amz-meta-tenant", - reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), - ) - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header( + "x-amz-meta-tenant", + reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), + ) + .build() + .unwrap(), + ); let err = signer .sign_at( &mut req, + &test_credentials(), Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap_err(); @@ -547,16 +578,16 @@ mod tests { // `HttpRequest::new` is public, so a hand-built request can carry // userinfo that the wire Host never has. let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Request::new( + let mut req = HttpRequest::new(reqwest::Request::new( reqwest::Method::GET, "https://user:pw@rest.example.com/v1/config" .parse() .unwrap(), - ); + )); assert_eq!(req.url().username(), "user"); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); assert_eq!(req.url().username(), ""); assert_eq!(req.url().password(), None); @@ -573,13 +604,13 @@ mod tests { // A catalog URI with a trailing slash produces `//v1/...`; the signed // path has to collapse it the way `Aws4Signer` does. let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Request::new( + let mut req = HttpRequest::new(reqwest::Request::new( reqwest::Method::GET, "https://rest.example.com//v1//config".parse().unwrap(), - ); + )); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); assert_signature_is( &req, @@ -593,28 +624,33 @@ mod tests { // Java's `updateRequestHeaders` moves a conflicting caller value to // `Original-` rather than dropping it, credentials included. + let creds = aws_credential_types::Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("signer-token".to_string()), + None, + "test", + ); let signer = SigV4Signer::new( - AwsCredentials { - access_key_id: "ak".to_string(), - secret_access_key: "sk".to_string().into(), - session_token: Some("signer-token".to_string().into()), - }, "us-east-1".to_string(), "execute-api".to_string(), PayloadHashMode::StandardAws, ); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("authorization", "Bearer caller-token") - .header("x-amz-date", "19700101T000000Z") - .header("x-amz-security-token", "caller-session") - .header("x-amz-content-sha256", "caller-hash") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("x-amz-date", "19700101T000000Z") + .header("x-amz-security-token", "caller-session") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); signer .sign_at( &mut req, + &creds, Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -657,16 +693,19 @@ mod tests { // it, so signing the caller's value would guarantee a mismatch. The // crate's own defaults carry that exclusion. let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("authorization", "Bearer caller-token") - .header("user-agent", "example/1.0") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("user-agent", "example/1.0") + .build() + .unwrap(), + ); signer .sign_at( &mut req, + &test_credentials(), Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -702,16 +741,19 @@ mod tests { use chrono::TimeZone; let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("authorization", "Bearer first") - .header("authorization", "Bearer second") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer first") + .header("authorization", "Bearer second") + .build() + .unwrap(), + ); signer .sign_at( &mut req, + &test_credentials(), Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -735,17 +777,19 @@ mod tests { // would make the request fail verification. Java's `AbstractAws4Signer` // ignores them too. let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("expect", "100-continue") - .header("connection", "keep-alive") - .header("x-forwarded-for", "203.0.113.7") - .header("x-tenant", "acme") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("expect", "100-continue") + .header("connection", "keep-alive") + .header("x-forwarded-for", "203.0.113.7") + .header("x-tenant", "acme") + .build() + .unwrap(), + ); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); let auth = req .headers() @@ -777,24 +821,29 @@ mod tests { // Both carry a credential, so a `Debug`-formatted request must not // print them. + let creds = aws_credential_types::Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("session-token".to_string()), + None, + "test", + ); let signer = SigV4Signer::new( - AwsCredentials { - access_key_id: "ak".to_string(), - secret_access_key: "sk".to_string().into(), - session_token: Some("session-token".to_string().into()), - }, "us-east-1".to_string(), "execute-api".to_string(), PayloadHashMode::StandardAws, ); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); signer .sign_at( &mut req, + &creds, Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -806,7 +855,7 @@ mod tests { .unwrap() .is_sensitive() ); - let debug = format!("{req:?}"); + let debug = format!("{:?}", req.headers()); assert!(!debug.contains("session-token"), "{debug}"); } @@ -817,15 +866,18 @@ mod tests { // The signer overwrites `x-amz-content-sha256`; the caller's value // moves aside instead of vanishing, after signing as Java does. let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("x-amz-content-sha256", "caller-supplied") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("x-amz-content-sha256", "caller-supplied") + .build() + .unwrap(), + ); signer .sign_at( &mut req, + &test_credentials(), Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); @@ -846,25 +898,27 @@ mod tests { // What a non-AWS S3-compatible catalog vends: its own signing name // rather than `execute-api`, its own region, and STS credentials. + let creds = aws_credential_types::Credentials::new( + "STS.EXAMPLEACCESSKEYID", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("example-session-token".to_string()), + None, + "test", + ); let signer = SigV4Signer::new( - AwsCredentials { - access_key_id: "STS.EXAMPLEACCESSKEYID".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: Some("example-session-token".to_string().into()), - }, - "us-east-1".into(), - "custom-service".into(), + "us-east-1".to_string(), + "custom-service".to_string(), PayloadHashMode::IcebergRest, ); - let mut req = reqwest::Client::new() - .get("https://catalog.example.com/v1/config?warehouse=my-catalog") - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://catalog.example.com/v1/config?warehouse=my-catalog") + .build() + .unwrap(), + ); let now = Utc.with_ymd_and_hms(2026, 8, 26, 12, 0, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &creds, now).unwrap(); assert_signature_is( &req, @@ -907,27 +961,28 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; #[test] fn signs_request_iceberg_mode() { - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: Some("SESSIONTOKEN".to_string().into()), - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("SESSIONTOKEN".to_string()), + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "glue".into(), + "us-east-1".to_string(), + "glue".to_string(), PayloadHashMode::IcebergRest, ); let client = reqwest::Client::new(); - let mut req = client - .post("https://rest.example.com/v1/namespaces") - .body("{}") - .build() - .unwrap(); + let mut req = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .body("{}") + .build() + .unwrap(), + ); - signer.sign(&mut req).unwrap(); + signer.sign(&mut req, &creds).unwrap(); let h = req.headers(); assert!( @@ -947,28 +1002,29 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; /// (mirrors Java's `TestRESTSigV4AuthSession::authenticateWithoutBody`). #[test] fn signs_empty_body_and_all_headers() { - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: None, - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None::, + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "glue".into(), + "us-east-1".to_string(), + "glue".to_string(), PayloadHashMode::IcebergRest, ); let client = reqwest::Client::new(); - let mut req = client - .get("https://rest.example.com/v1/config") - .header("content-type", "application/json") - .header("content-encoding", "gzip") - .build() - .unwrap(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/config") + .header("content-type", "application/json") + .header("content-encoding", "gzip") + .build() + .unwrap(), + ); - signer.sign(&mut req).unwrap(); + signer.sign(&mut req, &creds).unwrap(); let h = req.headers(); assert_eq!(h.get("x-amz-content-sha256").unwrap(), EMPTY_HEX); @@ -996,28 +1052,29 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; // invariant ours to keep. use chrono::TimeZone; - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: None, - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None::, + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "execute-api".into(), + "us-east-1".to_string(), + "execute-api".to_string(), PayloadHashMode::IcebergRest, ); let body = br#"{"namespace":["ns"]}"#; - let mut req = reqwest::Client::new() - .post("https://rest.example.com/v1/namespaces") - .body(body.to_vec()) - .build() - .unwrap(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .post("https://rest.example.com/v1/namespaces") + .body(body.to_vec()) + .build() + .unwrap(), + ); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &creds, now).unwrap(); // The header carries base64 while the pinned signature covers hex. assert_eq!( @@ -1038,27 +1095,28 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; fn signs_host_with_non_default_port() { use chrono::TimeZone; - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: None, - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None::, + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "glue".into(), + "us-east-1".to_string(), + "glue".to_string(), PayloadHashMode::IcebergRest, ); let client = reqwest::Client::new(); - let mut req = client - .get("https://rest.example.com:8181/v1/config") - .build() - .unwrap(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com:8181/v1/config") + .build() + .unwrap(), + ); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &creds, now).unwrap(); assert_signature_is( &req, "f7801a4ecac5fe6dcc4ee385223428ec4833f21d0c2fc10d2fb00694b2c0def7", @@ -1072,27 +1130,28 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; fn canonical_uri_is_aws_double_encoded() { use chrono::TimeZone; - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: None, - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None::, + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "glue".into(), + "us-east-1".to_string(), + "glue".to_string(), PayloadHashMode::IcebergRest, ); let client = reqwest::Client::new(); - let mut req = client - .get("https://rest.example.com/v1/namespaces/a%2Cb/tables/x,y") - .build() - .unwrap(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%2Cb/tables/x,y") + .build() + .unwrap(), + ); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); - signer.sign_at(&mut req, now).unwrap(); + signer.sign_at(&mut req, &creds, now).unwrap(); assert_signature_is( &req, "4d966b6fc2dfb62be5a603e4e07e5dc85b2af1c7e6a181c5646bfbf38ddfa543", @@ -1101,27 +1160,28 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; #[test] fn signs_request_standard_mode_uses_hex_header() { - let creds = AwsCredentials { - access_key_id: "AKIDEXAMPLE".into(), - secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - .to_string() - .into(), - session_token: None, - }; + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + None::, + None, + "test", + ); let signer = SigV4Signer::new( - creds, - "us-east-1".into(), - "glue".into(), + "us-east-1".to_string(), + "glue".to_string(), PayloadHashMode::StandardAws, ); let client = reqwest::Client::new(); - let mut req = client - .post("https://rest.example.com/v1/namespaces") - .body("hello") - .build() - .unwrap(); + let mut req = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .body("hello") + .build() + .unwrap(), + ); - signer.sign(&mut req).unwrap(); + signer.sign(&mut req, &creds).unwrap(); // StandardAws keeps the header in hex. assert_eq!( diff --git a/crates/catalog/rest/src/auth/sigv4/mod.rs b/crates/catalog/rest/src/auth/sigv4/mod.rs deleted file mode 100644 index 6f1e9dea30..0000000000 --- a/crates/catalog/rest/src/auth/sigv4/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! AWS SigV4 request signing for the REST catalog. - -mod signer; - -pub use signer::{AwsCredentials, PayloadHashMode, SigV4Signer}; diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs index 7a3c8e084c..872355fb73 100644 --- a/crates/catalog/rest/src/request.rs +++ b/crates/catalog/rest/src/request.rs @@ -19,6 +19,8 @@ use http::{HeaderMap, Method}; use iceberg::Result; +#[cfg(feature = "sigv4")] +use reqwest::Url; use reqwest::{Request, RequestBuilder}; /// An outgoing REST request being authenticated by an @@ -48,6 +50,19 @@ impl HttpRequest { self.inner } + /// The request URL. + #[cfg(feature = "sigv4")] + pub(crate) fn url(&self) -> &Url { + self.inner.url() + } + + /// The mutable request URL, for signers that must rewrite it before + /// canonicalizing, e.g. to drop userinfo the wire `Host` never carries. + #[cfg(feature = "sigv4")] + pub(crate) fn url_mut(&mut self) -> &mut Url { + self.inner.url_mut() + } + /// The request method. pub fn method(&self) -> &Method { self.inner.method() From 3e252bdd6c8ef9cbd95188639e90aa6cc5872b5e Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 28 Aug 2026 00:21:02 -0400 Subject: [PATCH 04/30] docs(rest): trim the SigV4 signer comments --- crates/catalog/rest/src/auth/sigv4.rs | 72 +++++++++++---------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 3a592ac7eb..ab4f993085 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -70,8 +70,8 @@ fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec hmac_sha256(&k_service, b"aws4_request") } -/// Computes the value of the `x-amz-content-sha256` header. `None` is a request -/// with no body at all, which the two modes disagree about. +/// The `x-amz-content-sha256` value. `None` means no body at all, which the +/// two modes encode differently. fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { match mode { PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), @@ -82,8 +82,8 @@ fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { } } -/// AWS SigV4 signer following Iceberg Java's `RESTSigV4AuthSession`: it adds the -/// required amz headers and signs all request headers except a small blacklist. +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. #[derive(Clone)] pub struct SigV4Signer { region: String, @@ -101,17 +101,15 @@ impl SigV4Signer { } } - /// Signs `request` in place, rewriting it where signing requires it: an - /// existing `Authorization` moves to `Original-Authorization` and is signed - /// over, userinfo is dropped from the URL, and a `+` in the query becomes - /// `%20`. + /// Signs `request` in place, rewriting it as signing requires: an existing + /// `Authorization` becomes `Original-Authorization`, userinfo leaves the + /// URL, and a `+` in the query becomes `%20`. /// - /// That last one means a `+` is taken to be an encoded space, which is what - /// `RequestBuilder::query` writes; encode a literal plus as `%2B`. AWS - /// requires spaces as `%20` in a signed URL either way. + /// A `+` is therefore taken to be an encoded space; write a literal plus as + /// `%2B`. /// - /// Fails rather than sign a request whose body is streaming or whose - /// headers are not UTF-8, since neither can be canonicalized faithfully. + /// Fails rather than sign a streaming body or a non-UTF-8 header, neither + /// of which canonicalizes faithfully. pub fn sign( &self, request: &mut crate::HttpRequest, @@ -133,8 +131,7 @@ impl SigV4Signer { use aws_sigv4::sign::v4; use tracing::subscriber::NoSubscriber; - // `HttpRequestBody` already separates absent from empty, which Java - // relies on: it branches on `encodedBody() == null`. + // Java branches on `encodedBody() == null`, so absent and empty differ. let body: Option> = match request.body() { crate::HttpRequestBody::Empty => None, crate::HttpRequestBody::Buffered(bytes) => Some(bytes.to_vec()), @@ -149,8 +146,7 @@ impl SigV4Signer { convert_headers(request); - // Relocated after signing, as Java does, so the `Original-` copy is - // not itself signed. + // Relocated after signing, so the `Original-` copy is not signed. let displaced_content_hash: Vec<_> = request .headers() .get_all(CONTENT_SHA256) @@ -163,17 +159,15 @@ impl SigV4Signer { .insert(CONTENT_SHA256, content_header.parse().unwrap()); // The wire Host never carries userinfo, so signing it would mismatch - // and would feed the password to the HMAC. `RequestBuilder` strips it, - // a hand-built request does not. + // and feed the password to the HMAC. Only hand-built requests have it. if !request.url().username().is_empty() || request.url().password().is_some() { let url = request.url_mut(); let _ = url.set_username(""); let _ = url.set_password(None); } - // Verifiers disagree on `+` in a query — a literal under RFC 3986, a - // space under form encoding — and reqwest writes spaces as `+`. Rewrite - // it so the wire and the canonical request agree either way. + // Both AWS and Java read `+` as a space, so this leaves the signature + // alone; it makes the sent URL agree with an RFC 3986 verifier too. if let Some(query) = request.url().query().filter(|q| q.contains('+')) { let unambiguous = query.replace('+', "%20"); request.url_mut().set_query(Some(&unambiguous)); @@ -183,11 +177,10 @@ impl SigV4Signer { // `Aws4Signer` defaults: normalize the path and double-encode it. settings.percent_encoding_mode = PercentEncodingMode::Double; settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; - // The header is ours to set: IcebergRest mode puts base64 there, - // while the signer hashes the body in hex for the canonical request. + // The header is ours to set: IcebergRest puts base64 there, while the + // canonical request keeps hex. settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; - // Java's `AbstractAws4Signer` ignores these too; the crate's defaults - // do not. + // In Java's ignore list but not the crate's defaults. let mut excluded = settings.excluded_headers.take().unwrap_or_default(); excluded.extend([ "expect".into(), @@ -209,8 +202,8 @@ impl SigV4Signer { })? .into(); - // Skipping a header that is not UTF-8 would leave it unsigned but still - // on the wire, which AWS rejects for `x-amz-*` and is hard to diagnose. + // Skipping one would leave it unsigned but still on the wire, which + // AWS rejects for `x-amz-*` and is hard to diagnose. let headers: Vec<(&str, &str)> = request .headers() .iter() @@ -235,11 +228,8 @@ impl SigV4Signer { Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) })?; - // `aws_sigv4` traces the request it signs, and its redaction list covers - // `authorization` but not the `Original-` copy we just made, so a - // delegate's bearer token would be printed verbatim at trace level. - // Signing is synchronous and logs nothing else worth keeping, so mute - // it for the call. + // The crate traces what it signs, and redacts `authorization` but not + // the `Original-` copy, so a bearer token would be logged verbatim. let signed = tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)); let (instructions, _signature) = signed @@ -250,9 +240,8 @@ impl SigV4Signer { } } -/// Renames `Authorization` to `Original-Authorization` so SigV4 can take the -/// name, as Java's `convertHeaders` does. Runs *before* signing, so the -/// relocated copy is itself signed over. +/// Java's `convertHeaders`: renames `Authorization` so SigV4 can take the +/// name. Runs before signing, so the relocated copy is signed too. fn convert_headers(request: &mut crate::HttpRequest) { let displaced: Vec<_> = request .headers() @@ -270,9 +259,8 @@ fn convert_headers(request: &mut crate::HttpRequest) { } } -/// Installs the signed headers, moving any conflicting caller value aside -/// rather than dropping it, as Java's `updateRequestHeaders` does. Runs -/// *after* signing, so these `Original-` copies are not themselves signed. +/// Java's `updateRequestHeaders`: installs the signed headers, moving a +/// conflicting caller value aside rather than dropping it. fn update_request_headers( request: &mut crate::HttpRequest, instructions: aws_sigv4::http_request::SigningInstructions, @@ -303,7 +291,6 @@ fn update_request_headers( Ok(()) } -/// The payload-hash header SigV4 sets and Java relocates on conflict. const CONTENT_SHA256: reqwest::header::HeaderName = reqwest::header::HeaderName::from_static("x-amz-content-sha256"); const AMZ_DATE: reqwest::header::HeaderName = @@ -330,9 +317,8 @@ fn relocated_name(name: &str) -> Option { } } -/// Moves `name`'s current values to `Original-` when they differ from the -/// value about to be signed, so a caller's header is not silently dropped -/// (Java's `RESTSigV4AuthSession.updateRequestHeaders`). +/// Moves `name`'s values aside when they differ from the one about to be +/// signed, so a caller's header is not silently dropped. fn relocate_conflicting( headers: &mut reqwest::header::HeaderMap, name: &str, From fc32f42049ea4c59a2513762d73f382df45b0e4f Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 28 Aug 2026 01:18:06 -0400 Subject: [PATCH 05/30] refactor(rest): break sign_at into named steps --- crates/catalog/rest/Cargo.toml | 4 +- crates/catalog/rest/src/auth/sigv4.rs | 137 ++++++++++++++------------ 2 files changed, 77 insertions(+), 64 deletions(-) diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index d395a9adf8..9dea2ab9d3 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -33,13 +33,13 @@ repository = { workspace = true } default = [] # SigV4 request signing, for catalogs that authenticate the AWS way. Off by # default so the AWS crates are not pulled in unless they are used. -sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:sha2"] +sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:base64", "dep:sha2"] [dependencies] async-trait = { workspace = true } aws-credential-types = { workspace = true, optional = true } aws-sigv4 = { workspace = true, optional = true } -base64 = { workspace = true } +base64 = { workspace = true, optional = true } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index ab4f993085..2b2541c599 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -124,24 +124,11 @@ impl SigV4Signer { credentials: &aws_credential_types::Credentials, now: DateTime, ) -> Result<()> { - use aws_sigv4::http_request::{ - PayloadChecksumKind, PercentEncodingMode, SignableBody, SignableRequest, - SigningSettings, UriPathNormalizationMode, sign, - }; + use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; use aws_sigv4::sign::v4; use tracing::subscriber::NoSubscriber; - // Java branches on `encodedBody() == null`, so absent and empty differ. - let body: Option> = match request.body() { - crate::HttpRequestBody::Empty => None, - crate::HttpRequestBody::Buffered(bytes) => Some(bytes.to_vec()), - crate::HttpRequestBody::Streaming => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "cannot sign a streaming request body", - )); - } - }; + let body = signable_body(request)?; let content_header = content_sha256_header(body.as_deref(), self.mode); convert_headers(request); @@ -158,36 +145,7 @@ impl SigV4Signer { .headers_mut() .insert(CONTENT_SHA256, content_header.parse().unwrap()); - // The wire Host never carries userinfo, so signing it would mismatch - // and feed the password to the HMAC. Only hand-built requests have it. - if !request.url().username().is_empty() || request.url().password().is_some() { - let url = request.url_mut(); - let _ = url.set_username(""); - let _ = url.set_password(None); - } - - // Both AWS and Java read `+` as a space, so this leaves the signature - // alone; it makes the sent URL agree with an RFC 3986 verifier too. - if let Some(query) = request.url().query().filter(|q| q.contains('+')) { - let unambiguous = query.replace('+', "%20"); - request.url_mut().set_query(Some(&unambiguous)); - } - - let mut settings = SigningSettings::default(); - // `Aws4Signer` defaults: normalize the path and double-encode it. - settings.percent_encoding_mode = PercentEncodingMode::Double; - settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; - // The header is ours to set: IcebergRest puts base64 there, while the - // canonical request keeps hex. - settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; - // In Java's ignore list but not the crate's defaults. - let mut excluded = settings.excluded_headers.take().unwrap_or_default(); - excluded.extend([ - "expect".into(), - "connection".into(), - "x-forwarded-for".into(), - ]); - settings.excluded_headers = Some(excluded); + rewrite_url_for_signing(request); let identity = credentials.clone().into(); let params = v4::SigningParams::builder() @@ -195,29 +153,14 @@ impl SigV4Signer { .region(&self.region) .name(&self.service) .time(now.into()) - .settings(settings) + .settings(signing_settings()) .build() .map_err(|e| { Error::new(ErrorKind::Unexpected, "failed to build SigV4 params").with_source(e) })? .into(); - // Skipping one would leave it unsigned but still on the wire, which - // AWS rejects for `x-amz-*` and is hard to diagnose. - let headers: Vec<(&str, &str)> = request - .headers() - .iter() - .map(|(n, v)| { - let v = v.to_str().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("cannot sign non-UTF-8 header value for `{n}`"), - ) - .with_source(e) - })?; - Ok((n.as_str(), v)) - }) - .collect::>()?; + let headers = signable_headers(request)?; let signable = SignableRequest::new( request.method().as_str(), request.url_str(), @@ -240,6 +183,76 @@ impl SigV4Signer { } } +/// The body to sign. Java branches on `encodedBody() == null`, so an absent +/// body and an empty one hash differently. +fn signable_body(request: &crate::HttpRequest) -> Result>> { + match request.body() { + crate::HttpRequestBody::Empty => Ok(None), + crate::HttpRequestBody::Buffered(bytes) => Ok(Some(bytes.to_vec())), + crate::HttpRequestBody::Streaming => Err(Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + )), + } +} + +/// The headers to sign. Skipping a non-UTF-8 one would leave it unsigned but +/// still on the wire, which AWS rejects for `x-amz-*` and is hard to diagnose. +fn signable_headers(request: &crate::HttpRequest) -> Result> { + request + .headers() + .iter() + .map(|(n, v)| { + let v = v.to_str().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("cannot sign non-UTF-8 header value for `{n}`"), + ) + .with_source(e) + })?; + Ok((n.as_str(), v)) + }) + .collect() +} + +/// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the +/// query. Both AWS and Java read `+` as a space, so the signature is unchanged; +/// this makes the sent URL agree with an RFC 3986 verifier too. +fn rewrite_url_for_signing(request: &mut crate::HttpRequest) { + if !request.url().username().is_empty() || request.url().password().is_some() { + let url = request.url_mut(); + let _ = url.set_username(""); + let _ = url.set_password(None); + } + if let Some(query) = request.url().query().filter(|q| q.contains('+')) { + let unambiguous = query.replace('+', "%20"); + request.url_mut().set_query(Some(&unambiguous)); + } +} + +/// `Aws4Signer`'s settings: normalized double-encoded path, and Java's ignore +/// list, which the crate's defaults cover only in part. +fn signing_settings() -> aws_sigv4::http_request::SigningSettings { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SigningSettings, UriPathNormalizationMode, + }; + + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Double; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; + // The header is ours to set: IcebergRest puts base64 there, while the + // canonical request keeps hex. + settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; + let mut excluded = settings.excluded_headers.take().unwrap_or_default(); + excluded.extend([ + "expect".into(), + "connection".into(), + "x-forwarded-for".into(), + ]); + settings.excluded_headers = Some(excluded); + settings +} + /// Java's `convertHeaders`: renames `Authorization` so SigV4 can take the /// name. Runs before signing, so the relocated copy is signed too. fn convert_headers(request: &mut crate::HttpRequest) { From 89d8f3fbd3bd4df8c88c729771e067d0bc776ffd Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 20 Sep 2026 14:47:34 -0400 Subject: [PATCH 06/30] docs(rest): state the redirect and log-always limits of the signer --- crates/catalog/rest/src/auth/sigv4.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 2b2541c599..387a6bb264 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -31,7 +31,9 @@ const EMPTY_BODY_HEX_SHA256: &str = #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PayloadHashMode { /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex - /// when there is none; the canonical request always uses hex. + /// when there is none; the canonical request always uses hex. A header the + /// caller already set is replaced and moved to `Original-x-amz-content-sha256`, + /// where Java signs the caller's value as-is. IcebergRest, /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). StandardAws, @@ -110,6 +112,14 @@ impl SigV4Signer { /// /// Fails rather than sign a streaming body or a non-UTF-8 header, neither /// of which canonicalizes faithfully. + /// + /// Send the result through a client that does not follow redirects: a + /// redirect replays a signature made for another URL, and across hosts + /// reqwest drops `Authorization` but keeps `Original-Authorization`. + /// + /// `tracing` subscribers are muted while signing, since `aws_sigv4` traces + /// the headers it is given. The `tracing/log-always` bridge to `log` is not, + /// and still forwards them at trace level. pub fn sign( &self, request: &mut crate::HttpRequest, From 26bf79d56b32df23ad2130601f91de93f8c4a793 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 20 Sep 2026 19:27:05 -0400 Subject: [PATCH 07/30] fix(rest): keep relocation headers out of the signature, and the tracing flag untouched --- crates/catalog/rest/src/auth/sigv4.rs | 75 +++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 387a6bb264..cad26a6411 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -31,9 +31,10 @@ const EMPTY_BODY_HEX_SHA256: &str = #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PayloadHashMode { /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex - /// when there is none; the canonical request always uses hex. A header the - /// caller already set is replaced and moved to `Original-x-amz-content-sha256`, - /// where Java signs the caller's value as-is. + /// when there is none; the canonical request always uses hex. A caller-set + /// header is replaced, and moved to `Original-x-amz-content-sha256` when it + /// differed. Java replaces it too for a bodiless request, but signs the + /// caller's value when a body is present. IcebergRest, /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). StandardAws, @@ -183,8 +184,14 @@ impl SigV4Signer { // The crate traces what it signs, and redacts `authorization` but not // the `Original-` copy, so a bearer token would be logged verbatim. - let signed = - tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)); + // Only when a subscriber exists: `with_default` sets tracing's global + // "a dispatcher was installed" flag for good, which would silently + // divert every later event away from an app's `tracing/log` bridge. + let signed = if tracing::dispatcher::has_been_set() { + tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) + } else { + sign(signable, ¶ms) + }; let (instructions, _signature) = signed .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? .into_parts(); @@ -258,6 +265,11 @@ fn signing_settings() -> aws_sigv4::http_request::SigningSettings { "expect".into(), "connection".into(), "x-forwarded-for".into(), + // Relocation appends to these after signing, so a caller-supplied one + // would otherwise be signed and then changed on the wire. + "original-x-amz-date".into(), + "original-x-amz-content-sha256".into(), + "original-x-amz-security-token".into(), ]); settings.excluded_headers = Some(excluded); settings @@ -580,6 +592,59 @@ mod tests { assert!(err.message().contains("x-amz-meta-tenant"), "{err}"); } + /// A caller-supplied `Original-x-amz-*` must not be signed: relocation + /// appends to it afterwards, which would change a signed value on the wire. + #[test] + fn a_caller_supplied_relocation_header_is_not_signed() { + use chrono::TimeZone; + + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("x-amz-content-sha256", "caller-hash") + .header("original-x-amz-content-sha256", "previous") + .build() + .unwrap(), + ); + + signer + .sign_at( + &mut req, + &test_credentials(), + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); + + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth + .split("SignedHeaders=") + .nth(1) + .unwrap() + .split(',') + .next() + .unwrap(); + assert!( + !signed + .split(';') + .any(|h| h == "original-x-amz-content-sha256"), + "{signed}" + ); + // Both values still travel, they are just outside the signature. + let relocated: Vec<_> = req + .headers() + .get_all("original-x-amz-content-sha256") + .iter() + .map(|v| v.to_str().unwrap()) + .collect(); + assert_eq!(relocated, ["previous", "caller-hash"]); + } + #[test] fn userinfo_is_stripped_before_signing() { use chrono::TimeZone; From 8edf54449d6d97e54bad7e682dafa1293af16a1b Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 22 Sep 2026 04:52:58 -0400 Subject: [PATCH 08/30] docs(rest): name the log bridge that stays unmuted --- crates/catalog/rest/src/auth/sigv4.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index cad26a6411..5076bcbe39 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -118,9 +118,10 @@ impl SigV4Signer { /// redirect replays a signature made for another URL, and across hosts /// reqwest drops `Authorization` but keeps `Original-Authorization`. /// - /// `tracing` subscribers are muted while signing, since `aws_sigv4` traces - /// the headers it is given. The `tracing/log-always` bridge to `log` is not, - /// and still forwards them at trace level. + /// `aws_sigv4` traces the headers it is given, and its redaction list does + /// not cover the `Original-` copy. An installed `tracing` subscriber is + /// muted for the call; with no subscriber, `tracing`'s `log` bridge still + /// forwards those events, so keep `aws_sigv4` below trace level there. pub fn sign( &self, request: &mut crate::HttpRequest, @@ -184,9 +185,10 @@ impl SigV4Signer { // The crate traces what it signs, and redacts `authorization` but not // the `Original-` copy, so a bearer token would be logged verbatim. - // Only when a subscriber exists: `with_default` sets tracing's global - // "a dispatcher was installed" flag for good, which would silently - // divert every later event away from an app's `tracing/log` bridge. + // Only when a subscriber exists. `with_default` sets tracing's global + // "a dispatcher was installed" flag for good, so doing it unasked would + // silently divert every later event away from an app's `log` bridge — + // a worse trade than a trace-level exposure the operator opted into. let signed = if tracing::dispatcher::has_been_set() { tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) } else { From df66c944740aa017eaa05972528a87fcefcc0122 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Mon, 28 Sep 2026 00:15:13 -0400 Subject: [PATCH 09/30] docs(rest): log-always also bypasses the muted subscriber --- crates/catalog/rest/src/auth/sigv4.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 5076bcbe39..d0eb0b957a 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -120,8 +120,9 @@ impl SigV4Signer { /// /// `aws_sigv4` traces the headers it is given, and its redaction list does /// not cover the `Original-` copy. An installed `tracing` subscriber is - /// muted for the call; with no subscriber, `tracing`'s `log` bridge still - /// forwards those events, so keep `aws_sigv4` below trace level there. + /// muted for the call; with no subscriber, or with `tracing`'s `log-always` + /// feature, its `log` bridge still forwards those events, so keep + /// `aws_sigv4` below trace level there. pub fn sign( &self, request: &mut crate::HttpRequest, From b9fa6cbb1d7c8c3ef54ac0e23b4e5da2e1eba8cf Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:42:23 -0400 Subject: [PATCH 10/30] fix(rest): return an error instead of unwrapping the content hash header --- crates/catalog/rest/src/auth/sigv4.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index d0eb0b957a..7cc0a40ce1 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -154,9 +154,10 @@ impl SigV4Signer { .filter(|v| v.as_bytes() != content_header.as_bytes()) .cloned() .collect(); - request - .headers_mut() - .insert(CONTENT_SHA256, content_header.parse().unwrap()); + let content_value = content_header.parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid x-amz-content-sha256 value").with_source(e) + })?; + request.headers_mut().insert(CONTENT_SHA256, content_value); rewrite_url_for_signing(request); From 5d6b772aff6c4022b8e1e561eeecc812102d9077 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:42:51 -0400 Subject: [PATCH 11/30] refactor(rest): make PayloadHashMode non_exhaustive and SigV4Signer::new take impl Into --- crates/catalog/rest/src/auth/sigv4.rs | 73 +++++++-------------------- 1 file changed, 19 insertions(+), 54 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 7cc0a40ce1..8601862961 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -29,6 +29,7 @@ const EMPTY_BODY_HEX_SHA256: &str = /// How the payload hash is encoded in the `x-amz-content-sha256` header. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] pub enum PayloadHashMode { /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex /// when there is none; the canonical request always uses hex. A caller-set @@ -96,10 +97,14 @@ pub struct SigV4Signer { impl SigV4Signer { /// Creates a new SigV4 signer. - pub fn new(region: String, service: String, mode: PayloadHashMode) -> Self { + pub fn new( + region: impl Into, + service: impl Into, + mode: PayloadHashMode, + ) -> Self { Self { - region, - service, + region: region.into(), + service: service.into(), mode, } } @@ -409,11 +414,7 @@ mod tests { ); assert!(request.url().query().unwrap().contains("my+ns")); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "execute-api".to_string(), - PayloadHashMode::StandardAws, - ); + let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); signer .sign_at(&mut request, &test_credentials(), now) @@ -499,7 +500,7 @@ mod tests { } fn test_signer(mode: PayloadHashMode) -> SigV4Signer { - SigV4Signer::new("us-east-1".to_string(), "execute-api".to_string(), mode) + SigV4Signer::new("us-east-1", "execute-api", mode) } fn test_credentials() -> aws_credential_types::Credentials { @@ -709,11 +710,7 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "execute-api".to_string(), - PayloadHashMode::StandardAws, - ); + let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -906,11 +903,7 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "execute-api".to_string(), - PayloadHashMode::StandardAws, - ); + let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -983,11 +976,7 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "custom-service".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "custom-service", PayloadHashMode::IcebergRest); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://catalog.example.com/v1/config?warehouse=my-catalog") @@ -1046,11 +1035,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "glue".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1087,11 +1072,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "glue".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1137,11 +1118,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "execute-api".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::IcebergRest); let body = br#"{"namespace":["ns"]}"#; let mut req = HttpRequest::new( reqwest::Client::new() @@ -1180,11 +1157,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "glue".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1215,11 +1188,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "glue".to_string(), - PayloadHashMode::IcebergRest, - ); + let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1245,11 +1214,7 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new( - "us-east-1".to_string(), - "glue".to_string(), - PayloadHashMode::StandardAws, - ); + let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::StandardAws); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client From f1472931da5b51c6aadf85c4ea2b4b0d20b1aec3 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:43:08 -0400 Subject: [PATCH 12/30] docs(rest): note the base64 header quirk and credential freshness --- crates/catalog/rest/src/auth/sigv4.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 8601862961..d1a57ee336 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -36,8 +36,13 @@ pub enum PayloadHashMode { /// header is replaced, and moved to `Original-x-amz-content-sha256` when it /// differed. Java replaces it too for a bodiless request, but signs the /// caller's value when a body is present. + /// + /// The base64 header comes from how Java configures the AWS SDK, not from + /// SigV4. Signatures verify either way, but a server that checks the + /// header against the body's hex hash rejects it. IcebergRest, - /// Standard AWS SigV4 style: hex everywhere (e.g. AWS Glue). + /// Standard AWS SigV4 style, as botocore (and so PyIceberg) signs: hex + /// everywhere. StandardAws, } @@ -113,6 +118,10 @@ impl SigV4Signer { /// `Authorization` becomes `Original-Authorization`, userinfo leaves the /// URL, and a `+` in the query becomes `%20`. /// + /// Signs with exactly the `credentials` given and never refreshes them: + /// resolve temporary ones (STS, IRSA, an instance role) afresh for each + /// call, as Java does per request. + /// /// A `+` is therefore taken to be an encoded space; write a literal plus as /// `%2B`. /// From b205a4051d6a153f4de2b6cc744cbe5333753aaf Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:43:23 -0400 Subject: [PATCH 13/30] docs(rest): keep the plus note next to the rewrite it explains --- crates/catalog/rest/src/auth/sigv4.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index d1a57ee336..3f92d43afd 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -118,13 +118,13 @@ impl SigV4Signer { /// `Authorization` becomes `Original-Authorization`, userinfo leaves the /// URL, and a `+` in the query becomes `%20`. /// + /// A `+` is therefore taken to be an encoded space; write a literal plus as + /// `%2B`. + /// /// Signs with exactly the `credentials` given and never refreshes them: /// resolve temporary ones (STS, IRSA, an instance role) afresh for each /// call, as Java does per request. /// - /// A `+` is therefore taken to be an encoded space; write a literal plus as - /// `%2B`. - /// /// Fails rather than sign a streaming body or a non-UTF-8 header, neither /// of which canonicalizes faithfully. /// From 14e0f05191d3ee46de7d88abff6b4a31bcca3139 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:44:45 -0400 Subject: [PATCH 14/30] fix(rest): spell out the headers Java leaves unsigned --- crates/catalog/rest/src/auth/sigv4.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 3f92d43afd..2e7e9bc224 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -266,7 +266,7 @@ fn rewrite_url_for_signing(request: &mut crate::HttpRequest) { } /// `Aws4Signer`'s settings: normalized double-encoded path, and Java's ignore -/// list, which the crate's defaults cover only in part. +/// list. fn signing_settings() -> aws_sigv4::http_request::SigningSettings { use aws_sigv4::http_request::{ PayloadChecksumKind, PercentEncodingMode, SigningSettings, UriPathNormalizationMode, @@ -280,8 +280,14 @@ fn signing_settings() -> aws_sigv4::http_request::SigningSettings { settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; let mut excluded = settings.excluded_headers.take().unwrap_or_default(); excluded.extend([ - "expect".into(), + // Java's list, spelled out even where the crate's defaults overlap, so + // a minor release cannot start signing a header a proxy rewrites. "connection".into(), + "expect".into(), + "transfer-encoding".into(), + "user-agent".into(), + "x-amzn-trace-id".into(), + // Not on Java's list, but a proxy appends to it as well. "x-forwarded-for".into(), // Relocation appends to these after signing, so a caller-supplied one // would otherwise be signed and then changed on the wire. @@ -859,7 +865,7 @@ mod tests { // A proxy or an HTTP/2 hop may drop or rewrite these, so signing them // would make the request fail verification. Java's `AbstractAws4Signer` - // ignores them too. + // ignores the first two as well. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() From 44033c9b0cfacdf1bf55ae650659396cd4a8575b Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:45:32 -0400 Subject: [PATCH 15/30] test(rest): check signatures against Iceberg Java --- crates/catalog/rest/src/auth/sigv4.rs | 60 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 2e7e9bc224..ff870fab7b 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -502,8 +502,8 @@ mod tests { ); } - /// The signature the previous hand-rolled signer produced for this case, - /// pinned so a change in canonicalization is caught. + /// Pins a signature this crate produced, so a change in canonicalization + /// is caught; `signatures_match_iceberg_java` is the check against Java. fn assert_signature_is(req: &HttpRequest, expected: &str) { let auth = req .headers() @@ -1161,6 +1161,62 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; ); } + /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1, AWS SDK + /// 2.29.52) sent for the same requests: a check against an independent + /// implementation, where the other pins in this file only catch changes. + #[test] + fn signatures_match_iceberg_java() { + use chrono::TimeZone; + + let creds = aws_credential_types::Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("example-session-token".to_string()), + None, + "test", + ); + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = Utc.with_ymd_and_hms(2026, 10, 2, 11, 40, 30).unwrap(); + let client = reqwest::Client::new(); + + // A body, so the header carries base64, and a token to relocate. + let mut post = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .header("content-type", "application/json") + .header("authorization", "Bearer delegate-token") + .body(r#"{"namespace":["a","b"],"properties":{}}"#) + .build() + .unwrap(), + ); + signer.sign_at(&mut post, &creds, now).unwrap(); + assert_eq!( + post.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=content-type;host;original-authorization;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=c5d6bfcecd19c5421e8696c67465f6909b5e09398608d8e79a9eb391e04de556" + ); + assert_eq!( + post.headers().get("x-amz-content-sha256").unwrap(), + "n1CB8Yl4L77sBwHTc5qTpUEhlsHHOXFzg0dV0fbOwXs=" + ); + + // No body, a multi-level namespace and an encoded query. + let mut get = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%1Fb/tables/x,y?pageToken=a%20b%2Fc") + .build() + .unwrap(), + ); + signer.sign_at(&mut get, &creds, now).unwrap(); + assert_eq!( + get.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=5f6832ec82fc821b08cd3eac3a86ffc3e0c333571fd0a5796e43f2534f23329f" + ); + } + #[test] fn signs_host_with_non_default_port() { use chrono::TimeZone; From 0f02a729235626450c9d44e6b6b71b6bc1473e18 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:46:23 -0400 Subject: [PATCH 16/30] test(rest): pin the signing tests to a fixed time --- crates/catalog/rest/src/auth/sigv4.rs | 66 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index ff870fab7b..345e10e75c 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -1043,6 +1043,8 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; #[test] fn signs_request_iceberg_mode() { + use chrono::TimeZone; + let creds = aws_credential_types::Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", @@ -1060,7 +1062,13 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; .unwrap(), ); - signer.sign(&mut req, &creds).unwrap(); + signer + .sign_at( + &mut req, + &creds, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); let h = req.headers(); assert!( @@ -1070,16 +1078,22 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; .unwrap() .starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") ); - assert!(h.contains_key("x-amz-date")); + assert_eq!(h.get("x-amz-date").unwrap(), "20150830T123600Z"); assert_eq!(h.get("x-amz-security-token").unwrap(), "SESSIONTOKEN"); let csha = h.get("x-amz-content-sha256").unwrap().to_str().unwrap(); assert_eq!(csha, "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o="); + assert_signature_is( + &req, + "effad6acde583dd14ba7aff52b2b83776a54421c010fe82067f166819057cb32", + ); } /// Empty body uses the hex constant and existing headers are signed too /// (mirrors Java's `TestRESTSigV4AuthSession::authenticateWithoutBody`). #[test] fn signs_empty_body_and_all_headers() { + use chrono::TimeZone; + let creds = aws_credential_types::Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", @@ -1098,7 +1112,13 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; .unwrap(), ); - signer.sign(&mut req, &creds).unwrap(); + signer + .sign_at( + &mut req, + &creds, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); let h = req.headers(); assert_eq!(h.get("x-amz-content-sha256").unwrap(), EMPTY_HEX); @@ -1108,6 +1128,10 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; assert!(auth.contains( "SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date" )); + assert_signature_is( + &req, + "eaef7eb88d9cd810031684748671d8a3c9394ea5168622a212ed041b670b9777", + ); } /// The signed `host` must include an explicit non-default port, matching @@ -1278,6 +1302,8 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; #[test] fn signs_request_standard_mode_uses_hex_header() { + use chrono::TimeZone; + let creds = aws_credential_types::Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", @@ -1295,12 +1321,44 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; .unwrap(), ); - signer.sign(&mut req, &creds).unwrap(); + signer + .sign_at( + &mut req, + &creds, + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap(); // StandardAws keeps the header in hex. assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" ); + assert_signature_is( + &req, + "3531ce95a486df958b89f812277d764ea1e0bba4ad674b03ca137fdb59de1eac", + ); + } + + /// The one test on the live clock: everything else pins the time. + #[test] + fn sign_stamps_the_current_time() { + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + let before = Utc::now(); + + signer.sign(&mut req, &test_credentials()).unwrap(); + + let date = req.headers().get("x-amz-date").unwrap().to_str().unwrap(); + let stamped = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") + .unwrap() + .and_utc(); + assert_eq!(date.len(), "YYYYMMDDTHHMMSSZ".len(), "{date}"); + assert!((stamped - before).num_seconds().abs() < 60, "{date}"); } } From 5aab197c6019f204ad1eb571095eb1fcfc9769f7 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:51:42 -0400 Subject: [PATCH 17/30] test(rest): check standard mode against the AWS test suite --- crates/catalog/rest/src/auth/sigv4.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 345e10e75c..b86ba12f18 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -1300,8 +1300,10 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; ); } + /// The AWS SigV4 test suite's `post-x-www-form-urlencoded` case, which + /// signs a hex `x-amz-content-sha256` as this mode does. #[test] - fn signs_request_standard_mode_uses_hex_header() { + fn standard_mode_matches_the_aws_test_suite() { use chrono::TimeZone; let creds = aws_credential_types::Credentials::new( @@ -1311,12 +1313,13 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; None, "test", ); - let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::StandardAws); - let client = reqwest::Client::new(); + let signer = SigV4Signer::new("us-east-1", "service", PayloadHashMode::StandardAws); let mut req = HttpRequest::new( - client - .post("https://rest.example.com/v1/namespaces") - .body("hello") + reqwest::Client::new() + .post("https://example.amazonaws.com/") + .header("content-type", "application/x-www-form-urlencoded") + .header("content-length", "13") + .body("Param1=value1") .build() .unwrap(), ); @@ -1329,14 +1332,15 @@ bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; ) .unwrap(); - // StandardAws keeps the header in hex. assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), - "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + "9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e" ); - assert_signature_is( - &req, - "3531ce95a486df958b89f812277d764ea1e0bba4ad674b03ca137fdb59de1eac", + assert_eq!( + req.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, \ + SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=d3875051da38690788ef43de4db0d8f280229d82040bfac253562e56c3f20e0b" ); } From 22af03675bdd49b5ddabda35c510e0fcda2c9a25 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:54:14 -0400 Subject: [PATCH 18/30] test(rest): drop the helper-only AWS vector and the hmac dev-dependency --- Cargo.lock | 1 - Cargo.toml | 1 - crates/catalog/rest/Cargo.toml | 1 - crates/catalog/rest/src/auth/sigv4.rs | 42 --------------------------- 4 files changed, 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 416153a369..2b8f019413 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2836,7 +2836,6 @@ dependencies = [ "bytes", "chrono", "futures", - "hmac 0.12.1", "http 1.5.0", "iceberg", "iceberg_test_utils", diff --git a/Cargo.toml b/Cargo.toml index 5cf55d2a76..54d14085e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,6 @@ fnv = "1.0.7" form_urlencoded = "1.2.2" futures = "0.3" hive_metastore = "0.2.0" -hmac = "0.12" http = "1.2" iceberg = { version = "0.10.0", path = "./crates/iceberg" } iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" } diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 9dea2ab9d3..6526129fb4 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -57,7 +57,6 @@ uuid = { workspace = true, features = ["v4"] } [dev-dependencies] bytes = { workspace = true } futures = { workspace = true } -hmac = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } mockito = { workspace = true } # `stream` lets tests build a streaming body to exercise HttpRequestBody::Streaming. diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index b86ba12f18..6eface3c82 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -18,8 +18,6 @@ //! AWS SigV4 request signing for the REST catalog. use chrono::{DateTime, Utc}; -#[cfg(test)] -use hmac::{Hmac, Mac}; use iceberg::{Error, ErrorKind, Result}; use sha2::{Digest, Sha256}; @@ -46,23 +44,10 @@ pub enum PayloadHashMode { StandardAws, } -/// Derives the AWS SigV4 signing key. -#[cfg(test)] -fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { - let mut mac = as Mac>::new_from_slice(key).expect("HMAC takes a key of any size"); - mac.update(data); - mac.finalize().into_bytes().to_vec() -} - fn hex_sha256(data: &[u8]) -> String { encode_hex(&Sha256::digest(data)) } -#[cfg(test)] -fn hex_hmac_sha256(key: &[u8], data: &[u8]) -> String { - encode_hex(&hmac_sha256(key, data)) -} - fn encode_hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -71,14 +56,6 @@ fn base64_encode(bytes: &[u8]) -> String { base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) } -#[cfg(test)] -fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec { - let k_date = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes()); - let k_region = hmac_sha256(&k_date, region.as_bytes()); - let k_service = hmac_sha256(&k_region, service.as_bytes()); - hmac_sha256(&k_service, b"aws4_request") -} - /// The `x-amz-content-sha256` value. `None` means no body at all, which the /// two modes encode differently. fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { @@ -1022,25 +999,6 @@ mod tests { ); } - #[test] - fn signing_key_and_signature_match_aws_vector() { - let secret = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; - let date = "20150830"; - let region = "us-east-1"; - let service = "service"; - let key = signing_key(secret, date, region, service); - - let string_to_sign = "AWS4-HMAC-SHA256\n\ -20150830T123600Z\n\ -20150830/us-east-1/service/aws4_request\n\ -bb579772317eb040ac9ed261061d46c1f17a8133879d6129b6e1c25292927e63"; - let sig = hex_hmac_sha256(&key, string_to_sign.as_bytes()); - assert_eq!( - sig, - "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" - ); - } - #[test] fn signs_request_iceberg_mode() { use chrono::TimeZone; From 9e9ff82042e09e62c5b4bf58be7f6522b1e36328 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:57:00 -0400 Subject: [PATCH 19/30] fix(rest): decide the tracing mute with public tracing API --- crates/catalog/rest/src/auth/sigv4.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 6eface3c82..e7f59f6268 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -110,10 +110,10 @@ impl SigV4Signer { /// reqwest drops `Authorization` but keeps `Original-Authorization`. /// /// `aws_sigv4` traces the headers it is given, and its redaction list does - /// not cover the `Original-` copy. An installed `tracing` subscriber is - /// muted for the call; with no subscriber, or with `tracing`'s `log-always` - /// feature, its `log` bridge still forwards those events, so keep - /// `aws_sigv4` below trace level there. + /// not cover the `Original-` copy. A `tracing` subscriber that could record + /// that is muted for the call; with no subscriber, or with `tracing`'s + /// `log-always` feature, its `log` bridge still forwards those events, so + /// keep `aws_sigv4` below trace level there. pub fn sign( &self, request: &mut crate::HttpRequest, @@ -130,6 +130,7 @@ impl SigV4Signer { ) -> Result<()> { use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; use aws_sigv4::sign::v4; + use tracing::level_filters::LevelFilter; use tracing::subscriber::NoSubscriber; let body = signable_body(request)?; @@ -178,11 +179,11 @@ impl SigV4Signer { // The crate traces what it signs, and redacts `authorization` but not // the `Original-` copy, so a bearer token would be logged verbatim. - // Only when a subscriber exists. `with_default` sets tracing's global - // "a dispatcher was installed" flag for good, so doing it unasked would - // silently divert every later event away from an app's `log` bridge — - // a worse trade than a trace-level exposure the operator opted into. - let signed = if tracing::dispatcher::has_been_set() { + // Mute it only when a subscriber could record trace events (the max + // level stays `OFF` until one is registered): `with_default` marks a + // dispatcher as set for good, which would silently divert every later + // event away from an app's `log` bridge. + let signed = if LevelFilter::current() == LevelFilter::TRACE { tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) } else { sign(signable, ¶ms) @@ -543,6 +544,11 @@ mod tests { let log = CapturedLog::default(); tracing::subscriber::with_default(log.clone(), || { + // `sign_at` mutes only at this level, which is what is under test. + assert_eq!( + tracing::level_filters::LevelFilter::current(), + tracing::level_filters::LevelFilter::TRACE + ); signer .sign_at( &mut req, @@ -1175,7 +1181,8 @@ mod tests { assert_eq!( post.headers().get("authorization").unwrap(), "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ - SignedHeaders=content-type;host;original-authorization;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + SignedHeaders=content-type;host;original-authorization;\ + x-amz-content-sha256;x-amz-date;x-amz-security-token, \ Signature=c5d6bfcecd19c5421e8696c67465f6909b5e09398608d8e79a9eb391e04de556" ); assert_eq!( From c36c4574e3f97e60d7fe0c550f4911660dca5082 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:57:22 -0400 Subject: [PATCH 20/30] docs(rest): tighten the payload hash and credential notes --- crates/catalog/rest/src/auth/sigv4.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index e7f59f6268..74635a2d5e 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -36,11 +36,11 @@ pub enum PayloadHashMode { /// caller's value when a body is present. /// /// The base64 header comes from how Java configures the AWS SDK, not from - /// SigV4. Signatures verify either way, but a server that checks the - /// header against the body's hex hash rejects it. + /// SigV4. A verifier that hashes the body itself accepts it; one that + /// takes the payload hash from the header, or checks the header against + /// the hex hash, rejects it. IcebergRest, - /// Standard AWS SigV4 style, as botocore (and so PyIceberg) signs: hex - /// everywhere. + /// Standard AWS SigV4 style: hex everywhere. StandardAws, } @@ -99,8 +99,8 @@ impl SigV4Signer { /// `%2B`. /// /// Signs with exactly the `credentials` given and never refreshes them: - /// resolve temporary ones (STS, IRSA, an instance role) afresh for each - /// call, as Java does per request. + /// with temporary ones (STS, IRSA, an instance role), resolve them from + /// their provider before each call, as Java's session does per request. /// /// Fails rather than sign a streaming body or a non-UTF-8 header, neither /// of which canonicalizes faithfully. @@ -1150,7 +1150,8 @@ mod tests { } /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1, AWS SDK - /// 2.29.52) sent for the same requests: a check against an independent + /// 2.29.52, an `AuthSession.EMPTY` delegate, these credentials as `rest.*` + /// properties) sent for the same requests: a check against an independent /// implementation, where the other pins in this file only catch changes. #[test] fn signatures_match_iceberg_java() { From 2e6f50708401366afc688a13f60477f6c0c38c93 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 07:57:40 -0400 Subject: [PATCH 21/30] test(rest): bound the live-clock stamp by readings around the call --- crates/catalog/rest/src/auth/sigv4.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 74635a2d5e..15bed0c2b0 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -1313,6 +1313,8 @@ mod tests { /// The one test on the live clock: everything else pins the time. #[test] fn sign_stamps_the_current_time() { + use chrono::SubsecRound; + let signer = test_signer(PayloadHashMode::IcebergRest); let mut req = HttpRequest::new( reqwest::Client::new() @@ -1321,14 +1323,18 @@ mod tests { .unwrap(), ); let before = Utc::now(); - signer.sign(&mut req, &test_credentials()).unwrap(); + let after = Utc::now(); let date = req.headers().get("x-amz-date").unwrap().to_str().unwrap(); let stamped = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") .unwrap() .and_utc(); assert_eq!(date.len(), "YYYYMMDDTHHMMSSZ".len(), "{date}"); - assert!((stamped - before).num_seconds().abs() < 60, "{date}"); + // The header drops sub-seconds, so compare against a truncated start. + assert!( + before.trunc_subsecs(0) <= stamped && stamped <= after, + "{date}" + ); } } From 7824a3b932cfde0afc26eacce2a4df2679d02200 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 08:01:55 -0400 Subject: [PATCH 22/30] chore(rest): update the public API for the signer changes --- crates/catalog/rest/public-api.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 98502cef9d..43bd5ae146 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -14,7 +14,7 @@ impl<'a> core::fmt::Debug for iceberg_catalog_rest::HttpRequestBody<'a> pub fn iceberg_catalog_rest::HttpRequestBody<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl<'a> core::marker::Copy for iceberg_catalog_rest::HttpRequestBody<'a> impl<'a> core::marker::StructuralPartialEq for iceberg_catalog_rest::HttpRequestBody<'a> -pub enum iceberg_catalog_rest::PayloadHashMode +#[non_exhaustive] pub enum iceberg_catalog_rest::PayloadHashMode pub iceberg_catalog_rest::PayloadHashMode::IcebergRest pub iceberg_catalog_rest::PayloadHashMode::StandardAws impl core::clone::Clone for iceberg_catalog_rest::PayloadHashMode @@ -345,7 +345,7 @@ impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_catalog_rest::SigV4Signer impl iceberg_catalog_rest::SigV4Signer -pub fn iceberg_catalog_rest::SigV4Signer::new(region: alloc::string::String, service: alloc::string::String, mode: iceberg_catalog_rest::PayloadHashMode) -> Self +pub fn iceberg_catalog_rest::SigV4Signer::new(region: impl core::convert::Into, service: impl core::convert::Into, mode: iceberg_catalog_rest::PayloadHashMode) -> Self pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer From aba93f666bc7e6644cd011e7df7a6f158c94a5f4 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 2 Oct 2026 08:12:41 -0400 Subject: [PATCH 23/30] docs(rest): simplify the SigV4 signer comments --- crates/catalog/rest/Cargo.toml | 3 +- crates/catalog/rest/src/auth/sigv4.rs | 192 ++++++++++---------------- crates/catalog/rest/src/request.rs | 3 +- 3 files changed, 76 insertions(+), 122 deletions(-) diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 6526129fb4..0a8cf6b6e3 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -31,8 +31,7 @@ repository = { workspace = true } [features] default = [] -# SigV4 request signing, for catalogs that authenticate the AWS way. Off by -# default so the AWS crates are not pulled in unless they are used. +# SigV4 request signing. Off by default to keep the AWS crates out. sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:base64", "dep:sha2"] [dependencies] diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 15bed0c2b0..2d5cc8740c 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -29,18 +29,12 @@ const EMPTY_BODY_HEX_SHA256: &str = #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum PayloadHashMode { - /// Iceberg Java's RESTSigV4 style: base64 header when there is a body, hex - /// when there is none; the canonical request always uses hex. A caller-set - /// header is replaced, and moved to `Original-x-amz-content-sha256` when it - /// differed. Java replaces it too for a bodiless request, but signs the - /// caller's value when a body is present. - /// - /// The base64 header comes from how Java configures the AWS SDK, not from - /// SigV4. A verifier that hashes the body itself accepts it; one that - /// takes the payload hash from the header, or checks the header against - /// the hex hash, rejects it. + /// Iceberg Java's style: a base64 header when there is a body, hex when + /// there is none, and hex in the canonical request. The base64 comes from + /// how Java configures the AWS SDK, not from SigV4, so a verifier that + /// trusts the header instead of hashing the body rejects it. IcebergRest, - /// Standard AWS SigV4 style: hex everywhere. + /// Standard SigV4: hex everywhere. StandardAws, } @@ -56,8 +50,7 @@ fn base64_encode(bytes: &[u8]) -> String { base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) } -/// The `x-amz-content-sha256` value. `None` means no body at all, which the -/// two modes encode differently. +/// The `x-amz-content-sha256` value; `None` is no body at all. fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { match mode { PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), @@ -91,29 +84,27 @@ impl SigV4Signer { } } - /// Signs `request` in place, rewriting it as signing requires: an existing - /// `Authorization` becomes `Original-Authorization`, userinfo leaves the - /// URL, and a `+` in the query becomes `%20`. - /// - /// A `+` is therefore taken to be an encoded space; write a literal plus as - /// `%2B`. + /// Signs `request` in place. An existing `Authorization` moves to + /// `Original-Authorization`, and a caller's `x-amz-date`, + /// `x-amz-content-sha256` or `x-amz-security-token` that the signer + /// overwrites moves to `Original-` (Java keeps a caller's content + /// hash when there is a body). Userinfo leaves the URL, and a `+` in the + /// query becomes `%20`, so write a literal plus as `%2B`. /// - /// Signs with exactly the `credentials` given and never refreshes them: - /// with temporary ones (STS, IRSA, an instance role), resolve them from - /// their provider before each call, as Java's session does per request. + /// Uses `credentials` as given and never refreshes them: resolve temporary + /// ones from their provider before each call. /// - /// Fails rather than sign a streaming body or a non-UTF-8 header, neither - /// of which canonicalizes faithfully. + /// Fails on a streaming body or a non-UTF-8 header, which cannot be + /// canonicalized faithfully. /// /// Send the result through a client that does not follow redirects: a - /// redirect replays a signature made for another URL, and across hosts - /// reqwest drops `Authorization` but keeps `Original-Authorization`. + /// redirect replays the signature, and across hosts reqwest drops + /// `Authorization` but keeps `Original-Authorization`. /// - /// `aws_sigv4` traces the headers it is given, and its redaction list does - /// not cover the `Original-` copy. A `tracing` subscriber that could record - /// that is muted for the call; with no subscriber, or with `tracing`'s - /// `log-always` feature, its `log` bridge still forwards those events, so - /// keep `aws_sigv4` below trace level there. + /// `aws_sigv4` traces requests without redacting `Original-Authorization`. + /// A `tracing` subscriber is muted for the call, but the `log` bridge (no + /// subscriber, or `log-always`) still forwards those events, so keep + /// `aws_sigv4` below trace level there. pub fn sign( &self, request: &mut crate::HttpRequest, @@ -177,12 +168,10 @@ impl SigV4Signer { Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) })?; - // The crate traces what it signs, and redacts `authorization` but not - // the `Original-` copy, so a bearer token would be logged verbatim. - // Mute it only when a subscriber could record trace events (the max - // level stays `OFF` until one is registered): `with_default` marks a - // dispatcher as set for good, which would silently divert every later - // event away from an app's `log` bridge. + // `aws_sigv4` traces the request, `Original-Authorization` included. + // Mute it only when a subscriber could record that (the max level is + // `OFF` until one is registered): `with_default` marks tracing as in + // use for good, which turns off its `log` fallback process-wide. let signed = if LevelFilter::current() == LevelFilter::TRACE { tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) } else { @@ -196,8 +185,7 @@ impl SigV4Signer { } } -/// The body to sign. Java branches on `encodedBody() == null`, so an absent -/// body and an empty one hash differently. +/// The body to sign; as in Java, an absent body and an empty one differ. fn signable_body(request: &crate::HttpRequest) -> Result>> { match request.body() { crate::HttpRequestBody::Empty => Ok(None), @@ -209,8 +197,8 @@ fn signable_body(request: &crate::HttpRequest) -> Result>> { } } -/// The headers to sign. Skipping a non-UTF-8 one would leave it unsigned but -/// still on the wire, which AWS rejects for `x-amz-*` and is hard to diagnose. +/// The headers to sign. A non-UTF-8 one is an error: skipping it would send +/// it unsigned. fn signable_headers(request: &crate::HttpRequest) -> Result> { request .headers() @@ -229,8 +217,8 @@ fn signable_headers(request: &crate::HttpRequest) -> Result> { } /// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the -/// query. Both AWS and Java read `+` as a space, so the signature is unchanged; -/// this makes the sent URL agree with an RFC 3986 verifier too. +/// query as `%20`: a space to AWS and Java either way, but unambiguous to any +/// verifier. fn rewrite_url_for_signing(request: &mut crate::HttpRequest) { if !request.url().username().is_empty() || request.url().password().is_some() { let url = request.url_mut(); @@ -253,22 +241,19 @@ fn signing_settings() -> aws_sigv4::http_request::SigningSettings { let mut settings = SigningSettings::default(); settings.percent_encoding_mode = PercentEncodingMode::Double; settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; - // The header is ours to set: IcebergRest puts base64 there, while the - // canonical request keeps hex. + // We set the header ourselves: base64 for IcebergRest. settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; let mut excluded = settings.excluded_headers.take().unwrap_or_default(); excluded.extend([ - // Java's list, spelled out even where the crate's defaults overlap, so - // a minor release cannot start signing a header a proxy rewrites. + // Java's list, spelled out rather than left to the crate's defaults. "connection".into(), "expect".into(), "transfer-encoding".into(), "user-agent".into(), "x-amzn-trace-id".into(), - // Not on Java's list, but a proxy appends to it as well. + // Not Java's, but proxies append to it too. "x-forwarded-for".into(), - // Relocation appends to these after signing, so a caller-supplied one - // would otherwise be signed and then changed on the wire. + // Relocation appends to these after signing. "original-x-amz-date".into(), "original-x-amz-content-sha256".into(), "original-x-amz-security-token".into(), @@ -277,8 +262,8 @@ fn signing_settings() -> aws_sigv4::http_request::SigningSettings { settings } -/// Java's `convertHeaders`: renames `Authorization` so SigV4 can take the -/// name. Runs before signing, so the relocated copy is signed too. +/// Java's `convertHeaders`: moves `Authorization` aside before signing, so the +/// moved copy is signed. fn convert_headers(request: &mut crate::HttpRequest) { let displaced: Vec<_> = request .headers() @@ -296,8 +281,8 @@ fn convert_headers(request: &mut crate::HttpRequest) { } } -/// Java's `updateRequestHeaders`: installs the signed headers, moving a -/// conflicting caller value aside rather than dropping it. +/// Java's `updateRequestHeaders`: installs the signed headers, moving +/// conflicting caller values aside. fn update_request_headers( request: &mut crate::HttpRequest, instructions: aws_sigv4::http_request::SigningInstructions, @@ -354,8 +339,7 @@ fn relocated_name(name: &str) -> Option { } } -/// Moves `name`'s values aside when they differ from the one about to be -/// signed, so a caller's header is not silently dropped. +/// Moves `name`'s values that differ from `signed` to `relocated`. fn relocate_conflicting( headers: &mut reqwest::header::HeaderMap, name: &str, @@ -369,7 +353,7 @@ fn relocate_conflicting( .cloned() .collect(); for mut value in conflicting { - // The original may carry a credential (e.g. a session token). + // It may carry a credential, e.g. a session token. value.set_sensitive(true); headers.append(relocated.clone(), value); } @@ -396,8 +380,7 @@ mod tests { fn signing_rewrites_an_ambiguous_plus_out_of_the_query() { use chrono::TimeZone; - // reqwest writes a space as `+`, which verifiers read either as a - // literal plus or as a space. Signing rewrites it to `%20`. + // reqwest writes a space as `+`; signing makes it `%20`. let mut request = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/namespaces") @@ -413,7 +396,6 @@ mod tests { .sign_at(&mut request, &test_credentials(), now) .unwrap(); - // The request that goes out no longer carries the ambiguous form. let query = request.url().query().unwrap(); assert!(!query.contains('+'), "{query}"); assert!(query.contains("my%20ns"), "{query}"); @@ -431,8 +413,7 @@ mod tests { assert_eq!(e, EMPTY_HEX); } - /// Java branches on `encodedBody() == null`, so a body that is present but - /// empty is hashed like any other rather than taking the absent-body path. + /// As in Java, an empty body is hashed, unlike an absent one. #[test] fn content_sha256_header_separates_an_empty_body_from_an_absent_one() { let empty = content_sha256_header(Some(b""), PayloadHashMode::IcebergRest); @@ -443,8 +424,7 @@ mod tests { ); } - /// The same distinction, but through `sign_at`, so that collapsing the two - /// while reading the body off the request cannot go unnoticed. + /// The same, through `sign_at`. #[test] fn signing_separates_an_empty_body_from_an_absent_one() { use chrono::TimeZone; @@ -480,8 +460,8 @@ mod tests { ); } - /// Pins a signature this crate produced, so a change in canonicalization - /// is caught; `signatures_match_iceberg_java` is the check against Java. + /// Pins a signature this crate produced; `signatures_match_iceberg_java` + /// checks against Java. fn assert_signature_is(req: &HttpRequest, expected: &str) { let auth = req .headers() @@ -500,7 +480,7 @@ mod tests { aws_credential_types::Credentials::new("ak", "sk", None::, None, "test") } - /// Collects every event field a subscriber would have been handed. + /// Records every event field. #[derive(Clone, Default)] struct CapturedLog(std::sync::Arc>); @@ -526,8 +506,7 @@ mod tests { fn exit(&self, _: &tracing::Id) {} } - /// `aws_sigv4` traces the headers it is given, and its redaction list does - /// not cover the `Original-` copy of a relocated bearer token. + /// `aws_sigv4` does not redact `Original-Authorization` itself. #[test] fn signing_does_not_trace_a_relocated_bearer_token() { use chrono::TimeZone; @@ -556,8 +535,7 @@ mod tests { Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), ) .unwrap(); - // Without this the assertion below would pass even if nothing was - // ever captured. + // Proves the capture works. tracing::trace!(canary = "subscriber-is-live"); }); @@ -595,8 +573,8 @@ mod tests { assert!(err.message().contains("x-amz-meta-tenant"), "{err}"); } - /// A caller-supplied `Original-x-amz-*` must not be signed: relocation - /// appends to it afterwards, which would change a signed value on the wire. + /// Relocation appends to a caller's `Original-x-amz-*` after signing, so it + /// must stay unsigned. #[test] fn a_caller_supplied_relocation_header_is_not_signed() { use chrono::TimeZone; @@ -652,8 +630,7 @@ mod tests { fn userinfo_is_stripped_before_signing() { use chrono::TimeZone; - // `HttpRequest::new` is public, so a hand-built request can carry - // userinfo that the wire Host never has. + // A hand-built request can carry userinfo; the wire `Host` never does. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new(reqwest::Request::new( reqwest::Method::GET, @@ -678,8 +655,8 @@ mod tests { fn a_doubled_slash_in_the_path_is_normalized() { use chrono::TimeZone; - // A catalog URI with a trailing slash produces `//v1/...`; the signed - // path has to collapse it the way `Aws4Signer` does. + // A trailing slash on the catalog URI gives `//v1/...`, which + // `Aws4Signer` collapses. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new(reqwest::Request::new( reqwest::Method::GET, @@ -699,8 +676,7 @@ mod tests { fn caller_headers_the_signer_overwrites_are_relocated() { use chrono::TimeZone; - // Java's `updateRequestHeaders` moves a conflicting caller value to - // `Original-` rather than dropping it, credentials included. + // As in Java, conflicting caller values move to `Original-`. let creds = aws_credential_types::Credentials::new( "ak".to_string(), "sk".to_string(), @@ -762,9 +738,8 @@ mod tests { fn an_existing_authorization_is_never_signed() { use chrono::TimeZone; - // `authorization` must stay out of `SignedHeaders`: the signer replaces - // it, so signing the caller's value would guarantee a mismatch. The - // crate's own defaults carry that exclusion. + // The signer replaces `authorization`, and a proxy may rewrite + // `user-agent`, so neither is signed. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() @@ -799,16 +774,14 @@ mod tests { for excluded in ["authorization", "user-agent"] { assert!(!signed.split(';').any(|h| h == excluded), "{signed}"); } - // The relocated copy, on the other hand, is signed over — that is the - // point of renaming it before signing rather than after. + // The relocated copy is signed: it is renamed before signing. assert!( signed.split(';').any(|h| h == "original-authorization"), "{signed}" ); } - /// Java groups all `Authorization` values under the relocated name, so - /// repeated credentials must survive together and stay redacted. + /// As in Java, every `Authorization` value is relocated, and stays redacted. #[test] fn every_repeated_authorization_is_relocated_and_kept_sensitive() { use chrono::TimeZone; @@ -846,9 +819,7 @@ mod tests { fn hop_by_hop_headers_are_not_signed() { use chrono::TimeZone; - // A proxy or an HTTP/2 hop may drop or rewrite these, so signing them - // would make the request fail verification. Java's `AbstractAws4Signer` - // ignores the first two as well. + // A proxy may drop or rewrite these, so they are not signed. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() @@ -892,8 +863,7 @@ mod tests { fn signed_credentials_are_marked_sensitive() { use chrono::TimeZone; - // Both carry a credential, so a `Debug`-formatted request must not - // print them. + // Both carry a credential, so `Debug` must not print them. let creds = aws_credential_types::Credentials::new( "ak".to_string(), "sk".to_string(), @@ -932,8 +902,7 @@ mod tests { fn a_caller_content_hash_is_relocated_not_dropped() { use chrono::TimeZone; - // The signer overwrites `x-amz-content-sha256`; the caller's value - // moves aside instead of vanishing, after signing as Java does. + // The caller's value moves aside rather than vanishing. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() @@ -965,8 +934,8 @@ mod tests { fn signs_with_a_non_default_service_and_session_token() { use chrono::TimeZone; - // What a non-AWS S3-compatible catalog vends: its own signing name - // rather than `execute-api`, its own region, and STS credentials. + // As a non-AWS catalog might vend: its own signing name, and STS + // credentials. let creds = aws_credential_types::Credentials::new( "STS.EXAMPLEACCESSKEYID", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", @@ -1052,8 +1021,8 @@ mod tests { ); } - /// Empty body uses the hex constant and existing headers are signed too - /// (mirrors Java's `TestRESTSigV4AuthSession::authenticateWithoutBody`). + /// An empty body hashes to the hex constant, and caller headers are signed + /// (Java's `authenticateWithoutBody`). #[test] fn signs_empty_body_and_all_headers() { use chrono::TimeZone; @@ -1098,20 +1067,9 @@ mod tests { ); } - /// The signed `host` must include an explicit non-default port, matching - /// what reqwest/hyper put on the wire and what the AWS SDK signs. #[test] fn iceberg_mode_signs_the_hex_payload_hash_not_the_base64_header() { - // The IcebergRest split: `x-amz-content-sha256` carries base64, but the - // canonical request must hash in hex. A body is required to tell them - // apart — every other signing test uses an empty one, where the header - // is the hex constant and the two values coincide. - // - // Java has no counterpart: there the split lives inside the AWS SDK - // (`SignerChecksumParams` puts a base64 checksum in the header while - // `Aws4Signer` canonicalizes hex), so `TestRESTSigV4Signer` only checks - // that the header is present. Reimplementing the signer makes the - // invariant ours to keep. + // Needs a body: without one the header is the hex constant too. use chrono::TimeZone; let creds = aws_credential_types::Credentials::new( @@ -1149,10 +1107,8 @@ mod tests { ); } - /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1, AWS SDK - /// 2.29.52, an `AuthSession.EMPTY` delegate, these credentials as `rest.*` - /// properties) sent for the same requests: a check against an independent - /// implementation, where the other pins in this file only catch changes. + /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1) sent for + /// these requests, given these credentials as `rest.*` properties. #[test] fn signatures_match_iceberg_java() { use chrono::TimeZone; @@ -1207,6 +1163,7 @@ mod tests { ); } + /// The signed `host` keeps a non-default port, as on the wire. #[test] fn signs_host_with_non_default_port() { use chrono::TimeZone; @@ -1235,9 +1192,8 @@ mod tests { ); } - /// AWS SDK v2 parity (`doubleUrlEncode`): the canonical URI encodes the - /// serialized path once more — literal `,` becomes `%2C`, an encoded - /// `%2C` becomes `%252C` — while plain paths stay byte-identical. + /// Like `Aws4Signer`, the canonical path is encoded again: `,` becomes `%2C` + /// and `%2C` becomes `%252C`. #[test] fn canonical_uri_is_aws_double_encoded() { use chrono::TimeZone; @@ -1266,8 +1222,8 @@ mod tests { ); } - /// The AWS SigV4 test suite's `post-x-www-form-urlencoded` case, which - /// signs a hex `x-amz-content-sha256` as this mode does. + /// The AWS SigV4 test suite's `post-x-www-form-urlencoded`, which signs a + /// hex `x-amz-content-sha256` like this mode. #[test] fn standard_mode_matches_the_aws_test_suite() { use chrono::TimeZone; diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs index 872355fb73..8c6dd7f571 100644 --- a/crates/catalog/rest/src/request.rs +++ b/crates/catalog/rest/src/request.rs @@ -56,8 +56,7 @@ impl HttpRequest { self.inner.url() } - /// The mutable request URL, for signers that must rewrite it before - /// canonicalizing, e.g. to drop userinfo the wire `Host` never carries. + /// The mutable request URL, for signers that rewrite it. #[cfg(feature = "sigv4")] pub(crate) fn url_mut(&mut self) -> &mut Url { self.inner.url_mut() From e241d07813e97458f8a0e5338d38c855447504cd Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 7 Oct 2026 21:55:02 -0400 Subject: [PATCH 24/30] refactor(rest): build SigV4Signer with a typed builder --- crates/catalog/rest/src/auth/sigv4.rs | 89 +++++++++++++++++++-------- 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 2d5cc8740c..d5f0f0ca1b 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -20,6 +20,7 @@ use chrono::{DateTime, Utc}; use iceberg::{Error, ErrorKind, Result}; use sha2::{Digest, Sha256}; +use typed_builder::TypedBuilder; /// Hex SHA-256 of the empty string. const EMPTY_BODY_HEX_SHA256: &str = @@ -63,27 +64,21 @@ fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { /// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` /// does. Carries no credentials, so one signer serves every session. -#[derive(Clone)] +/// +/// Built with [`SigV4Signer::builder`]. +#[derive(Clone, TypedBuilder)] pub struct SigV4Signer { + /// The signing region, e.g. `us-east-1`. + #[builder(setter(into))] region: String, + /// The signing name, e.g. `execute-api`. + #[builder(setter(into))] service: String, + /// How the payload hash is encoded. mode: PayloadHashMode, } impl SigV4Signer { - /// Creates a new SigV4 signer. - pub fn new( - region: impl Into, - service: impl Into, - mode: PayloadHashMode, - ) -> Self { - Self { - region: region.into(), - service: service.into(), - mode, - } - } - /// Signs `request` in place. An existing `Authorization` moves to /// `Original-Authorization`, and a caller's `x-amz-date`, /// `x-amz-content-sha256` or `x-amz-security-token` that the signer @@ -390,7 +385,11 @@ mod tests { ); assert!(request.url().query().unwrap().contains("my+ns")); - let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(PayloadHashMode::StandardAws) + .build(); let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); signer .sign_at(&mut request, &test_credentials(), now) @@ -473,7 +472,11 @@ mod tests { } fn test_signer(mode: PayloadHashMode) -> SigV4Signer { - SigV4Signer::new("us-east-1", "execute-api", mode) + SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(mode) + .build() } fn test_credentials() -> aws_credential_types::Credentials { @@ -684,7 +687,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(PayloadHashMode::StandardAws) + .build(); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -871,7 +878,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::StandardAws); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(PayloadHashMode::StandardAws) + .build(); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -943,7 +954,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "custom-service", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("custom-service") + .mode(PayloadHashMode::IcebergRest) + .build(); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://catalog.example.com/v1/config?warehouse=my-catalog") @@ -985,7 +1000,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1034,7 +1053,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1079,7 +1102,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "execute-api", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(PayloadHashMode::IcebergRest) + .build(); let body = br#"{"namespace":["ns"]}"#; let mut req = HttpRequest::new( reqwest::Client::new() @@ -1175,7 +1202,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1205,7 +1236,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "glue", PayloadHashMode::IcebergRest); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); let client = reqwest::Client::new(); let mut req = HttpRequest::new( client @@ -1235,7 +1270,11 @@ mod tests { None, "test", ); - let signer = SigV4Signer::new("us-east-1", "service", PayloadHashMode::StandardAws); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("service") + .mode(PayloadHashMode::StandardAws) + .build(); let mut req = HttpRequest::new( reqwest::Client::new() .post("https://example.amazonaws.com/") From ee1a9fc48624c304cd2395830611cbc0565af1bb Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 7 Oct 2026 21:55:10 -0400 Subject: [PATCH 25/30] fix(rest): propagate a failed userinfo strip instead of signing the URL as is --- crates/catalog/rest/src/auth/sigv4.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index d5f0f0ca1b..8026d5c7e0 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -137,7 +137,7 @@ impl SigV4Signer { })?; request.headers_mut().insert(CONTENT_SHA256, content_value); - rewrite_url_for_signing(request); + rewrite_url_for_signing(request)?; let identity = credentials.clone().into(); let params = v4::SigningParams::builder() @@ -214,16 +214,23 @@ fn signable_headers(request: &crate::HttpRequest) -> Result> { /// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the /// query as `%20`: a space to AWS and Java either way, but unambiguous to any /// verifier. -fn rewrite_url_for_signing(request: &mut crate::HttpRequest) { +fn rewrite_url_for_signing(request: &mut crate::HttpRequest) -> Result<()> { if !request.url().username().is_empty() || request.url().password().is_some() { let url = request.url_mut(); - let _ = url.set_username(""); - let _ = url.set_password(None); + url.set_username("") + .and_then(|()| url.set_password(None)) + .map_err(|()| { + Error::new( + ErrorKind::DataInvalid, + "cannot strip userinfo from the request URL", + ) + })?; } if let Some(query) = request.url().query().filter(|q| q.contains('+')) { let unambiguous = query.replace('+', "%20"); request.url_mut().set_query(Some(&unambiguous)); } + Ok(()) } /// `Aws4Signer`'s settings: normalized double-encoded path, and Java's ignore From 7a79bfc9e0e2266fb7c46f13b0122367e47327bd Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 7 Oct 2026 21:55:11 -0400 Subject: [PATCH 26/30] docs(rest): name the flag the tracing mute is gated to avoid --- crates/catalog/rest/src/auth/sigv4.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 8026d5c7e0..4e89fb71a6 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -165,8 +165,9 @@ impl SigV4Signer { // `aws_sigv4` traces the request, `Original-Authorization` included. // Mute it only when a subscriber could record that (the max level is - // `OFF` until one is registered): `with_default` marks tracing as in - // use for good, which turns off its `log` fallback process-wide. + // `OFF` until one is registered): `with_default` also sets + // tracing-core's `EXISTS` flag, which nothing clears, and with the + // `log` feature tracing then stops forwarding events to `log` for good. let signed = if LevelFilter::current() == LevelFilter::TRACE { tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) } else { From edc9d36b3d7924ce6795113bb21c9700b349e202 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 7 Oct 2026 21:55:13 -0400 Subject: [PATCH 27/30] docs(rest): say that as_bytes merges an absent body with an empty one --- crates/catalog/rest/src/request.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs index 8c6dd7f571..a2a8907a41 100644 --- a/crates/catalog/rest/src/request.rs +++ b/crates/catalog/rest/src/request.rs @@ -110,6 +110,10 @@ pub enum HttpRequestBody<'a> { impl<'a> HttpRequestBody<'a> { /// The signable bytes: empty for [`Self::Empty`], the buffer for /// [`Self::Buffered`], and `None` for [`Self::Streaming`]. + /// + /// An absent body and an empty one both give `&[]`. Where the two must + /// differ, as in Iceberg Java's SigV4 payload hashing, match on the + /// variants instead. pub fn as_bytes(&self) -> Option<&'a [u8]> { match self { HttpRequestBody::Empty => Some(&[]), From b367eb398f8c4e2727215f35c2cc1f2ad44f2e14 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 7 Oct 2026 21:55:33 -0400 Subject: [PATCH 28/30] chore(rest): update the public API for the signer builder --- crates/catalog/rest/public-api.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 43bd5ae146..733a44727d 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -345,12 +345,13 @@ impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_catalog_rest::SigV4Signer impl iceberg_catalog_rest::SigV4Signer -pub fn iceberg_catalog_rest::SigV4Signer::new(region: impl core::convert::Into, service: impl core::convert::Into, mode: iceberg_catalog_rest::PayloadHashMode) -> Self pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())> pub struct iceberg_catalog_rest::StorageCredential pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String From 0dab39cf12089cf830936fb56a6e964e7050e1c1 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Thu, 8 Oct 2026 22:03:44 -0400 Subject: [PATCH 29/30] fix(rest): validate requests before signing --- crates/catalog/rest/public-api.txt | 1 + crates/catalog/rest/src/auth/mod.rs | 2 + crates/catalog/rest/src/auth/sigv4.rs | 206 ++++++++++++++------------ 3 files changed, 111 insertions(+), 98 deletions(-) diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index 733a44727d..ddfafb11d6 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -1,4 +1,5 @@ pub mod iceberg_catalog_rest +pub use iceberg_catalog_rest::Credentials pub enum iceberg_catalog_rest::HttpRequestBody<'a> pub iceberg_catalog_rest::HttpRequestBody::Buffered(&'a [u8]) pub iceberg_catalog_rest::HttpRequestBody::Empty diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index cc6910483a..a3aab4df90 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -27,6 +27,8 @@ use std::fmt::Debug; use std::sync::Arc; use async_trait::async_trait; +#[cfg(feature = "sigv4")] +pub use aws_credential_types::Credentials; use iceberg::{Result, SessionContext}; pub use oauth2::OAuth2Manager; #[cfg(feature = "sigv4")] diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 4e89fb71a6..8984ea5485 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -22,9 +22,7 @@ use iceberg::{Error, ErrorKind, Result}; use sha2::{Digest, Sha256}; use typed_builder::TypedBuilder; -/// Hex SHA-256 of the empty string. -const EMPTY_BODY_HEX_SHA256: &str = - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +use super::Credentials; /// How the payload hash is encoded in the `x-amz-content-sha256` header. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -39,10 +37,6 @@ pub enum PayloadHashMode { StandardAws, } -fn hex_sha256(data: &[u8]) -> String { - encode_hex(&Sha256::digest(data)) -} - fn encode_hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -51,15 +45,15 @@ fn base64_encode(bytes: &[u8]) -> String { base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) } -/// The `x-amz-content-sha256` value; `None` is no body at all. -fn content_sha256_header(body: Option<&[u8]>, mode: PayloadHashMode) -> String { - match mode { - PayloadHashMode::StandardAws => hex_sha256(body.unwrap_or_default()), - PayloadHashMode::IcebergRest => match body { - None => EMPTY_BODY_HEX_SHA256.to_string(), - Some(body) => base64_encode(&Sha256::digest(body)), - }, - } +/// The content header and canonical payload hash, computed from one digest. +fn payload_hashes(body: Option<&[u8]>, mode: PayloadHashMode) -> (String, String) { + let digest = Sha256::digest(body.unwrap_or_default()); + let hex = encode_hex(&digest); + let header = match mode { + PayloadHashMode::IcebergRest if body.is_some() => base64_encode(&digest), + _ => hex.clone(), + }; + (header, hex) } /// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` @@ -89,8 +83,9 @@ impl SigV4Signer { /// Uses `credentials` as given and never refreshes them: resolve temporary /// ones from their provider before each call. /// - /// Fails on a streaming body or a non-UTF-8 header, which cannot be - /// canonicalized faithfully. + /// A streaming body or non-UTF-8 header is rejected before modifying the + /// request. Other signing errors may leave it partially modified; rebuild + /// the request before retrying. /// /// Send the result through a client that does not follow redirects: a /// redirect replays the signature, and across hosts reqwest drops @@ -100,18 +95,14 @@ impl SigV4Signer { /// A `tracing` subscriber is muted for the call, but the `log` bridge (no /// subscriber, or `log-always`) still forwards those events, so keep /// `aws_sigv4` below trace level there. - pub fn sign( - &self, - request: &mut crate::HttpRequest, - credentials: &aws_credential_types::Credentials, - ) -> Result<()> { + pub fn sign(&self, request: &mut crate::HttpRequest, credentials: &Credentials) -> Result<()> { self.sign_at(request, credentials, Utc::now()) } fn sign_at( &self, request: &mut crate::HttpRequest, - credentials: &aws_credential_types::Credentials, + credentials: &Credentials, now: DateTime, ) -> Result<()> { use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; @@ -119,8 +110,9 @@ impl SigV4Signer { use tracing::level_filters::LevelFilter; use tracing::subscriber::NoSubscriber; - let body = signable_body(request)?; - let content_header = content_sha256_header(body.as_deref(), self.mode); + // Validate before relocating or replacing any caller headers. + signable_headers(request).try_for_each(|header| header.map(|_| ()))?; + let (content_header, payload_hash) = payload_hashes(signable_body(request)?, self.mode); convert_headers(request); @@ -152,12 +144,12 @@ impl SigV4Signer { })? .into(); - let headers = signable_headers(request)?; + let headers = signable_headers(request).collect::>>()?; let signable = SignableRequest::new( request.method().as_str(), request.url_str(), headers.into_iter(), - SignableBody::Bytes(body.as_deref().unwrap_or_default()), + SignableBody::Precomputed(payload_hash), ) .map_err(|e| { Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) @@ -182,10 +174,10 @@ impl SigV4Signer { } /// The body to sign; as in Java, an absent body and an empty one differ. -fn signable_body(request: &crate::HttpRequest) -> Result>> { +fn signable_body(request: &crate::HttpRequest) -> Result> { match request.body() { crate::HttpRequestBody::Empty => Ok(None), - crate::HttpRequestBody::Buffered(bytes) => Ok(Some(bytes.to_vec())), + crate::HttpRequestBody::Buffered(bytes) => Ok(Some(bytes)), crate::HttpRequestBody::Streaming => Err(Error::new( ErrorKind::FeatureUnsupported, "cannot sign a streaming request body", @@ -195,21 +187,17 @@ fn signable_body(request: &crate::HttpRequest) -> Result>> { /// The headers to sign. A non-UTF-8 one is an error: skipping it would send /// it unsigned. -fn signable_headers(request: &crate::HttpRequest) -> Result> { - request - .headers() - .iter() - .map(|(n, v)| { - let v = v.to_str().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("cannot sign non-UTF-8 header value for `{n}`"), - ) - .with_source(e) - })?; - Ok((n.as_str(), v)) - }) - .collect() +fn signable_headers(request: &crate::HttpRequest) -> impl Iterator> { + request.headers().iter().map(|(n, v)| { + let v = v.to_str().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("cannot sign non-UTF-8 header value for `{n}`"), + ) + .with_source(e) + })?; + Ok((n.as_str(), v)) + }) } /// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the @@ -244,7 +232,7 @@ fn signing_settings() -> aws_sigv4::http_request::SigningSettings { let mut settings = SigningSettings::default(); settings.percent_encoding_mode = PercentEncodingMode::Double; settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; - // We set the header ourselves: base64 for IcebergRest. + // We set and relocate the content hash ourselves, outside the instructions. settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; let mut excluded = settings.excluded_headers.take().unwrap_or_default(); excluded.extend([ @@ -336,7 +324,6 @@ const RELOCATED_SECURITY_TOKEN: reqwest::header::HeaderName = fn relocated_name(name: &str) -> Option { match name { n if n == AMZ_DATE => Some(RELOCATED_AMZ_DATE), - n if n == CONTENT_SHA256 => Some(RELOCATED_CONTENT_SHA256), n if n == SECURITY_TOKEN => Some(RELOCATED_SECURITY_TOKEN), _ => None, } @@ -413,22 +400,19 @@ mod tests { } #[test] - fn content_sha256_header_iceberg_mode() { - let v = content_sha256_header(Some(b"hello"), PayloadHashMode::IcebergRest); + fn payload_hashes_iceberg_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::IcebergRest).0; assert_eq!(v, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="); - let e = content_sha256_header(None, PayloadHashMode::IcebergRest); + let e = payload_hashes(None, PayloadHashMode::IcebergRest).0; assert_eq!(e, EMPTY_HEX); } /// As in Java, an empty body is hashed, unlike an absent one. #[test] - fn content_sha256_header_separates_an_empty_body_from_an_absent_one() { - let empty = content_sha256_header(Some(b""), PayloadHashMode::IcebergRest); + fn payload_hashes_separates_an_empty_body_from_an_absent_one() { + let empty = payload_hashes(Some(b""), PayloadHashMode::IcebergRest).0; assert_eq!(empty, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="); - assert_ne!( - empty, - content_sha256_header(None, PayloadHashMode::IcebergRest) - ); + assert_ne!(empty, payload_hashes(None, PayloadHashMode::IcebergRest).0); } /// The same, through `sign_at`. @@ -459,8 +443,8 @@ mod tests { } #[test] - fn content_sha256_header_standard_mode() { - let v = content_sha256_header(Some(b"hello"), PayloadHashMode::StandardAws); + fn payload_hashes_standard_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::StandardAws).0; assert_eq!( v, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" @@ -487,8 +471,8 @@ mod tests { .build() } - fn test_credentials() -> aws_credential_types::Credentials { - aws_credential_types::Credentials::new("ak", "sk", None::, None, "test") + fn test_credentials() -> Credentials { + Credentials::new("ak", "sk", None::, None, "test") } /// Records every event field. @@ -558,30 +542,40 @@ mod tests { } #[test] - fn a_non_utf8_header_value_is_rejected_rather_than_left_unsigned() { + fn a_non_utf8_header_is_rejected_without_changing_the_request() { use chrono::TimeZone; let signer = test_signer(PayloadHashMode::IcebergRest); - let mut req = HttpRequest::new( - reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header( - "x-amz-meta-tenant", - reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), - ) - .build() - .unwrap(), - ); + for name in ["x-amz-meta-tenant", "authorization", "x-amz-content-sha256"] { + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config?warehouse=my+catalog" + .parse() + .unwrap(), + )); + req.headers_mut() + .insert("authorization", "Bearer delegate-token".parse().unwrap()); + req.headers_mut() + .insert("x-amz-content-sha256", "caller-hash".parse().unwrap()); + req.headers_mut().insert( + name, + reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); - let err = signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap_err(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("x-amz-meta-tenant"), "{err}"); + let err = signer + .sign_at( + &mut req, + &test_credentials(), + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(name), "{err}"); + assert_eq!(req.headers(), &headers, "{name}"); + assert_eq!(req.url(), &url, "{name}"); + } } /// Relocation appends to a caller's `Original-x-amz-*` after signing, so it @@ -656,9 +650,16 @@ mod tests { assert_eq!(req.url().username(), ""); assert_eq!(req.url().password(), None); - assert_signature_is( - &req, - "0f4a3487bcff9dd16bf0a42d06c24dc49b2366e8a928dc9666f8424cf5b306b3", + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), ); } @@ -677,9 +678,16 @@ mod tests { signer.sign_at(&mut req, &test_credentials(), now).unwrap(); - assert_signature_is( - &req, - "0f4a3487bcff9dd16bf0a42d06c24dc49b2366e8a928dc9666f8424cf5b306b3", + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), ); } @@ -688,7 +696,7 @@ mod tests { use chrono::TimeZone; // As in Java, conflicting caller values move to `Original-`. - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "ak".to_string(), "sk".to_string(), Some("signer-token".to_string()), @@ -879,7 +887,7 @@ mod tests { use chrono::TimeZone; // Both carry a credential, so `Debug` must not print them. - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "ak".to_string(), "sk".to_string(), Some("session-token".to_string()), @@ -955,7 +963,7 @@ mod tests { // As a non-AWS catalog might vend: its own signing name, and STS // credentials. - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "STS.EXAMPLEACCESSKEYID", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", Some("example-session-token".to_string()), @@ -1001,7 +1009,7 @@ mod tests { fn signs_request_iceberg_mode() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", Some("SESSIONTOKEN".to_string()), @@ -1054,7 +1062,7 @@ mod tests { fn signs_empty_body_and_all_headers() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", None::, @@ -1103,7 +1111,7 @@ mod tests { // Needs a body: without one the header is the hex constant too. use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", None::, @@ -1130,11 +1138,13 @@ mod tests { // The header carries base64 while the pinned signature covers hex. assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), - content_sha256_header(Some(body), PayloadHashMode::IcebergRest).as_str() + payload_hashes(Some(body), PayloadHashMode::IcebergRest) + .0 + .as_str() ); assert_ne!( req.headers().get("x-amz-content-sha256").unwrap(), - hex_sha256(body).as_str() + encode_hex(&Sha256::digest(body)).as_str() ); assert_signature_is( &req, @@ -1148,7 +1158,7 @@ mod tests { fn signatures_match_iceberg_java() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", Some("example-session-token".to_string()), @@ -1203,7 +1213,7 @@ mod tests { fn signs_host_with_non_default_port() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", None::, @@ -1237,7 +1247,7 @@ mod tests { fn canonical_uri_is_aws_double_encoded() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", None::, @@ -1271,7 +1281,7 @@ mod tests { fn standard_mode_matches_the_aws_test_suite() { use chrono::TimeZone; - let creds = aws_credential_types::Credentials::new( + let creds = Credentials::new( "AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", None::, From 693de9d34047c8c1b7ee100cec4ef11c0a4f5800 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 9 Oct 2026 10:32:11 -0700 Subject: [PATCH 30/30] fix(rest): address SigV4 review findings --- crates/catalog/rest/src/auth/sigv4.rs | 568 ++++++++++---------------- crates/catalog/rest/src/request.rs | 4 - 2 files changed, 220 insertions(+), 352 deletions(-) diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs index 8984ea5485..b7b4ae5bf1 100644 --- a/crates/catalog/rest/src/auth/sigv4.rs +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -60,7 +60,7 @@ fn payload_hashes(body: Option<&[u8]>, mode: PayloadHashMode) -> (String, String /// does. Carries no credentials, so one signer serves every session. /// /// Built with [`SigV4Signer::builder`]. -#[derive(Clone, TypedBuilder)] +#[derive(Clone, Debug, TypedBuilder)] pub struct SigV4Signer { /// The signing region, e.g. `us-east-1`. #[builder(setter(into))] @@ -83,9 +83,8 @@ impl SigV4Signer { /// Uses `credentials` as given and never refreshes them: resolve temporary /// ones from their provider before each call. /// - /// A streaming body or non-UTF-8 header is rejected before modifying the - /// request. Other signing errors may leave it partially modified; rebuild - /// the request before retrying. + /// On error, including for a streaming body or a non-UTF-8 header, the + /// request is left unchanged. /// /// Send the result through a client that does not follow redirects: a /// redirect replays the signature, and across hosts reqwest drops @@ -104,13 +103,29 @@ impl SigV4Signer { request: &mut crate::HttpRequest, credentials: &Credentials, now: DateTime, + ) -> Result<()> { + let headers = request.headers().clone(); + let url = request.url().clone(); + let signed = self.sign_in_place(request, credentials, now); + if signed.is_err() { + *request.headers_mut() = headers; + *request.url_mut() = url; + } + signed + } + + fn sign_in_place( + &self, + request: &mut crate::HttpRequest, + credentials: &Credentials, + now: DateTime, ) -> Result<()> { use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; use aws_sigv4::sign::v4; + use tracing::dispatcher::Dispatch; use tracing::level_filters::LevelFilter; - use tracing::subscriber::NoSubscriber; - // Validate before relocating or replacing any caller headers. + // Before the content hash is displaced, which takes it out of signing. signable_headers(request).try_for_each(|header| header.map(|_| ()))?; let (content_header, payload_hash) = payload_hashes(signable_body(request)?, self.mode); @@ -161,7 +176,7 @@ impl SigV4Signer { // tracing-core's `EXISTS` flag, which nothing clears, and with the // `log` feature tracing then stops forwarding events to `log` for good. let signed = if LevelFilter::current() == LevelFilter::TRACE { - tracing::subscriber::with_default(NoSubscriber::default(), || sign(signable, ¶ms)) + tracing::dispatcher::with_default(&Dispatch::none(), || sign(signable, ¶ms)) } else { sign(signable, ¶ms) }; @@ -189,7 +204,7 @@ fn signable_body(request: &crate::HttpRequest) -> Result> { /// it unsigned. fn signable_headers(request: &crate::HttpRequest) -> impl Iterator> { request.headers().iter().map(|(n, v)| { - let v = v.to_str().map_err(|e| { + let v = std::str::from_utf8(v.as_bytes()).map_err(|e| { Error::new( ErrorKind::DataInvalid, format!("cannot sign non-UTF-8 header value for `{n}`"), @@ -349,18 +364,10 @@ fn relocate_conflicting( } } -impl std::fmt::Debug for SigV4Signer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SigV4Signer") - .field("region", &self.region) - .field("service", &self.service) - .field("mode", &self.mode) - .finish_non_exhaustive() - } -} - #[cfg(test)] mod tests { + use chrono::TimeZone; + use super::*; use crate::HttpRequest; @@ -368,8 +375,6 @@ mod tests { #[test] fn signing_rewrites_an_ambiguous_plus_out_of_the_query() { - use chrono::TimeZone; - // reqwest writes a space as `+`; signing makes it `%20`. let mut request = HttpRequest::new( reqwest::Client::new() @@ -380,12 +385,8 @@ mod tests { ); assert!(request.url().query().unwrap().contains("my+ns")); - let signer = SigV4Signer::builder() - .region("us-east-1") - .service("execute-api") - .mode(PayloadHashMode::StandardAws) - .build(); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let signer = test_signer(PayloadHashMode::StandardAws); + let now = test_time(); signer .sign_at(&mut request, &test_credentials(), now) .unwrap(); @@ -409,19 +410,9 @@ mod tests { /// As in Java, an empty body is hashed, unlike an absent one. #[test] - fn payload_hashes_separates_an_empty_body_from_an_absent_one() { - let empty = payload_hashes(Some(b""), PayloadHashMode::IcebergRest).0; - assert_eq!(empty, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="); - assert_ne!(empty, payload_hashes(None, PayloadHashMode::IcebergRest).0); - } - - /// The same, through `sign_at`. - #[test] fn signing_separates_an_empty_body_from_an_absent_one() { - use chrono::TimeZone; - let signer = test_signer(PayloadHashMode::IcebergRest); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); let hash_of = |builder: reqwest::RequestBuilder| { let mut req = HttpRequest::new(builder.build().unwrap()); signer.sign_at(&mut req, &test_credentials(), now).unwrap(); @@ -475,6 +466,52 @@ mod tests { Credentials::new("ak", "sk", None::, None, "test") } + fn example_credentials(session_token: Option<&str>) -> Credentials { + Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + session_token.map(str::to_string), + None, + "test", + ) + } + + fn test_time() -> DateTime { + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap() + } + + fn signed_headers(req: &HttpRequest) -> Vec { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth.split("SignedHeaders=").nth(1).unwrap(); + let signed = signed.split(',').next().unwrap(); + signed.split(';').map(str::to_string).collect() + } + + /// Every header value as `(name, value)`, sorted. + fn header_list(req: &HttpRequest) -> Vec<(String, String)> { + let mut headers: Vec<_> = req + .headers() + .iter() + .map(|(n, v)| (n.to_string(), v.to_str().unwrap().to_string())) + .collect(); + headers.sort(); + headers + } + + fn sorted_pairs(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + let mut pairs: Vec<_> = pairs + .iter() + .map(|(n, v)| (n.to_string(), v.to_string())) + .collect(); + pairs.sort(); + pairs + } + /// Records every event field. #[derive(Clone, Default)] struct CapturedLog(std::sync::Arc>); @@ -504,8 +541,6 @@ mod tests { /// `aws_sigv4` does not redact `Original-Authorization` itself. #[test] fn signing_does_not_trace_a_relocated_bearer_token() { - use chrono::TimeZone; - const TOKEN: &str = "Bearer topsecretdelegatetoken"; let signer = test_signer(PayloadHashMode::IcebergRest); let mut req = HttpRequest::new( @@ -524,27 +559,19 @@ mod tests { tracing::level_filters::LevelFilter::TRACE ); signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) + .sign_at(&mut req, &test_credentials(), test_time()) .unwrap(); - // Proves the capture works. tracing::trace!(canary = "subscriber-is-live"); }); let captured = log.0.lock().unwrap().clone(); assert!(captured.contains("subscriber-is-live"), "captured nothing"); assert!(!captured.contains(TOKEN), "{captured}"); - // The token still travels, it is just not logged. assert_eq!(req.headers().get(RELOCATED_AUTHORIZATION).unwrap(), TOKEN); } #[test] fn a_non_utf8_header_is_rejected_without_changing_the_request() { - use chrono::TimeZone; - let signer = test_signer(PayloadHashMode::IcebergRest); for name in ["x-amz-meta-tenant", "authorization", "x-amz-content-sha256"] { let mut req = HttpRequest::new(reqwest::Request::new( @@ -565,11 +592,7 @@ mod tests { let url = req.url().clone(); let err = signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) + .sign_at(&mut req, &test_credentials(), test_time()) .unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains(name), "{err}"); @@ -578,12 +601,63 @@ mod tests { } } + #[test] + fn a_non_ascii_utf8_header_is_signed() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + req.headers_mut().insert( + "x-tenant", + reqwest::header::HeaderValue::from_bytes("Zürich".as_bytes()).unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); + } + + #[test] + fn a_signing_failure_after_relocation_restores_the_request() { + let signer = test_signer(PayloadHashMode::IcebergRest); + // Longer than `http::Uri` accepts, so signing fails after the headers + // and query were rewritten. + let url = format!( + "https://rest.example.com/v1/config?warehouse=my+catalog&pad={}", + "a".repeat(70_000) + ); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get(url) + .header("authorization", "Bearer delegate-token") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); + + let err = signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + assert_eq!(req.headers(), &headers); + assert_eq!(req.url(), &url); + } + /// Relocation appends to a caller's `Original-x-amz-*` after signing, so it /// must stay unsigned. #[test] fn a_caller_supplied_relocation_header_is_not_signed() { - use chrono::TimeZone; - let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() @@ -595,33 +669,14 @@ mod tests { ); signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) + .sign_at(&mut req, &test_credentials(), test_time()) .unwrap(); - let auth = req - .headers() - .get("authorization") - .unwrap() - .to_str() - .unwrap(); - let signed = auth - .split("SignedHeaders=") - .nth(1) - .unwrap() - .split(',') - .next() - .unwrap(); - assert!( - !signed - .split(';') - .any(|h| h == "original-x-amz-content-sha256"), - "{signed}" - ); - // Both values still travel, they are just outside the signature. + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date" + ]); let relocated: Vec<_> = req .headers() .get_all("original-x-amz-content-sha256") @@ -633,8 +688,6 @@ mod tests { #[test] fn userinfo_is_stripped_before_signing() { - use chrono::TimeZone; - // A hand-built request can carry userinfo; the wire `Host` never does. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new(reqwest::Request::new( @@ -644,7 +697,7 @@ mod tests { .unwrap(), )); assert_eq!(req.url().username(), "user"); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &test_credentials(), now).unwrap(); @@ -665,8 +718,6 @@ mod tests { #[test] fn a_doubled_slash_in_the_path_is_normalized() { - use chrono::TimeZone; - // A trailing slash on the catalog URI gives `//v1/...`, which // `Aws4Signer` collapses. let signer = test_signer(PayloadHashMode::StandardAws); @@ -674,7 +725,7 @@ mod tests { reqwest::Method::GET, "https://rest.example.com//v1//config".parse().unwrap(), )); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &test_credentials(), now).unwrap(); @@ -693,8 +744,6 @@ mod tests { #[test] fn caller_headers_the_signer_overwrites_are_relocated() { - use chrono::TimeZone; - // As in Java, conflicting caller values move to `Original-`. let creds = Credentials::new( "ak".to_string(), @@ -703,11 +752,7 @@ mod tests { None, "test", ); - let signer = SigV4Signer::builder() - .region("us-east-1") - .service("execute-api") - .mode(PayloadHashMode::StandardAws) - .build(); + let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -719,48 +764,35 @@ mod tests { .unwrap(), ); - signer - .sign_at( - &mut req, - &creds, - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); + signer.sign_at(&mut req, &creds, test_time()).unwrap(); - let h = req.headers(); - assert_eq!( - h.get("original-authorization").unwrap(), - "Bearer caller-token" - ); - assert_eq!(h.get("original-x-amz-date").unwrap(), "19700101T000000Z"); assert_eq!( - h.get("original-x-amz-content-sha256").unwrap(), - "caller-hash" + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=ak/20150830/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;original-authorization;x-amz-content-sha256;x-amz-date;\ + x-amz-security-token, \ + Signature=87560cb735277b284a547d63f57cfe3a721f9bce0b08a9402a58d754d2ed707c" + ), + ("original-authorization", "Bearer caller-token"), + ("original-x-amz-content-sha256", "caller-hash"), + ("original-x-amz-date", "19700101T000000Z"), + ("original-x-amz-security-token", "caller-session"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "signer-token"), + ]) ); - let token = h.get("original-x-amz-security-token").unwrap(); - assert_eq!(token, "caller-session"); // Relocated originals may be credentials themselves. - assert!(token.is_sensitive()); - assert!( - h.get("original-x-amz-content-sha256") - .unwrap() - .is_sensitive() - ); - // And the signer's own values took their place. - assert!( - h.get("authorization") - .unwrap() - .to_str() - .unwrap() - .starts_with("AWS4-HMAC-SHA256 ") - ); - assert_eq!(h.get("x-amz-security-token").unwrap(), "signer-token"); + for relocated in [RELOCATED_SECURITY_TOKEN, RELOCATED_CONTENT_SHA256] { + assert!(req.headers().get(relocated).unwrap().is_sensitive()); + } } #[test] fn an_existing_authorization_is_never_signed() { - use chrono::TimeZone; - // The signer replaces `authorization`, and a proxy may rewrite // `user-agent`, so neither is signed. let signer = test_signer(PayloadHashMode::StandardAws); @@ -774,41 +806,20 @@ mod tests { ); signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) + .sign_at(&mut req, &test_credentials(), test_time()) .unwrap(); - let auth = req - .headers() - .get("authorization") - .unwrap() - .to_str() - .unwrap(); - let signed = auth - .split("SignedHeaders=") - .nth(1) - .unwrap() - .split(',') - .next() - .unwrap(); - for excluded in ["authorization", "user-agent"] { - assert!(!signed.split(';').any(|h| h == excluded), "{signed}"); - } - // The relocated copy is signed: it is renamed before signing. - assert!( - signed.split(';').any(|h| h == "original-authorization"), - "{signed}" - ); + assert_eq!(signed_headers(&req), [ + "host", + "original-authorization", + "x-amz-content-sha256", + "x-amz-date" + ]); } /// As in Java, every `Authorization` value is relocated, and stays redacted. #[test] fn every_repeated_authorization_is_relocated_and_kept_sensitive() { - use chrono::TimeZone; - let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() @@ -820,11 +831,7 @@ mod tests { ); signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) + .sign_at(&mut req, &test_credentials(), test_time()) .unwrap(); let relocated: Vec<_> = req @@ -840,8 +847,6 @@ mod tests { #[test] fn hop_by_hop_headers_are_not_signed() { - use chrono::TimeZone; - // A proxy may drop or rewrite these, so they are not signed. let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( @@ -854,28 +859,16 @@ mod tests { .build() .unwrap(), ); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &test_credentials(), now).unwrap(); - let auth = req - .headers() - .get("authorization") - .unwrap() - .to_str() - .unwrap(); - let signed = auth - .split("SignedHeaders=") - .nth(1) - .unwrap() - .split(',') - .next() - .unwrap(); - for skipped in ["expect", "connection", "x-forwarded-for"] { - assert!(!signed.split(';').any(|h| h == skipped), "{signed}"); - } - // An ordinary caller header is still signed. - assert!(signed.split(';').any(|h| h == "x-tenant"), "{signed}"); + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); assert_signature_is( &req, "f938221412ed6b55cf3db380ce6ded476419ad3d7db4c76d932031c98465ce79", @@ -884,8 +877,6 @@ mod tests { #[test] fn signed_credentials_are_marked_sensitive() { - use chrono::TimeZone; - // Both carry a credential, so `Debug` must not print them. let creds = Credentials::new( "ak".to_string(), @@ -894,11 +885,7 @@ mod tests { None, "test", ); - let signer = SigV4Signer::builder() - .region("us-east-1") - .service("execute-api") - .mode(PayloadHashMode::StandardAws) - .build(); + let signer = test_signer(PayloadHashMode::StandardAws); let mut req = HttpRequest::new( reqwest::Client::new() .get("https://rest.example.com/v1/config") @@ -906,13 +893,7 @@ mod tests { .unwrap(), ); - signer - .sign_at( - &mut req, - &creds, - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); + signer.sign_at(&mut req, &creds, test_time()).unwrap(); assert!(req.headers().get("authorization").unwrap().is_sensitive()); assert!( @@ -925,42 +906,8 @@ mod tests { assert!(!debug.contains("session-token"), "{debug}"); } - #[test] - fn a_caller_content_hash_is_relocated_not_dropped() { - use chrono::TimeZone; - - // The caller's value moves aside rather than vanishing. - let signer = test_signer(PayloadHashMode::StandardAws); - let mut req = HttpRequest::new( - reqwest::Client::new() - .get("https://rest.example.com/v1/config") - .header("x-amz-content-sha256", "caller-supplied") - .build() - .unwrap(), - ); - - signer - .sign_at( - &mut req, - &test_credentials(), - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); - - assert_eq!( - req.headers().get("original-x-amz-content-sha256").unwrap(), - "caller-supplied" - ); - assert_eq!( - req.headers().get("x-amz-content-sha256").unwrap(), - EMPTY_HEX - ); - } - #[test] fn signs_with_a_non_default_service_and_session_token() { - use chrono::TimeZone; - // As a non-AWS catalog might vend: its own signing name, and STS // credentials. let creds = Credentials::new( @@ -1007,15 +954,7 @@ mod tests { #[test] fn signs_request_iceberg_mode() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - Some("SESSIONTOKEN".to_string()), - None, - "test", - ); + let creds = example_credentials(Some("SESSIONTOKEN")); let signer = SigV4Signer::builder() .region("us-east-1") .service("glue") @@ -1030,29 +969,24 @@ mod tests { .unwrap(), ); - signer - .sign_at( - &mut req, - &creds, - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); + signer.sign_at(&mut req, &creds, test_time()).unwrap(); - let h = req.headers(); - assert!( - h.get("authorization") - .unwrap() - .to_str() - .unwrap() - .starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") - ); - assert_eq!(h.get("x-amz-date").unwrap(), "20150830T123600Z"); - assert_eq!(h.get("x-amz-security-token").unwrap(), "SESSIONTOKEN"); - let csha = h.get("x-amz-content-sha256").unwrap().to_str().unwrap(); - assert_eq!(csha, "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o="); - assert_signature_is( - &req, - "effad6acde583dd14ba7aff52b2b83776a54421c010fe82067f166819057cb32", + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=effad6acde583dd14ba7aff52b2b83776a54421c010fe82067f166819057cb32" + ), + ( + "x-amz-content-sha256", + "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o=" + ), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "SESSIONTOKEN"), + ]) ); } @@ -1060,15 +994,7 @@ mod tests { /// (Java's `authenticateWithoutBody`). #[test] fn signs_empty_body_and_all_headers() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None::, - None, - "test", - ); + let creds = example_credentials(None); let signer = SigV4Signer::builder() .region("us-east-1") .service("glue") @@ -1084,45 +1010,30 @@ mod tests { .unwrap(), ); - signer - .sign_at( - &mut req, - &creds, - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); + signer.sign_at(&mut req, &creds, test_time()).unwrap(); - let h = req.headers(); - assert_eq!(h.get("x-amz-content-sha256").unwrap(), EMPTY_HEX); - assert!(!h.contains_key("x-amz-security-token")); - let auth = h.get("authorization").unwrap().to_str().unwrap(); - assert!(auth.starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/")); - assert!(auth.contains( - "SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date" - )); - assert_signature_is( - &req, - "eaef7eb88d9cd810031684748671d8a3c9394ea5168622a212ed041b670b9777", + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=eaef7eb88d9cd810031684748671d8a3c9394ea5168622a212ed041b670b9777" + ), + ("content-encoding", "gzip"), + ("content-type", "application/json"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ]) ); } #[test] fn iceberg_mode_signs_the_hex_payload_hash_not_the_base64_header() { // Needs a body: without one the header is the hex constant too. - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None::, - None, - "test", - ); - let signer = SigV4Signer::builder() - .region("us-east-1") - .service("execute-api") - .mode(PayloadHashMode::IcebergRest) - .build(); + let creds = example_credentials(None); + let signer = test_signer(PayloadHashMode::IcebergRest); let body = br#"{"namespace":["ns"]}"#; let mut req = HttpRequest::new( reqwest::Client::new() @@ -1131,11 +1042,10 @@ mod tests { .build() .unwrap(), ); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &creds, now).unwrap(); - // The header carries base64 while the pinned signature covers hex. assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), payload_hashes(Some(body), PayloadHashMode::IcebergRest) @@ -1156,15 +1066,7 @@ mod tests { /// these requests, given these credentials as `rest.*` properties. #[test] fn signatures_match_iceberg_java() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - Some("example-session-token".to_string()), - None, - "test", - ); + let creds = example_credentials(Some("example-session-token")); let signer = test_signer(PayloadHashMode::IcebergRest); let now = Utc.with_ymd_and_hms(2026, 10, 2, 11, 40, 30).unwrap(); let client = reqwest::Client::new(); @@ -1211,15 +1113,7 @@ mod tests { /// The signed `host` keeps a non-default port, as on the wire. #[test] fn signs_host_with_non_default_port() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None::, - None, - "test", - ); + let creds = example_credentials(None); let signer = SigV4Signer::builder() .region("us-east-1") .service("glue") @@ -1232,7 +1126,7 @@ mod tests { .build() .unwrap(), ); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &creds, now).unwrap(); assert_signature_is( @@ -1245,15 +1139,7 @@ mod tests { /// and `%2C` becomes `%252C`. #[test] fn canonical_uri_is_aws_double_encoded() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None::, - None, - "test", - ); + let creds = example_credentials(None); let signer = SigV4Signer::builder() .region("us-east-1") .service("glue") @@ -1266,7 +1152,7 @@ mod tests { .build() .unwrap(), ); - let now = Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(); + let now = test_time(); signer.sign_at(&mut req, &creds, now).unwrap(); assert_signature_is( @@ -1279,15 +1165,7 @@ mod tests { /// hex `x-amz-content-sha256` like this mode. #[test] fn standard_mode_matches_the_aws_test_suite() { - use chrono::TimeZone; - - let creds = Credentials::new( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - None::, - None, - "test", - ); + let creds = example_credentials(None); let signer = SigV4Signer::builder() .region("us-east-1") .service("service") @@ -1303,13 +1181,7 @@ mod tests { .unwrap(), ); - signer - .sign_at( - &mut req, - &creds, - Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap(), - ) - .unwrap(); + signer.sign_at(&mut req, &creds, test_time()).unwrap(); assert_eq!( req.headers().get("x-amz-content-sha256").unwrap(), diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs index a2a8907a41..8c6dd7f571 100644 --- a/crates/catalog/rest/src/request.rs +++ b/crates/catalog/rest/src/request.rs @@ -110,10 +110,6 @@ pub enum HttpRequestBody<'a> { impl<'a> HttpRequestBody<'a> { /// The signable bytes: empty for [`Self::Empty`], the buffer for /// [`Self::Buffered`], and `None` for [`Self::Streaming`]. - /// - /// An absent body and an empty one both give `&[]`. Where the two must - /// differ, as in Iceberg Java's SigV4 payload hashing, match on the - /// variants instead. pub fn as_bytes(&self) -> Option<&'a [u8]> { match self { HttpRequestBody::Empty => Some(&[]),