Skip to content

Commit c2463cf

Browse files
Replace S3 remote signer headers instead of appending them
1 parent e3564a5 commit c2463cf

2 files changed

Lines changed: 38 additions & 0 deletions

File tree

‎pyiceberg/io/fsspec.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ def __call__(self, request: "AWSRequest", operation_name: str | None = None, **_
160160
raise SignError(f"Failed to sign request {response.status_code}: {signer_body}") from e
161161

162162
for key, value in response_json["headers"].items():
163+
# Replace instead of append, so a header the request already has is not sent twice
164+
del request.headers[key]
163165
request.headers.add_header(key, ", ".join(value))
164166

165167
request.url = response_json["uri"]

‎tests/io/test_fsspec.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1030,6 +1030,42 @@ def test_s3v4_rest_signer(requests_mock: Mocker) -> None:
10301030
}
10311031

10321032

1033+
def test_s3v4_rest_signer_replaces_existing_headers(requests_mock: Mocker) -> None:
1034+
requests_mock.post(
1035+
f"{TEST_URI}/v1/aws/s3/sign",
1036+
json={
1037+
"uri": "https://bucket/metadata/file.avro",
1038+
"headers": {
1039+
"Authorization": ["AWS4-HMAC-SHA256 Credential=..., SignedHeaders=..., Signature=..."],
1040+
"x-amz-content-sha256": ["UNSIGNED-PAYLOAD"],
1041+
"User-Agent": ["Botocore/1.27.59"],
1042+
},
1043+
"extensions": {},
1044+
},
1045+
status_code=200,
1046+
)
1047+
1048+
request = AWSRequest(
1049+
method="PUT",
1050+
url="https://bucket/metadata/file.avro",
1051+
headers={
1052+
"User-Agent": "Botocore/1.27.59",
1053+
"X-Amz-Content-SHA256": "STREAMING-UNSIGNED-PAYLOAD-TRAILER",
1054+
},
1055+
data=b"",
1056+
params={},
1057+
auth_path="/metadata/file.avro",
1058+
)
1059+
request.context = {"client_region": "us-west-2"}
1060+
1061+
signer = S3V4RestSigner(properties={"token": "abc", "uri": TEST_URI})
1062+
signer(request)
1063+
1064+
assert request.headers.get_all("x-amz-content-sha256") == ["UNSIGNED-PAYLOAD"]
1065+
assert request.headers.get_all("User-Agent") == ["Botocore/1.27.59"]
1066+
assert len(request.headers.get_all("Authorization")) == 1
1067+
1068+
10331069
@pytest.mark.s3
10341070
def test_s3v4_rest_signer_write_read_delete(request: pytest.FixtureRequest, requests_mock: Mocker) -> None:
10351071
"""Test writing, reading, and deleting an S3 file through a REST signing service"""

0 commit comments

Comments
 (0)