diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 25843152a5..dbd271dab7 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -230,19 +230,27 @@ def _s3(properties: Properties) -> AbstractFileSystem: "config_kwargs": config_kwargs, } + session_kwargs: dict[str, Any] = {} if profile_name := get_first_property_value(properties, S3_PROFILE_NAME, AWS_PROFILE_NAME): s3_fs_kwargs["profile"] = profile_name + session_kwargs["profile"] = profile_name if s3_additional_kwargs: s3_fs_kwargs["s3_additional_kwargs"] = s3_additional_kwargs - fs = S3FileSystem(**s3_fs_kwargs) + if register_events: + from aiobotocore.session import AioSession - for event_name, event_function in register_events.items(): - fs.s3.meta.events.unregister(event_name, unique_id=1925) - fs.s3.meta.events.register_last(event_name, event_function, unique_id=1925) + # Register the signer on a session that only this filesystem uses. s3fs caches and shares + # the client between threads, and unregistering the signer on a shared client leaves a + # window in which a request is sent unsigned. + session = AioSession(**session_kwargs) + event_emitter = session.get_component("event_emitter") + for event_name, event_function in register_events.items(): + event_emitter.register_last(event_name, event_function, unique_id=1925) + s3_fs_kwargs["session"] = session - return fs + return S3FileSystem(**s3_fs_kwargs) def _gs(properties: Properties) -> AbstractFileSystem: diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index e7b2c82687..6253ed9091 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import asyncio import os import pickle import tempfile @@ -24,6 +25,7 @@ from unittest import mock import pytest +from botocore import UNSIGNED from botocore.auth import S3SigV4Auth from botocore.awsrequest import AWSRequest from botocore.config import Config @@ -960,6 +962,64 @@ def _test_fsspec_pickle_round_trip(fsspec_fileio: FsspecFileIO, location: str) - TEST_URI = "https://iceberg-test-signer" +def _sign_with_session(session: Any, uri: str) -> AWSRequest: + request = AWSRequest(method="PUT", url=uri, headers={}, data=b"", params={}) + request.context = {"client_region": "us-east-1"} + asyncio.run( + session.get_component("event_emitter").emit( + "before-sign.s3", + request=request, + signing_name="s3", + region_name="us-east-1", + signature_version=UNSIGNED, + request_signer=None, + operation_name="PutObject", + ) + ) + return request + + +def test_s3_registers_the_signer_on_the_session(requests_mock: Mocker) -> None: + new_uri = "https://other-bucket/object" + requests_mock.post( + f"{TEST_URI}/v1/aws/s3/sign", + json={"uri": new_uri, "headers": {"Authorization": ["AWS4-HMAC-SHA256 Credential=abc"]}, "extensions": {}}, + status_code=200, + ) + + with mock.patch("s3fs.S3FileSystem") as mock_s3fs: + fsspec._s3({"s3.signer": "S3V4RestSigner", "uri": TEST_URI, "s3.endpoint": "http://localhost:9000"}) + + session = mock_s3fs.call_args.kwargs["session"] + request = _sign_with_session(session, "https://bucket/object") + + assert request.url == new_uri + assert request.headers["Authorization"] == "AWS4-HMAC-SHA256 Credential=abc" + + +def test_s3_registers_the_signer_per_session(requests_mock: Mocker) -> None: + signers = [ + ("https://first-signer", "https://bucket/first"), + ("https://second-signer", "https://bucket/second"), + ] + for signer_uri, signed_uri in signers: + requests_mock.post( + f"{signer_uri}/v1/aws/s3/sign", + json={"uri": signed_uri, "headers": {"Authorization": ["AWS4-HMAC-SHA256 Credential=abc"]}, "extensions": {}}, + status_code=200, + ) + + sessions = [] + with mock.patch("s3fs.S3FileSystem") as mock_s3fs: + for signer_uri, _ in signers: + fsspec._s3({"s3.signer": "S3V4RestSigner", "uri": signer_uri, "s3.endpoint": "http://localhost:9000"}) + sessions.append(mock_s3fs.call_args.kwargs["session"]) + + assert sessions[0] is not sessions[1] + for session, (_, signed_uri) in zip(sessions, signers, strict=True): + assert _sign_with_session(session, "https://bucket/object").url == signed_uri + + def test_s3v4_rest_signer(requests_mock: Mocker) -> None: new_uri = "https://other-bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro" requests_mock.post(