diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 76b1ce616143..ef411d15eedc 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -231,6 +231,17 @@ std::shared_ptr S3RetryStrategy::GetAwsStandardRetryStrategy( S3Options::S3Options() { DCHECK(IsS3Initialized()) << "Must initialize S3 before using S3Options"; + + // Prefer AWS service-specific endpoint url + auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar); + if (s3_endpoint_env.ok()) { + endpoint_override = *s3_endpoint_env; + } else { + auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar); + if (endpoint_env.ok()) { + endpoint_override = *endpoint_env; + } + } } void S3Options::ConfigureDefaultCredentials() { @@ -413,17 +424,6 @@ Result S3Options::FromUriAndOptions(const ::arrow::util::Uri& uri, s3_options.ConfigureDefaultCredentials(); } - // Prefer AWS service-specific endpoint url - auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar); - if (s3_endpoint_env.ok()) { - s3_options.endpoint_override = *s3_endpoint_env; - } else { - auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar); - if (endpoint_env.ok()) { - s3_options.endpoint_override = *endpoint_env; - } - } - bool region_set = false; for (const auto& kv : options_map) { if (kv.first == "region") { @@ -1251,7 +1251,11 @@ class RegionResolver { if (instance_) { return instance_; } - auto maybe_resolver = Make(S3Options::Anonymous()); + auto options = S3Options::Anonymous(); + // Bucket regions are always resolved against AWS itself, regardless of + // the endpoint that may be configured through environment variables. + options.endpoint_override.clear(); + auto maybe_resolver = Make(std::move(options)); if (!maybe_resolver.ok()) { return maybe_resolver; } diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 5086b3407c17..3276bbdd0a83 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -125,6 +125,9 @@ struct ARROW_S3_EXPORT S3Options { double request_timeout = -1; /// If non-empty, override region with a connect string such as "localhost:9000" + /// + /// By default, this is initialized from the AWS_ENDPOINT_URL_S3 or + /// AWS_ENDPOINT_URL environment variable, if set. // XXX perhaps instead take a URL like "http://localhost:9000"? std::string endpoint_override; /// S3 connection transport, default "https" diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 06d25287a4f3..9fc31bff38b4 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -386,6 +386,22 @@ TEST_F(S3OptionsTest, FromUri) { } } +TEST_F(S3OptionsTest, EndpointFromEnvVar) { + ASSERT_EQ(S3Options::Defaults().endpoint_override, ""); + { + EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::Anonymous().endpoint_override, "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::FromAccessKey("access", "secret").endpoint_override, + "http://127.0.0.1:9000"); + + // The service-specific variable takes precedence + EnvVarGuard s3_endpoint_guard("AWS_ENDPOINT_URL_S3", "http://127.0.0.1:9001"); + ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9001"); + } + ASSERT_EQ(S3Options::Defaults().endpoint_override, ""); +} + TEST_F(S3OptionsTest, FromAccessKey) { S3Options options; @@ -524,6 +540,12 @@ TEST_F(S3RegionResolutionTest, PublicBucket) { ResolveS3BucketRegion("aws-earth-mo-atmospheric-ukv-prd")); } +TEST_F(S3RegionResolutionTest, EndpointEnvVarIgnored) { + // Region resolution must still query AWS when a custom endpoint is configured + EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000"); + ASSERT_OK_AND_EQ("us-east-1", ResolveS3BucketRegion("arrow-datasets")); +} + TEST_F(S3RegionResolutionTest, RestrictedBucket) { // usgs-landsat is a RequesterPays bucket from AWS Open Data Registry // https://github.com/awslabs/open-data-registry/blob/main/datasets/usgs-landsat.yaml diff --git a/docs/source/cpp/env_vars.rst b/docs/source/cpp/env_vars.rst index c6f5ee60324e..acbaa90b0f92 100644 --- a/docs/source/cpp/env_vars.rst +++ b/docs/source/cpp/env_vars.rst @@ -187,6 +187,8 @@ that changing their value later will have an effect. Endpoint URL used for S3-like storage, for example Minio or s3.scality. Alternatively, one can set :envvar:`AWS_ENDPOINT_URL_S3`. + An endpoint explicitly set in the S3 filesystem options or URI takes + precedence over these variables. .. envvar:: AWS_ENDPOINT_URL_S3 diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 6317bd3785f1..e19b199b8125 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -214,7 +214,9 @@ cdef class S3FileSystem(FileSystem): scheme : str, default 'https' S3 connection transport scheme. endpoint_override : str, default None - Override region with a connect string such as "localhost:9000" + Override region with a connect string such as "localhost:9000". + If not set, the ``AWS_ENDPOINT_URL_S3`` or ``AWS_ENDPOINT_URL`` + environment variable is used, if defined. background_writes : bool, default True Whether file writes will be issued in the background, without blocking. diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 416c75bccd2b..355cde8cc247 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -1237,6 +1237,35 @@ def test_gcs_options(pickle_with_and_without_subtree_filesystem): GcsFileSystem(credential_token_expiration=datetime.now()) +@pytest.mark.s3 +def test_s3fs_endpoint_from_env(s3_server, monkeypatch): + from pyarrow.fs import S3FileSystem + + host, port, access_key, secret_key = s3_server['connection'] + endpoint = f'http://{host}:{port}' + bucket = 'pyarrow-endpoint-from-env' + kwargs = dict(access_key=access_key, secret_key=secret_key, + allow_bucket_creation=True, allow_bucket_deletion=True) + + monkeypatch.setenv('AWS_ENDPOINT_URL', endpoint) + fs = S3FileSystem(**kwargs) + fs.create_dir(bucket) + assert fs.get_file_info(bucket).type == FileType.Directory + + # The service-specific variable takes precedence + monkeypatch.setenv('AWS_ENDPOINT_URL', 'http://localhost:1') + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', endpoint) + fs = S3FileSystem(**kwargs) + assert fs.get_file_info(bucket).type == FileType.Directory + + # An explicit endpoint_override takes precedence over the environment + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', 'http://localhost:1') + fs = S3FileSystem(endpoint_override=f'{host}:{port}', scheme='http', + **kwargs) + assert fs.get_file_info(bucket).type == FileType.Directory + fs.delete_dir(bucket) + + @pytest.mark.s3 def test_s3_options(pickle_with_and_without_subtree_filesystem): pickle_module = pickle_with_and_without_subtree_filesystem