Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions cpp/src/arrow/filesystem/s3fs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,17 @@ std::shared_ptr<S3RetryStrategy> S3RetryStrategy::GetAwsStandardRetryStrategy(

S3Options::S3Options() {
DCHECK(IsS3Initialized()) << "Must initialize S3 before using S3Options";

// Prefer AWS service-specific endpoint url
auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar);
if (s3_endpoint_env.ok()) {
endpoint_override = *s3_endpoint_env;
} else {
auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar);
if (endpoint_env.ok()) {
endpoint_override = *endpoint_env;
}
}
}

void S3Options::ConfigureDefaultCredentials() {
Expand Down Expand Up @@ -413,17 +424,6 @@ Result<S3Options> S3Options::FromUriAndOptions(const ::arrow::util::Uri& uri,
s3_options.ConfigureDefaultCredentials();
}

// Prefer AWS service-specific endpoint url
auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar);
if (s3_endpoint_env.ok()) {
s3_options.endpoint_override = *s3_endpoint_env;
} else {
auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar);
if (endpoint_env.ok()) {
s3_options.endpoint_override = *endpoint_env;
}
}

bool region_set = false;
for (const auto& kv : options_map) {
if (kv.first == "region") {
Expand Down Expand Up @@ -1251,7 +1251,11 @@ class RegionResolver {
if (instance_) {
return instance_;
}
auto maybe_resolver = Make(S3Options::Anonymous());
auto options = S3Options::Anonymous();
// Bucket regions are always resolved against AWS itself, regardless of
// the endpoint that may be configured through environment variables.
options.endpoint_override.clear();
auto maybe_resolver = Make(std::move(options));
if (!maybe_resolver.ok()) {
return maybe_resolver;
}
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/filesystem/s3fs.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ struct ARROW_S3_EXPORT S3Options {
double request_timeout = -1;

/// If non-empty, override region with a connect string such as "localhost:9000"
///
/// By default, this is initialized from the AWS_ENDPOINT_URL_S3 or
/// AWS_ENDPOINT_URL environment variable, if set.
// XXX perhaps instead take a URL like "http://localhost:9000"?
std::string endpoint_override;
/// S3 connection transport, default "https"
Expand Down
22 changes: 22 additions & 0 deletions cpp/src/arrow/filesystem/s3fs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,22 @@ TEST_F(S3OptionsTest, FromUri) {
}
}

TEST_F(S3OptionsTest, EndpointFromEnvVar) {
ASSERT_EQ(S3Options::Defaults().endpoint_override, "");
{
EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000");
ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9000");
ASSERT_EQ(S3Options::Anonymous().endpoint_override, "http://127.0.0.1:9000");
ASSERT_EQ(S3Options::FromAccessKey("access", "secret").endpoint_override,
"http://127.0.0.1:9000");

// The service-specific variable takes precedence
EnvVarGuard s3_endpoint_guard("AWS_ENDPOINT_URL_S3", "http://127.0.0.1:9001");
ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9001");
}
ASSERT_EQ(S3Options::Defaults().endpoint_override, "");
}

TEST_F(S3OptionsTest, FromAccessKey) {
S3Options options;

Expand Down Expand Up @@ -524,6 +540,12 @@ TEST_F(S3RegionResolutionTest, PublicBucket) {
ResolveS3BucketRegion("aws-earth-mo-atmospheric-ukv-prd"));
}

TEST_F(S3RegionResolutionTest, EndpointEnvVarIgnored) {
// Region resolution must still query AWS when a custom endpoint is configured
EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000");
ASSERT_OK_AND_EQ("us-east-1", ResolveS3BucketRegion("arrow-datasets"));
}

TEST_F(S3RegionResolutionTest, RestrictedBucket) {
// usgs-landsat is a RequesterPays bucket from AWS Open Data Registry
// https://github.com/awslabs/open-data-registry/blob/main/datasets/usgs-landsat.yaml
Expand Down
2 changes: 2 additions & 0 deletions docs/source/cpp/env_vars.rst
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ that changing their value later will have an effect.

Endpoint URL used for S3-like storage, for example Minio or s3.scality.
Alternatively, one can set :envvar:`AWS_ENDPOINT_URL_S3`.
An endpoint explicitly set in the S3 filesystem options or URI takes
precedence over these variables.

.. envvar:: AWS_ENDPOINT_URL_S3

Expand Down
4 changes: 3 additions & 1 deletion python/pyarrow/_s3fs.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,9 @@ cdef class S3FileSystem(FileSystem):
scheme : str, default 'https'
S3 connection transport scheme.
endpoint_override : str, default None
Override region with a connect string such as "localhost:9000"
Override region with a connect string such as "localhost:9000".
If not set, the ``AWS_ENDPOINT_URL_S3`` or ``AWS_ENDPOINT_URL``
environment variable is used, if defined.
background_writes : bool, default True
Whether file writes will be issued in the background, without
blocking.
Expand Down
29 changes: 29 additions & 0 deletions python/pyarrow/tests/test_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1237,6 +1237,35 @@ def test_gcs_options(pickle_with_and_without_subtree_filesystem):
GcsFileSystem(credential_token_expiration=datetime.now())


@pytest.mark.s3
def test_s3fs_endpoint_from_env(s3_server, monkeypatch):
from pyarrow.fs import S3FileSystem

host, port, access_key, secret_key = s3_server['connection']
endpoint = f'http://{host}:{port}'
bucket = 'pyarrow-endpoint-from-env'
kwargs = dict(access_key=access_key, secret_key=secret_key,
allow_bucket_creation=True, allow_bucket_deletion=True)

monkeypatch.setenv('AWS_ENDPOINT_URL', endpoint)
fs = S3FileSystem(**kwargs)
fs.create_dir(bucket)
assert fs.get_file_info(bucket).type == FileType.Directory

# The service-specific variable takes precedence
monkeypatch.setenv('AWS_ENDPOINT_URL', 'http://localhost:1')
monkeypatch.setenv('AWS_ENDPOINT_URL_S3', endpoint)
fs = S3FileSystem(**kwargs)
assert fs.get_file_info(bucket).type == FileType.Directory

# An explicit endpoint_override takes precedence over the environment
monkeypatch.setenv('AWS_ENDPOINT_URL_S3', 'http://localhost:1')
fs = S3FileSystem(endpoint_override=f'{host}:{port}', scheme='http',
**kwargs)
assert fs.get_file_info(bucket).type == FileType.Directory
fs.delete_dir(bucket)


@pytest.mark.s3
def test_s3_options(pickle_with_and_without_subtree_filesystem):
pickle_module = pickle_with_and_without_subtree_filesystem
Expand Down
Loading