From a85f340bef0930a23b3d2db21ae7c8d78619500a Mon Sep 17 00:00:00 2001 From: Furqan Khan <236162110+furqan-khan07@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:25:53 -0500 Subject: [PATCH] GH-49643: [C++][Python] Use AWS_ENDPOINT_URL env vars for all S3Options The AWS_ENDPOINT_URL_S3 and AWS_ENDPOINT_URL environment variables were only read in S3Options::FromUri. Options created with Defaults(), Anonymous(), FromAccessKey() or FromAssumeRole(), which is what pyarrow.fs.S3FileSystem() uses, ignored them and connected to AWS. Read the variables in the S3Options constructor instead, so that every way of creating S3Options starts from the same default endpoint. An explicitly set endpoint_override still takes precedence. The S3 region resolver keeps querying AWS regardless of these variables. --- cpp/src/arrow/filesystem/s3fs.cc | 28 +++++++++++++++----------- cpp/src/arrow/filesystem/s3fs.h | 3 +++ cpp/src/arrow/filesystem/s3fs_test.cc | 22 ++++++++++++++++++++ docs/source/cpp/env_vars.rst | 2 ++ python/pyarrow/_s3fs.pyx | 4 +++- python/pyarrow/tests/test_fs.py | 29 +++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 76b1ce616143..ef411d15eedc 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -231,6 +231,17 @@ std::shared_ptr S3RetryStrategy::GetAwsStandardRetryStrategy( S3Options::S3Options() { DCHECK(IsS3Initialized()) << "Must initialize S3 before using S3Options"; + + // Prefer AWS service-specific endpoint url + auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar); + if (s3_endpoint_env.ok()) { + endpoint_override = *s3_endpoint_env; + } else { + auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar); + if (endpoint_env.ok()) { + endpoint_override = *endpoint_env; + } + } } void S3Options::ConfigureDefaultCredentials() { @@ -413,17 +424,6 @@ Result S3Options::FromUriAndOptions(const ::arrow::util::Uri& uri, s3_options.ConfigureDefaultCredentials(); } - // Prefer AWS service-specific endpoint url - auto s3_endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlS3EnvVar); - if (s3_endpoint_env.ok()) { - s3_options.endpoint_override = *s3_endpoint_env; - } else { - auto endpoint_env = arrow::internal::GetEnvVar(kAwsEndpointUrlEnvVar); - if (endpoint_env.ok()) { - s3_options.endpoint_override = *endpoint_env; - } - } - bool region_set = false; for (const auto& kv : options_map) { if (kv.first == "region") { @@ -1251,7 +1251,11 @@ class RegionResolver { if (instance_) { return instance_; } - auto maybe_resolver = Make(S3Options::Anonymous()); + auto options = S3Options::Anonymous(); + // Bucket regions are always resolved against AWS itself, regardless of + // the endpoint that may be configured through environment variables. + options.endpoint_override.clear(); + auto maybe_resolver = Make(std::move(options)); if (!maybe_resolver.ok()) { return maybe_resolver; } diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 5086b3407c17..3276bbdd0a83 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -125,6 +125,9 @@ struct ARROW_S3_EXPORT S3Options { double request_timeout = -1; /// If non-empty, override region with a connect string such as "localhost:9000" + /// + /// By default, this is initialized from the AWS_ENDPOINT_URL_S3 or + /// AWS_ENDPOINT_URL environment variable, if set. // XXX perhaps instead take a URL like "http://localhost:9000"? std::string endpoint_override; /// S3 connection transport, default "https" diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 06d25287a4f3..9fc31bff38b4 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -386,6 +386,22 @@ TEST_F(S3OptionsTest, FromUri) { } } +TEST_F(S3OptionsTest, EndpointFromEnvVar) { + ASSERT_EQ(S3Options::Defaults().endpoint_override, ""); + { + EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::Anonymous().endpoint_override, "http://127.0.0.1:9000"); + ASSERT_EQ(S3Options::FromAccessKey("access", "secret").endpoint_override, + "http://127.0.0.1:9000"); + + // The service-specific variable takes precedence + EnvVarGuard s3_endpoint_guard("AWS_ENDPOINT_URL_S3", "http://127.0.0.1:9001"); + ASSERT_EQ(S3Options::Defaults().endpoint_override, "http://127.0.0.1:9001"); + } + ASSERT_EQ(S3Options::Defaults().endpoint_override, ""); +} + TEST_F(S3OptionsTest, FromAccessKey) { S3Options options; @@ -524,6 +540,12 @@ TEST_F(S3RegionResolutionTest, PublicBucket) { ResolveS3BucketRegion("aws-earth-mo-atmospheric-ukv-prd")); } +TEST_F(S3RegionResolutionTest, EndpointEnvVarIgnored) { + // Region resolution must still query AWS when a custom endpoint is configured + EnvVarGuard endpoint_guard("AWS_ENDPOINT_URL", "http://127.0.0.1:9000"); + ASSERT_OK_AND_EQ("us-east-1", ResolveS3BucketRegion("arrow-datasets")); +} + TEST_F(S3RegionResolutionTest, RestrictedBucket) { // usgs-landsat is a RequesterPays bucket from AWS Open Data Registry // https://github.com/awslabs/open-data-registry/blob/main/datasets/usgs-landsat.yaml diff --git a/docs/source/cpp/env_vars.rst b/docs/source/cpp/env_vars.rst index c6f5ee60324e..acbaa90b0f92 100644 --- a/docs/source/cpp/env_vars.rst +++ b/docs/source/cpp/env_vars.rst @@ -187,6 +187,8 @@ that changing their value later will have an effect. Endpoint URL used for S3-like storage, for example Minio or s3.scality. Alternatively, one can set :envvar:`AWS_ENDPOINT_URL_S3`. + An endpoint explicitly set in the S3 filesystem options or URI takes + precedence over these variables. .. envvar:: AWS_ENDPOINT_URL_S3 diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 6317bd3785f1..e19b199b8125 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -214,7 +214,9 @@ cdef class S3FileSystem(FileSystem): scheme : str, default 'https' S3 connection transport scheme. endpoint_override : str, default None - Override region with a connect string such as "localhost:9000" + Override region with a connect string such as "localhost:9000". + If not set, the ``AWS_ENDPOINT_URL_S3`` or ``AWS_ENDPOINT_URL`` + environment variable is used, if defined. background_writes : bool, default True Whether file writes will be issued in the background, without blocking. diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 416c75bccd2b..355cde8cc247 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -1237,6 +1237,35 @@ def test_gcs_options(pickle_with_and_without_subtree_filesystem): GcsFileSystem(credential_token_expiration=datetime.now()) +@pytest.mark.s3 +def test_s3fs_endpoint_from_env(s3_server, monkeypatch): + from pyarrow.fs import S3FileSystem + + host, port, access_key, secret_key = s3_server['connection'] + endpoint = f'http://{host}:{port}' + bucket = 'pyarrow-endpoint-from-env' + kwargs = dict(access_key=access_key, secret_key=secret_key, + allow_bucket_creation=True, allow_bucket_deletion=True) + + monkeypatch.setenv('AWS_ENDPOINT_URL', endpoint) + fs = S3FileSystem(**kwargs) + fs.create_dir(bucket) + assert fs.get_file_info(bucket).type == FileType.Directory + + # The service-specific variable takes precedence + monkeypatch.setenv('AWS_ENDPOINT_URL', 'http://localhost:1') + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', endpoint) + fs = S3FileSystem(**kwargs) + assert fs.get_file_info(bucket).type == FileType.Directory + + # An explicit endpoint_override takes precedence over the environment + monkeypatch.setenv('AWS_ENDPOINT_URL_S3', 'http://localhost:1') + fs = S3FileSystem(endpoint_override=f'{host}:{port}', scheme='http', + **kwargs) + assert fs.get_file_info(bucket).type == FileType.Directory + fs.delete_dir(bucket) + + @pytest.mark.s3 def test_s3_options(pickle_with_and_without_subtree_filesystem): pickle_module = pickle_with_and_without_subtree_filesystem