Skip to content

Commit 02a2d21

Browse files
feat(io): restore the oss scheme with a round trip test and docs
1 parent 268883e commit 02a2d21

5 files changed

Lines changed: 223 additions & 8 deletions

File tree

‎mkdocs/docs/file-io.md‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ implementations:
2929
| Registry name | Schemes |
3030
|---|---|
3131
| `arrow-fs-local` | paths without a scheme, `file` |
32-
| `arrow-fs-s3` | `s3`, `s3a`, `s3n` |
32+
| `arrow-fs-s3` | `s3`, `s3a`, `s3n`, `oss` |
3333

3434
The S3 implementation requires Arrow S3 support.
3535

@@ -56,6 +56,47 @@ For a REST catalog, set `io-impl` to the registry name. If it is omitted, the
5656
REST catalog uses `ResolvingFileIO` and selects a registered implementation for
5757
each file location's scheme.
5858
59+
## Configure S3
60+
61+
| Key | Example | Description |
62+
|---|---|---|
63+
| `s3.access-key-id` | `admin` | Static access key ID; must be set together with the secret key |
64+
| `s3.secret-access-key` | `password` | Static secret access key |
65+
| `s3.session-token` | `AQoDYXdzEJr...` | Session token, for temporary credentials |
66+
| `client.region` | `us-east-1` | Region to sign requests for |
67+
| `s3.endpoint` | `https://127.0.0.1:9000` | Endpoint to use instead of the AWS one |
68+
| `s3.path-style-access` | `true` | Address buckets as a path (`endpoint/bucket`) instead of a virtual host (`bucket.endpoint`). Only takes effect together with `s3.endpoint` |
69+
| `s3.ssl.enabled` | `true` | Scheme to use for the endpoint, overriding the one it carries |
70+
| `s3.connect-timeout-ms` | `1000` | Connection timeout |
71+
| `s3.socket-timeout-ms` | `5000` | Request timeout. Ignored outside Windows and macOS |
72+
73+
Without credentials, the AWS default credential chain is used, which covers
74+
environment variables, the shared configuration file, and the various role and
75+
identity providers.
76+
77+
### S3-compatible storage
78+
79+
Stores that speak the S3 API are served by the same implementation. The scheme
80+
selects it; `s3.endpoint` decides where requests actually go. A location keeps
81+
its own scheme and is canonicalized internally, so a credential vended for the
82+
`s3` prefix applies to it.
83+
84+
For Alibaba Cloud OSS, point `s3.endpoint` at the S3-compatible endpoint of the
85+
bucket's region and leave bucket addressing virtual-hosted, which the service
86+
requires — path-style requests are rejected with
87+
`SecondLevelDomainForbidden: Please use virtual hosted style to access`:
88+
89+
```cpp
90+
auto file_io = iceberg::FileIORegistry::Load(
91+
iceberg::FileIORegistry::kArrowS3FileIO,
92+
{{std::string(iceberg::arrow::S3Properties::kEndpoint),
93+
"https://s3.oss-cn-hangzhou.aliyuncs.com"},
94+
{std::string(iceberg::arrow::S3Properties::kClientRegion), "cn-hangzhou"},
95+
{std::string(iceberg::arrow::S3Properties::kPathStyleAccess), "false"}});
96+
97+
file_io.value()->NewInputFile("oss://bucket/path/to/file.parquet");
98+
```
99+
59100
## Register a custom FileIO
60101

61102
Register the factory before creating the catalog or resolver:

‎src/iceberg/arrow/s3/arrow_s3_file_io.cc‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,10 +177,18 @@ Result<std::shared_ptr<::arrow::fs::FileSystem>> BuildArrowS3FileSystem(
177177
return std::shared_ptr<::arrow::fs::FileSystem>(std::move(fs));
178178
}
179179

180+
// Rewrites a foreign alias to `s3://` so that locations and credential prefixes
181+
// compare equal regardless of which one a catalog used. Derived from kS3Schemes
182+
// rather than a list of its own: an alias missing here would not fail, it would
183+
// silently stop matching its credential and fall back to the default one.
180184
std::string CanonicalizeS3Scheme(std::string_view location) {
181-
for (std::string_view scheme : {"s3a://", "s3n://"}) {
182-
if (location.starts_with(scheme)) {
183-
return std::string("s3://").append(location.substr(scheme.size()));
185+
for (std::string_view scheme : kS3Schemes) {
186+
if (scheme == S3Properties::kS3Schema) {
187+
continue;
188+
}
189+
if (location.starts_with(scheme) &&
190+
location.substr(scheme.size()).starts_with("://")) {
191+
return std::string("s3://").append(location.substr(scheme.size() + 3));
184192
}
185193
}
186194
return std::string(location);

‎src/iceberg/arrow/s3/s3_properties.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,13 @@ struct S3Properties {
5858

5959
/// \brief URI schemes served by the Arrow S3 FileIO, lower-case.
6060
///
61-
/// Single source of truth: both the registry registration and IsS3Scheme derive
62-
/// from this list, so a new alias only has to be added here.
63-
inline constexpr std::array<std::string_view, 3> kS3Schemes = {"s3", "s3a", "s3n"};
61+
/// Single source of truth: the registry registration, IsS3Scheme and the
62+
/// canonicalization of foreign aliases all derive from this list, so a new alias
63+
/// only has to be added here.
64+
///
65+
/// `oss` is served here because the store is S3-compatible; see the FileIO
66+
/// documentation for how to configure it.
67+
inline constexpr std::array<std::string_view, 4> kS3Schemes = {"s3", "s3a", "s3n", "oss"};
6468

6569
/// \brief Return whether a normalized URI scheme is S3-compatible.
6670
inline constexpr bool IsS3Scheme(std::string_view scheme) {

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,10 +280,11 @@ endif()
280280

281281
if(ICEBERG_BUILD_REST)
282282
function(add_rest_iceberg_test test_name)
283+
set(options USE_BUNDLE)
283284
set(oneValueArgs)
284285
set(multiValueArgs SOURCES)
285286
cmake_parse_arguments(ARG
286-
""
287+
"${options}"
287288
"${oneValueArgs}"
288289
"${multiValueArgs}"
289290
${ARGN})
@@ -292,12 +293,22 @@ if(ICEBERG_BUILD_REST)
292293
target_include_directories(${test_name} PRIVATE "${CMAKE_BINARY_DIR}/iceberg/test/")
293294
target_sources(${test_name} PRIVATE ${ARG_SOURCES})
294295
target_link_libraries(${test_name} PRIVATE GTest::gmock_main iceberg_rest_static)
296+
if(ARG_USE_BUNDLE)
297+
target_link_libraries(${test_name}
298+
PRIVATE "$<IF:$<TARGET_EXISTS:iceberg_bundle_static>,iceberg_bundle_static,iceberg_bundle_shared>"
299+
)
300+
endif()
295301
if(MSVC_TOOLCHAIN)
296302
target_compile_options(${test_name} PRIVATE /bigobj)
297303
endif()
298304
add_test(NAME ${test_name} COMMAND ${test_name})
299305
endfunction()
300306

307+
if(ICEBERG_BUILD_BUNDLE)
308+
add_rest_iceberg_test(rest_arrow_file_io_test USE_BUNDLE SOURCES
309+
rest_arrow_file_io_test.cc)
310+
endif()
311+
301312
add_rest_iceberg_test(rest_catalog_test
302313
SOURCES
303314
auth_manager_test.cc
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
/// \file
21+
/// \brief Covers REST -> ResolvingFileIO -> registry -> Arrow FileIO against the
22+
/// real registered implementations, which mock delegates cannot exercise.
23+
24+
#include <algorithm>
25+
#include <cstdlib>
26+
#include <memory>
27+
#include <optional>
28+
#include <string>
29+
#include <string_view>
30+
#include <tuple>
31+
#include <unordered_map>
32+
#include <utility>
33+
#include <vector>
34+
35+
#include <gmock/gmock.h>
36+
#include <gtest/gtest.h>
37+
38+
#include "iceberg/arrow/arrow_io_util.h"
39+
#include "iceberg/arrow/arrow_register.h"
40+
#include "iceberg/catalog/rest/rest_file_io.h"
41+
#include "iceberg/logging/logger.h"
42+
#include "iceberg/storage_credential.h"
43+
#include "iceberg/test/logging_test_helpers.h"
44+
#include "iceberg/test/matchers.h"
45+
#include "iceberg/test/temp_file_test_base.h"
46+
47+
namespace iceberg::rest {
48+
49+
namespace {
50+
51+
class RestArrowFileIOTest : public TempFileTestBase {
52+
protected:
53+
static void SetUpTestSuite() { iceberg::arrow::RegisterAll(); }
54+
static void TearDownTestSuite() { std::ignore = iceberg::arrow::FinalizeS3(); }
55+
};
56+
57+
TEST_F(RestArrowFileIOTest, ReadsBackWhatItWroteThroughRealLocalFileIO) {
58+
auto io = MakeTableFileIO({{"warehouse", "logical_warehouse_name"}},
59+
/*table_config=*/{}, /*storage_credentials=*/{});
60+
ASSERT_THAT(io, IsOk());
61+
62+
const auto path = CreateNewTempFilePathWithSuffix(".txt");
63+
constexpr std::string_view kContent = "resolved through the real local FileIO";
64+
65+
ASSERT_THAT(io.value()->WriteFile(path, kContent), IsOk());
66+
EXPECT_THAT(io.value()->ReadFile(path, std::nullopt),
67+
HasValue(::testing::Eq(std::string(kContent))));
68+
EXPECT_THAT(io.value()->DeleteFile(path), IsOk());
69+
}
70+
71+
#if ICEBERG_S3_ENABLED
72+
73+
bool HasWarning(const CapturingLogger& logger) {
74+
const auto records = logger.records();
75+
return std::ranges::any_of(
76+
records, [](const LogMessage& record) { return record.level == LogLevel::kWarn; });
77+
}
78+
79+
std::optional<std::string> GetEnvIfSet(const char* key) {
80+
const char* value = std::getenv(key);
81+
if (value == nullptr || std::string_view(value).empty()) {
82+
return std::nullopt;
83+
}
84+
return std::string(value);
85+
}
86+
87+
/// Addresses the store an S3 test reaches as `s3://` the way a catalog vending
88+
/// `oss://` locations would.
89+
std::string AsOssUri(std::string_view uri) {
90+
const auto pos = uri.find("://");
91+
const auto authority = pos == std::string_view::npos ? uri : uri.substr(pos + 3);
92+
return std::string("oss://").append(authority);
93+
}
94+
95+
// Resolution and real I/O for an S3-compatible store addressed as `oss://`.
96+
// Credential routing is exercised but not proven: the same keys reach the
97+
// default client through the AWS chain, so a broken match would still pass.
98+
TEST_F(RestArrowFileIOTest, ReadsBackWhatItWroteThroughAnOssLocation) {
99+
const auto base_uri = GetEnvIfSet("ICEBERG_TEST_S3_URI");
100+
if (!base_uri.has_value()) {
101+
GTEST_SKIP() << "Set ICEBERG_TEST_S3_URI to enable the oss:// round trip";
102+
}
103+
104+
std::unordered_map<std::string, std::string> credential_config;
105+
if (const auto access_key = GetEnvIfSet("AWS_ACCESS_KEY_ID")) {
106+
credential_config["s3.access-key-id"] = *access_key;
107+
}
108+
if (const auto secret_key = GetEnvIfSet("AWS_SECRET_ACCESS_KEY")) {
109+
credential_config["s3.secret-access-key"] = *secret_key;
110+
}
111+
ASSERT_FALSE(credential_config.empty())
112+
<< "Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY alongside "
113+
"ICEBERG_TEST_S3_URI";
114+
115+
// Scoped to `s3`, while the data it grants access to is addressed as `oss://`.
116+
auto io = MakeTableFileIO({{"warehouse", "logical_warehouse_name"}},
117+
/*table_config=*/{},
118+
{{.prefix = "s3", .config = std::move(credential_config)}});
119+
ASSERT_THAT(io, IsOk());
120+
121+
const auto object_uri = AsOssUri(*base_uri) + "/iceberg_oss_scheme_round_trip.txt";
122+
constexpr std::string_view kContent = "resolved and written through an oss:// location";
123+
124+
ASSERT_THAT(io.value()->WriteFile(object_uri, kContent), IsOk());
125+
EXPECT_THAT(io.value()->ReadFile(object_uri, std::nullopt),
126+
HasValue(::testing::Eq(std::string(kContent))));
127+
EXPECT_THAT(io.value()->DeleteFile(object_uri), IsOk());
128+
}
129+
130+
TEST_F(RestArrowFileIOTest, AppliesOssCredentialThroughRealArrowS3FileIO) {
131+
auto logger = std::make_shared<CapturingLogger>();
132+
ScopedDefaultLogger scoped(logger);
133+
134+
auto io =
135+
MakeTableFileIO({{"warehouse", "logical_warehouse_name"}}, /*table_config=*/{},
136+
{{.prefix = "oss://bucket/table", .config = {{"k", "v"}}}});
137+
ASSERT_THAT(io, IsOk());
138+
139+
// Opening builds the delegate and applies the credential. The open itself hits
140+
// the network, so only the failure modes before that are asserted: a routing
141+
// break surfaces as kNotSupported, and a dropped credential as the warning.
142+
auto input = io.value()->NewInputFile("oss://bucket/table/data/file.parquet");
143+
EXPECT_THAT(input, ::testing::Not(IsError(ErrorKind::kNotSupported)));
144+
EXPECT_FALSE(HasWarning(*logger));
145+
}
146+
147+
#endif // ICEBERG_S3_ENABLED
148+
149+
} // namespace
150+
151+
} // namespace iceberg::rest

0 commit comments

Comments
 (0)