|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +/// \file |
| 21 | +/// \brief Covers REST -> ResolvingFileIO -> registry -> Arrow FileIO against the |
| 22 | +/// real registered implementations, which mock delegates cannot exercise. |
| 23 | + |
| 24 | +#include <algorithm> |
| 25 | +#include <cstdlib> |
| 26 | +#include <memory> |
| 27 | +#include <optional> |
| 28 | +#include <string> |
| 29 | +#include <string_view> |
| 30 | +#include <tuple> |
| 31 | +#include <unordered_map> |
| 32 | +#include <utility> |
| 33 | +#include <vector> |
| 34 | + |
| 35 | +#include <gmock/gmock.h> |
| 36 | +#include <gtest/gtest.h> |
| 37 | + |
| 38 | +#include "iceberg/arrow/arrow_io_util.h" |
| 39 | +#include "iceberg/arrow/arrow_register.h" |
| 40 | +#include "iceberg/catalog/rest/rest_file_io.h" |
| 41 | +#include "iceberg/logging/logger.h" |
| 42 | +#include "iceberg/storage_credential.h" |
| 43 | +#include "iceberg/test/logging_test_helpers.h" |
| 44 | +#include "iceberg/test/matchers.h" |
| 45 | +#include "iceberg/test/temp_file_test_base.h" |
| 46 | + |
| 47 | +namespace iceberg::rest { |
| 48 | + |
| 49 | +namespace { |
| 50 | + |
| 51 | +class RestArrowFileIOTest : public TempFileTestBase { |
| 52 | + protected: |
| 53 | + static void SetUpTestSuite() { iceberg::arrow::RegisterAll(); } |
| 54 | + static void TearDownTestSuite() { std::ignore = iceberg::arrow::FinalizeS3(); } |
| 55 | +}; |
| 56 | + |
| 57 | +TEST_F(RestArrowFileIOTest, ReadsBackWhatItWroteThroughRealLocalFileIO) { |
| 58 | + auto io = MakeTableFileIO({{"warehouse", "logical_warehouse_name"}}, |
| 59 | + /*table_config=*/{}, /*storage_credentials=*/{}); |
| 60 | + ASSERT_THAT(io, IsOk()); |
| 61 | + |
| 62 | + const auto path = CreateNewTempFilePathWithSuffix(".txt"); |
| 63 | + constexpr std::string_view kContent = "resolved through the real local FileIO"; |
| 64 | + |
| 65 | + ASSERT_THAT(io.value()->WriteFile(path, kContent), IsOk()); |
| 66 | + EXPECT_THAT(io.value()->ReadFile(path, std::nullopt), |
| 67 | + HasValue(::testing::Eq(std::string(kContent)))); |
| 68 | + EXPECT_THAT(io.value()->DeleteFile(path), IsOk()); |
| 69 | +} |
| 70 | + |
| 71 | +#if ICEBERG_S3_ENABLED |
| 72 | + |
| 73 | +bool HasWarning(const CapturingLogger& logger) { |
| 74 | + const auto records = logger.records(); |
| 75 | + return std::ranges::any_of( |
| 76 | + records, [](const LogMessage& record) { return record.level == LogLevel::kWarn; }); |
| 77 | +} |
| 78 | + |
| 79 | +std::optional<std::string> GetEnvIfSet(const char* key) { |
| 80 | + const char* value = std::getenv(key); |
| 81 | + if (value == nullptr || std::string_view(value).empty()) { |
| 82 | + return std::nullopt; |
| 83 | + } |
| 84 | + return std::string(value); |
| 85 | +} |
| 86 | + |
| 87 | +/// Addresses the store an S3 test reaches as `s3://` the way a catalog vending |
| 88 | +/// `oss://` locations would. |
| 89 | +std::string AsOssUri(std::string_view uri) { |
| 90 | + const auto pos = uri.find("://"); |
| 91 | + const auto authority = pos == std::string_view::npos ? uri : uri.substr(pos + 3); |
| 92 | + return std::string("oss://").append(authority); |
| 93 | +} |
| 94 | + |
| 95 | +// Resolution and real I/O for an S3-compatible store addressed as `oss://`. |
| 96 | +// Credential routing is exercised but not proven: the same keys reach the |
| 97 | +// default client through the AWS chain, so a broken match would still pass. |
| 98 | +TEST_F(RestArrowFileIOTest, ReadsBackWhatItWroteThroughAnOssLocation) { |
| 99 | + const auto base_uri = GetEnvIfSet("ICEBERG_TEST_S3_URI"); |
| 100 | + if (!base_uri.has_value()) { |
| 101 | + GTEST_SKIP() << "Set ICEBERG_TEST_S3_URI to enable the oss:// round trip"; |
| 102 | + } |
| 103 | + |
| 104 | + std::unordered_map<std::string, std::string> credential_config; |
| 105 | + if (const auto access_key = GetEnvIfSet("AWS_ACCESS_KEY_ID")) { |
| 106 | + credential_config["s3.access-key-id"] = *access_key; |
| 107 | + } |
| 108 | + if (const auto secret_key = GetEnvIfSet("AWS_SECRET_ACCESS_KEY")) { |
| 109 | + credential_config["s3.secret-access-key"] = *secret_key; |
| 110 | + } |
| 111 | + ASSERT_FALSE(credential_config.empty()) |
| 112 | + << "Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY alongside " |
| 113 | + "ICEBERG_TEST_S3_URI"; |
| 114 | + |
| 115 | + // Scoped to `s3`, while the data it grants access to is addressed as `oss://`. |
| 116 | + auto io = MakeTableFileIO({{"warehouse", "logical_warehouse_name"}}, |
| 117 | + /*table_config=*/{}, |
| 118 | + {{.prefix = "s3", .config = std::move(credential_config)}}); |
| 119 | + ASSERT_THAT(io, IsOk()); |
| 120 | + |
| 121 | + const auto object_uri = AsOssUri(*base_uri) + "/iceberg_oss_scheme_round_trip.txt"; |
| 122 | + constexpr std::string_view kContent = "resolved and written through an oss:// location"; |
| 123 | + |
| 124 | + ASSERT_THAT(io.value()->WriteFile(object_uri, kContent), IsOk()); |
| 125 | + EXPECT_THAT(io.value()->ReadFile(object_uri, std::nullopt), |
| 126 | + HasValue(::testing::Eq(std::string(kContent)))); |
| 127 | + EXPECT_THAT(io.value()->DeleteFile(object_uri), IsOk()); |
| 128 | +} |
| 129 | + |
| 130 | +TEST_F(RestArrowFileIOTest, AppliesOssCredentialThroughRealArrowS3FileIO) { |
| 131 | + auto logger = std::make_shared<CapturingLogger>(); |
| 132 | + ScopedDefaultLogger scoped(logger); |
| 133 | + |
| 134 | + auto io = |
| 135 | + MakeTableFileIO({{"warehouse", "logical_warehouse_name"}}, /*table_config=*/{}, |
| 136 | + {{.prefix = "oss://bucket/table", .config = {{"k", "v"}}}}); |
| 137 | + ASSERT_THAT(io, IsOk()); |
| 138 | + |
| 139 | + // Opening builds the delegate and applies the credential. The open itself hits |
| 140 | + // the network, so only the failure modes before that are asserted: a routing |
| 141 | + // break surfaces as kNotSupported, and a dropped credential as the warning. |
| 142 | + auto input = io.value()->NewInputFile("oss://bucket/table/data/file.parquet"); |
| 143 | + EXPECT_THAT(input, ::testing::Not(IsError(ErrorKind::kNotSupported))); |
| 144 | + EXPECT_FALSE(HasWarning(*logger)); |
| 145 | +} |
| 146 | + |
| 147 | +#endif // ICEBERG_S3_ENABLED |
| 148 | + |
| 149 | +} // namespace |
| 150 | + |
| 151 | +} // namespace iceberg::rest |
0 commit comments