From 912ccb94ba5629768330b2861b4aab51781f2df8 Mon Sep 17 00:00:00 2001 From: Zhuo Wang Date: Thu, 19 Nov 2026 11:32:23 +0800 Subject: [PATCH] test: add conformance tests over apache/iceberg-verification fixtures Pin apache/iceberg-verification as a git submodule at testing/iceberg-verification and add verification_test, which runs the schema and types surfaces of its table-spec fixtures through SchemaFromJson and TypeFromJson. The test is built only with -DICEBERG_BUILD_VERIFICATION_TESTS=ON (off by default) and is not run in CI yet. With the option on, CMake fails at configure time if the fixtures are missing. Every case that diverges from its fixture fails; advisory (normative_level: should) cases skip. Four cases currently fail on implementation gaps: - schema unknown/reject-unknown-required: SchemaFromJson accepts a required field of type unknown. - types primitive/decimal-9-2 and decimal-9-2-spaced: ToJson writes decimal(P,S) without the space the fixtures expect. - types primitive/decimal-precision-over-38: TypeFromJson throws from the DecimalType constructor instead of returning an error. --- .github/.licenserc.yaml | 2 + .gitmodules | 3 + CMakeLists.txt | 2 + README.md | 14 ++ dev/release/rat_exclude_files.txt | 1 + mkdocs/docs/getting-started.md | 1 + src/iceberg/test/CMakeLists.txt | 24 +++ src/iceberg/test/test_config.h.in | 1 + .../verification/schema_verification_test.cc | 150 ++++++++++++++++ .../verification/types_verification_test.cc | 165 ++++++++++++++++++ .../test/verification/verification_util.cc | 163 +++++++++++++++++ .../test/verification/verification_util.h | 105 +++++++++++ testing/iceberg-verification | 1 + 13 files changed, 632 insertions(+) create mode 100644 .gitmodules create mode 100644 src/iceberg/test/verification/schema_verification_test.cc create mode 100644 src/iceberg/test/verification/types_verification_test.cc create mode 100644 src/iceberg/test/verification/verification_util.cc create mode 100644 src/iceberg/test/verification/verification_util.h create mode 160000 testing/iceberg-verification diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index 80031cae30..9331a3624d 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -25,6 +25,7 @@ header: paths-ignore: - '**/.gitignore' + - '.gitmodules' - '.github/**' - 'LICENSE' - 'NOTICE' @@ -33,6 +34,7 @@ header: - 'src/iceberg/util/murmurhash3_internal.*' - 'src/iceberg/test/resources/**' - 'src/iceberg/catalog/hive/gen-cpp/**' + - 'testing/iceberg-verification/**' - 'thirdparty/**' comment: on-failure diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..796c28d1d3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "testing/iceberg-verification"] + path = testing/iceberg-verification + url = https://github.com/apache/iceberg-verification.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a9c3b2ff1..d724585e71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,8 @@ option(ICEBERG_ENCRYPTION "Build table encryption support (Parquet modular encryption, AES GCM streams)" OFF) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) +option(ICEBERG_BUILD_VERIFICATION_TESTS + "Build conformance tests over the apache/iceberg-verification fixtures" OFF) option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF) option(ICEBERG_BUILD_SQL_CATALOG "Build SQL catalog client" OFF) # Built-in SQL catalog database connectors. Disable all of them to build a SQL diff --git a/README.md b/README.md index 12755f65f3..75ba297fb7 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,19 @@ To run a specific test suite: ctest --test-dir build -R schema_test --output-on-failure ``` +`verification_test` runs the conformance fixtures from +[apache/iceberg-verification](https://github.com/apache/iceberg-verification), +pinned as a git submodule at `testing/iceberg-verification`. It is built only +with `-DICEBERG_BUILD_VERIFICATION_TESTS=ON`, and CMake fails at configure time +when the submodule is not checked out. To fetch it: + +```bash +git submodule update --init testing/iceberg-verification +``` + +Pass `-DICEBERG_VERIFICATION_DIR=/path/to/iceberg-verification` to run against +another checkout. + ### Build and Install Iceberg Bundle Library #### Vendored Apache Arrow (default) @@ -101,6 +114,7 @@ cmake --install build | `ICEBERG_BUILD_BUNDLE` | `ON` | Build the battery-included library | | `ICEBERG_BUILD_REST` | `ON` | Build REST catalog client | | `ICEBERG_BUILD_REST_INTEGRATION_TESTS` | `OFF` | Build REST catalog integration tests | +| `ICEBERG_BUILD_VERIFICATION_TESTS` | `OFF` | Build conformance tests over the apache/iceberg-verification fixtures | | `ICEBERG_SPDLOG` | `ON` | Use spdlog as the default logging backend | | `ICEBERG_BUILD_HIVE` | `OFF` | Build Hive (HMS) catalog client | | `ICEBERG_BUILD_SQL_CATALOG` | `OFF` | Build SQL catalog client | diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 9afe473d7e..7b5b3b10c8 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -17,6 +17,7 @@ .github/copilot-instructions.md .gitignore +.gitmodules LICENSE NOTICE build/** diff --git a/mkdocs/docs/getting-started.md b/mkdocs/docs/getting-started.md index 41d554c429..d35d9057a6 100644 --- a/mkdocs/docs/getting-started.md +++ b/mkdocs/docs/getting-started.md @@ -74,6 +74,7 @@ cmake --install build | `ICEBERG_BUILD_BUNDLE` | `ON` | Build the battery-included library | | `ICEBERG_BUILD_REST` | `ON` | Build REST catalog client | | `ICEBERG_BUILD_REST_INTEGRATION_TESTS` | `OFF` | Build REST catalog integration tests | +| `ICEBERG_BUILD_VERIFICATION_TESTS` | `OFF` | Build conformance tests over the apache/iceberg-verification fixtures | | `ICEBERG_BUILD_HIVE` | `OFF` | Build Hive (HMS) catalog client | | `ICEBERG_BUILD_SQL_CATALOG` | `OFF` | Build SQL catalog client | | `ICEBERG_SQL_SQLITE` | `OFF` | Build the SQLite connector for the SQL catalog | diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 1874fc25e8..3627e695d8 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -19,6 +19,22 @@ set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources") +# Conformance fixtures from apache/iceberg-verification, pinned as a git submodule. +if(ICEBERG_BUILD_VERIFICATION_TESTS) + set(ICEBERG_VERIFICATION_DIR + "${PROJECT_SOURCE_DIR}/testing/iceberg-verification" + CACHE PATH "Path to an apache/iceberg-verification checkout") + # The path is pasted into a C string literal, so turn Windows backslashes into + # forward slashes, which would otherwise read as escape sequences. + cmake_path(SET ICEBERG_VERIFICATION_DIR "${ICEBERG_VERIFICATION_DIR}") + if(NOT IS_DIRECTORY "${ICEBERG_VERIFICATION_DIR}/table-spec") + message(FATAL_ERROR "ICEBERG_BUILD_VERIFICATION_TESTS is ON but no " + "iceberg-verification fixtures were found at " + "${ICEBERG_VERIFICATION_DIR}. Run `git submodule update --init " + "testing/iceberg-verification` or set ICEBERG_VERIFICATION_DIR.") + endif() +endif() + configure_file("test_config.h.in" "test_config.h") # vcpkg's z-applocal POST_BUILD copies dependency DLLs into this shared test @@ -133,6 +149,14 @@ add_iceberg_test(json_serde_test metadata_serde_test.cc schema_json_test.cc) +if(ICEBERG_BUILD_VERIFICATION_TESTS) + add_iceberg_test(verification_test + SOURCES + verification/schema_verification_test.cc + verification/types_verification_test.cc + verification/verification_util.cc) +endif() + add_iceberg_test(scheduled_executor_test SOURCES scheduled_executor_test.cc) set_tests_properties(scheduled_executor_test PROPERTIES TIMEOUT 60) diff --git a/src/iceberg/test/test_config.h.in b/src/iceberg/test/test_config.h.in index 0f0ec5f66a..b9508dcfc6 100644 --- a/src/iceberg/test/test_config.h.in +++ b/src/iceberg/test/test_config.h.in @@ -20,3 +20,4 @@ #pragma once #define ICEBERG_TEST_RESOURCES "@ICEBERG_TEST_RESOURCES@" +#define ICEBERG_VERIFICATION_DIR "@ICEBERG_VERIFICATION_DIR@" diff --git a/src/iceberg/test/verification/schema_verification_test.cc b/src/iceberg/test/verification/schema_verification_test.cc new file mode 100644 index 0000000000..a56ad80080 --- /dev/null +++ b/src/iceberg/test/verification/schema_verification_test.cc @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Runs the `table-spec/schema` surface: project(parse(input)) == decoded, where the +// projection is one row per field in document order, read through public accessors. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "iceberg/json_serde_internal.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/test/verification/verification_util.h" +#include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" + +namespace iceberg::verification { + +namespace { + +constexpr std::array kSubdirs = {"core", "unknown"}; + +// Fields of the `decoded` projection specific to this surface. +constexpr std::string_view kSchemaId = "schema-id"; +constexpr std::string_view kIdentifierFieldIds = "identifier-field-ids"; +constexpr std::string_view kParent = "parent"; +constexpr std::string_view kDoc = "doc"; + +/// \brief The `type` of a row: an object for decimal and fixed, else the type name. +nlohmann::json ProjectFieldType(const Type& type) { + switch (type.type_id()) { + case TypeId::kDecimal: { + const auto& decimal = internal::checked_cast(type); + return {{kType, "decimal"}, + {kPrecision, decimal.precision()}, + {kScale, decimal.scale()}}; + } + case TypeId::kFixed: + return {{kType, "fixed"}, + {kLength, internal::checked_cast(type).length()}}; + default: + return std::string(ToString(type.type_id())); + } +} + +/// \brief Append a row for each field of `type`, each followed by its children. +void AppendRows(const NestedType& type, std::optional parent_id, + nlohmann::json* rows) { + for (const auto& field : type.fields()) { + rows->push_back({ + {kId, field.field_id()}, + {kName, std::string(field.name())}, + {kParent, parent_id.has_value() ? nlohmann::json(*parent_id) : nullptr}, + {kRequired, !field.optional()}, + {kType, ProjectFieldType(*field.type())}, + {kDoc, field.doc().empty() ? nullptr : nlohmann::json(std::string(field.doc()))}, + }); + if (field.type()->is_nested()) { + AppendRows(internal::checked_cast(*field.type()), + field.field_id(), rows); + } + } +} + +nlohmann::json ProjectSchema(const Schema& schema) { + nlohmann::json rows = nlohmann::json::array(); + AppendRows(schema, std::nullopt, &rows); + return {{kSchemaId, schema.schema_id()}, + {kIdentifierFieldIds, schema.IdentifierFieldIds()}, + {kFields, std::move(rows)}}; +} + +std::vector CheckCase(const nlohmann::json& c) { + std::optional>> parsed; + try { + parsed = SchemaFromJson(c.at(kInput)); + } catch (const std::exception& e) { + return {std::format("parse threw instead of returning an error: {}", e.what())}; + } + + if (!c.at(kValid).get()) { + if (parsed->has_value()) { + return {std::format("expected rejection, but parsed as {}", + parsed->value()->ToString())}; + } + return {}; + } + if (!parsed->has_value()) { + return {std::format("parse failed: {}", parsed->error().message)}; + } + + // Rows keep document order, which is the column order under test, but the spec + // calls identifier-field-ids a set. + auto sorted_ids = [](const nlohmann::json& ids) { + auto values = ids.get>(); + std::ranges::sort(values); + return values; + }; + auto expected = c.at(kDecoded); + auto actual = ProjectSchema(*parsed->value()); + expected[kIdentifierFieldIds] = sorted_ids(expected.at(kIdentifierFieldIds)); + actual[kIdentifierFieldIds] = sorted_ids(actual.at(kIdentifierFieldIds)); + std::vector diffs; + DiffJson(expected, actual, "", &diffs); + return diffs; +} + +class SchemaVerificationTest : public ::testing::TestWithParam {}; + +TEST_P(SchemaVerificationTest, Parse) { + const auto& c = GetParam(); + if (!c.load_error.empty()) { + FAIL() << c.load_error; + } + ReportOutcome(c, CheckCase(c.json)); +} + +INSTANTIATE_TEST_SUITE_P(Schema, SchemaVerificationTest, + ::testing::ValuesIn(LoadCases("schema", kSubdirs)), CaseName); +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(SchemaVerificationTest); + +} // namespace + +} // namespace iceberg::verification diff --git a/src/iceberg/test/verification/types_verification_test.cc b/src/iceberg/test/verification/types_verification_test.cc new file mode 100644 index 0000000000..f65831e152 --- /dev/null +++ b/src/iceberg/test/verification/types_verification_test.cc @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Runs the `table-spec/types` surface: parse(input) == decoded, and where the case +// carries `canonical`, re-serializing the parsed type reproduces it byte for byte. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "iceberg/json_serde_internal.h" +#include "iceberg/schema_field.h" +#include "iceberg/test/verification/verification_util.h" +#include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" + +namespace iceberg::verification { + +namespace { + +constexpr std::array kSubdirs = {"primitive", "nested", "variant", + "geospatial"}; + +// Fields specific to this surface: the case's `canonical` and the rest from the +// `decoded` projection. +constexpr std::string_view kCanonical = "canonical"; +constexpr std::string_view kCrs = "crs"; +constexpr std::string_view kAlgorithm = "algorithm"; +constexpr std::string_view kElementId = "element-id"; +constexpr std::string_view kElementRequired = "element-required"; +constexpr std::string_view kElement = "element"; +constexpr std::string_view kKeyId = "key-id"; +constexpr std::string_view kKey = "key"; +constexpr std::string_view kValueId = "value-id"; +constexpr std::string_view kValueRequired = "value-required"; +constexpr std::string_view kValue = "value"; + +/// \brief Map a type to the language-neutral `decoded` shape of the types surface. +nlohmann::json ProjectType(const Type& type) { + nlohmann::json json{{kType, std::string(ToString(type.type_id()))}}; + switch (type.type_id()) { + case TypeId::kDecimal: { + const auto& decimal = internal::checked_cast(type); + json[kPrecision] = decimal.precision(); + json[kScale] = decimal.scale(); + break; + } + case TypeId::kFixed: + json[kLength] = internal::checked_cast(type).length(); + break; + case TypeId::kGeometry: + json[kCrs] = std::string(internal::checked_cast(type).crs()); + break; + case TypeId::kGeography: { + const auto& geography = internal::checked_cast(type); + json[kCrs] = std::string(geography.crs()); + json[kAlgorithm] = std::string(ToString(geography.algorithm())); + break; + } + case TypeId::kStruct: { + json[kFields] = nlohmann::json::array(); + for (const auto& field : internal::checked_cast(type).fields()) { + json[kFields].push_back({{kId, field.field_id()}, + {kName, std::string(field.name())}, + {kRequired, !field.optional()}, + {kType, ProjectType(*field.type())}}); + } + break; + } + case TypeId::kList: { + const auto& element = internal::checked_cast(type).element(); + json[kElementId] = element.field_id(); + json[kElementRequired] = !element.optional(); + json[kElement] = ProjectType(*element.type()); + break; + } + case TypeId::kMap: { + const auto& map = internal::checked_cast(type); + json[kKeyId] = map.key().field_id(); + json[kKey] = ProjectType(*map.key().type()); + json[kValueId] = map.value().field_id(); + json[kValueRequired] = !map.value().optional(); + json[kValue] = ProjectType(*map.value().type()); + break; + } + default: + break; + } + return json; +} + +std::vector CheckCase(const nlohmann::json& c) { + std::optional>> parsed; + try { + parsed = TypeFromJson(c.at(kInput)); + } catch (const std::exception& e) { + return {std::format("parse threw instead of returning an error: {}", e.what())}; + } + + if (!c.at(kValid).get()) { + if (parsed->has_value()) { + return {std::format("expected rejection, but parsed as {}", + parsed->value()->ToString())}; + } + return {}; + } + if (!parsed->has_value()) { + return {std::format("parse failed: {}", parsed->error().message)}; + } + + const auto& type = *parsed->value(); + std::vector diffs; + DiffJson(c.at(kDecoded), ProjectType(type), "", &diffs); + if (c.contains(kCanonical)) { + auto written = ToJson(type); + if (!written.has_value()) { + diffs.push_back(std::format("serialize failed: {}", written.error().message)); + } else { + DiffJson(c[kCanonical], *written, std::string(kCanonical), &diffs); + } + } + return diffs; +} + +class TypesVerificationTest : public ::testing::TestWithParam {}; + +TEST_P(TypesVerificationTest, Parse) { + const auto& c = GetParam(); + if (!c.load_error.empty()) { + FAIL() << c.load_error; + } + ReportOutcome(c, CheckCase(c.json)); +} + +INSTANTIATE_TEST_SUITE_P(Types, TypesVerificationTest, + ::testing::ValuesIn(LoadCases("types", kSubdirs)), CaseName); +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TypesVerificationTest); + +} // namespace + +} // namespace iceberg::verification diff --git a/src/iceberg/test/verification/verification_util.cc b/src/iceberg/test/verification/verification_util.cc new file mode 100644 index 0000000000..75bf80832b --- /dev/null +++ b/src/iceberg/test/verification/verification_util.cc @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/test/verification/verification_util.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/test/test_config.h" + +namespace iceberg::verification { + +namespace { + +// Fixture fields read only by the loader and the outcome report. +constexpr std::string_view kCases = "cases"; +constexpr std::string_view kNormativeLevel = "normative_level"; + +std::filesystem::path TableSpecDir() { + return std::filesystem::path(ICEBERG_VERIFICATION_DIR) / "table-spec"; +} + +std::string JoinDiffs(const std::vector& diffs) { + std::string out; + for (const auto& diff : diffs) { + out += "\n "; + out += diff; + } + return out; +} + +} // namespace + +bool FixturesAvailable() { return std::filesystem::is_directory(TableSpecDir()); } + +std::vector LoadCases(std::string_view surface, + std::span subdirs) { + std::vector cases; + if (!FixturesAvailable()) { + return cases; + } + for (auto subdir : subdirs) { + auto path = TableSpecDir() / surface / subdir / "cases.json"; + auto load_error = [&](std::string message) { + cases.push_back({.subdir = std::string(subdir), + .id = "load", + .load_error = std::format("{}: {}", path.string(), message)}); + }; + std::ifstream file(path); + if (!file) { + load_error("cannot open file"); + continue; + } + nlohmann::json fixture; + try { + fixture = nlohmann::json::parse(file); + } catch (const std::exception& e) { + load_error(e.what()); + continue; + } + if (!fixture.contains(kCases) || !fixture[kCases].is_array()) { + load_error("no `cases` array"); + continue; + } + for (auto& json : fixture[kCases]) { + cases.push_back({.subdir = std::string(subdir), + .id = json.value(kId, std::string()), + .json = std::move(json)}); + } + } + return cases; +} + +std::string CaseName(const ::testing::TestParamInfo& info) { + std::string name = info.param.Label(); + for (auto& c : name) { + if (!std::isalnum(static_cast(c))) { + c = '_'; + } + } + return name; +} + +void DiffJson(const nlohmann::json& expected, const nlohmann::json& actual, + const std::string& path, std::vector* diffs) { + if (expected.is_object() && actual.is_object()) { + for (const auto& [key, value] : expected.items()) { + auto child = path.empty() ? key : path + "." + key; + if (!actual.contains(key)) { + diffs->push_back(std::format("{}: missing, expected {}", child, value.dump())); + } else { + DiffJson(value, actual.at(key), child, diffs); + } + } + for (const auto& [key, value] : actual.items()) { + if (!expected.contains(key)) { + auto child = path.empty() ? key : path + "." + key; + diffs->push_back(std::format("{}: unexpected {}", child, value.dump())); + } + } + return; + } + if (expected.is_array() && actual.is_array()) { + auto common = std::min(expected.size(), actual.size()); + for (size_t i = 0; i < common; ++i) { + DiffJson(expected[i], actual[i], std::format("{}[{}]", path, i), diffs); + } + if (expected.size() != actual.size()) { + diffs->push_back(std::format("{}: expected {} elements, got {}", path, + expected.size(), actual.size())); + } + return; + } + if (expected != actual) { + diffs->push_back(std::format("{}: expected {}, got {}", path.empty() ? "value" : path, + expected.dump(), actual.dump())); + } +} + +void ReportOutcome(const VerificationCase& c, const std::vector& diffs) { + const auto label = c.Label(); + if (diffs.empty()) { + return; + } + if (c.json.value(kNormativeLevel, std::string("must")) == "should") { + GTEST_SKIP() << "advisory_fail " << label << JoinDiffs(diffs); + } + for (const auto& diff : diffs) { + ADD_FAILURE() << label << " " << diff; + } +} + +// The parameterized suites have no cases when the fixtures are absent. CMake checks +// the directory at configure time; this test catches it going missing afterwards, +// so the run fails instead of passing with no cases. +TEST(VerificationFixtures, Available) { + ASSERT_TRUE(FixturesAvailable()) + << "iceberg-verification fixtures not found at " << ICEBERG_VERIFICATION_DIR + << "; run `git submodule update --init testing/iceberg-verification`"; +} + +} // namespace iceberg::verification diff --git a/src/iceberg/test/verification/verification_util.h b/src/iceberg/test/verification/verification_util.h new file mode 100644 index 0000000000..07e8b403bc --- /dev/null +++ b/src/iceberg/test/verification/verification_util.h @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/test/verification/verification_util.h +/// Helpers for running the apache/iceberg-verification conformance fixtures. +/// +/// Each surface under `table-spec/` holds one `cases.json` per subdirectory. A test +/// subscribes to subdirectories by name, runs every case through the public API, and +/// reports differences labeled `/ `, as the surface READMEs +/// require. + +#include +#include +#include +#include +#include + +#include +#include + +namespace iceberg::verification { + +/// \name Fields of a fixture `cases.json` and its case objects. +/// @{ +inline constexpr std::string_view kId = "id"; +inline constexpr std::string_view kInput = "input"; +inline constexpr std::string_view kValid = "valid"; +inline constexpr std::string_view kDecoded = "decoded"; +/// @} + +/// \name Fields of the `decoded` projections shared by the schema and types surfaces. +/// @{ +inline constexpr std::string_view kType = "type"; +inline constexpr std::string_view kName = "name"; +inline constexpr std::string_view kRequired = "required"; +inline constexpr std::string_view kFields = "fields"; +inline constexpr std::string_view kPrecision = "precision"; +inline constexpr std::string_view kScale = "scale"; +inline constexpr std::string_view kLength = "length"; +/// @} + +/// \brief A case from a fixture `cases.json`, or a fixture that failed to load. +struct VerificationCase { + /// \brief Subdirectory of the surface, e.g. `primitive`. + std::string subdir; + /// \brief Case id, unique within the surface. + std::string id; + /// \brief The case object as written in the fixture. + nlohmann::json json; + /// \brief Set when the fixture could not be loaded; the case then fails with it. + std::string load_error; + + /// \brief `/`, the label the fixture READMEs use. + std::string Label() const { return subdir + "/" + id; } +}; + +/// \brief Print a case as its label in GTest output. +inline void PrintTo(const VerificationCase& c, std::ostream* os) { *os << c.Label(); } + +/// \brief Whether the fixture checkout at ICEBERG_VERIFICATION_DIR is present. +bool FixturesAvailable(); + +/// \brief Load the cases of the given subdirectories of `table-spec//`. +/// +/// Returns no cases when the fixtures are absent. A subscribed subdirectory whose +/// `cases.json` is missing or malformed yields one case carrying the load error, so +/// the problem fails a test instead of silently dropping cases. +std::vector LoadCases(std::string_view surface, + std::span subdirs); + +/// \brief GTest parameter name for a case, derived from its label. +std::string CaseName(const ::testing::TestParamInfo& info); + +/// \brief Append every difference between `expected` and `actual` to `diffs`. +/// +/// Each difference is labeled with its path, e.g. `fields[2].parent`. +void DiffJson(const nlohmann::json& expected, const nlohmann::json& actual, + const std::string& path, std::vector* diffs); + +/// \brief Turn the differences found for a case into a test outcome. +/// +/// No differences pass. Differences fail the test, unless the case is advisory +/// (`normative_level: should`), in which case the test is skipped with the +/// differences in the message. +void ReportOutcome(const VerificationCase& c, const std::vector& diffs); + +} // namespace iceberg::verification diff --git a/testing/iceberg-verification b/testing/iceberg-verification new file mode 160000 index 0000000000..fb0be4d6cb --- /dev/null +++ b/testing/iceberg-verification @@ -0,0 +1 @@ +Subproject commit fb0be4d6cb2af5529a176b976b4a775dc87b778f