From 7f49f2c4289d98667cea9bc7f09f72d2a9372db4 Mon Sep 17 00:00:00 2001 From: Zhuo Wang Date: Thu, 19 Nov 2026 11:46:55 +0800 Subject: [PATCH] fix: reject required unknown fields in Schema::Make The spec requires a field of type unknown to be optional and forbids an unknown map key. Schema::Validate already checked this, but only table metadata parsing and schema updates call it, so SchemaFromJson and other Schema::Make callers accepted such schemas. Run the nullability check in both Schema::Make overloads. Java does the same check when it constructs a NestedField. Closes #1141. --- src/iceberg/schema.cc | 2 ++ src/iceberg/schema.h | 6 ++++-- src/iceberg/test/schema_json_test.cc | 30 ++++++++++++++++++++++++++++ src/iceberg/test/schema_test.cc | 16 +++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/iceberg/schema.cc b/src/iceberg/schema.cc index 8f103a9d16..2979256a52 100644 --- a/src/iceberg/schema.cc +++ b/src/iceberg/schema.cc @@ -191,6 +191,7 @@ Result> Schema::Make(std::vector fields, std::vector identifier_field_ids, GetId get_id) { auto schema = std::make_unique(std::move(fields), schema_id, std::move(get_id)); + ICEBERG_RETURN_UNEXPECTED(ValidateFieldNullability(*schema)); if (!identifier_field_ids.empty()) { auto id_to_parent = IndexParents(*schema); @@ -208,6 +209,7 @@ Result> Schema::Make( std::vector fields, int32_t schema_id, const std::vector& identifier_field_names, GetId get_id) { auto schema = std::make_unique(std::move(fields), schema_id, std::move(get_id)); + ICEBERG_RETURN_UNEXPECTED(ValidateFieldNullability(*schema)); std::vector fresh_identifier_ids; for (const auto& name : identifier_field_names) { diff --git a/src/iceberg/schema.h b/src/iceberg/schema.h index f43d064930..feac4f2af2 100644 --- a/src/iceberg/schema.h +++ b/src/iceberg/schema.h @@ -74,7 +74,8 @@ class ICEBERG_EXPORT Schema : public StructType { /// \param schema_id The unique identifier for this schema (default:kInitialSchemaId). /// \param identifier_field_ids Field IDs that uniquely identify rows in the table. /// \param get_id Function mapping each original field ID to its reassigned ID. - /// \return A new Schema instance or Status if failed. + /// \return A new Schema instance, or an error if a field of type unknown is + /// required, a map key is of type unknown, or an identifier field is invalid. static Result> Make(std::vector fields, int32_t schema_id, std::vector identifier_field_ids, @@ -87,7 +88,8 @@ class ICEBERG_EXPORT Schema : public StructType { /// \param identifier_field_names Canonical names of fields that uniquely identify rows /// in the table. /// \param get_id Function mapping each original field ID to its reassigned ID. - /// \return A new Schema instance or Status if failed. + /// \return A new Schema instance, or an error if a field of type unknown is + /// required, a map key is of type unknown, or an identifier field is invalid. static Result> Make( std::vector fields, int32_t schema_id, const std::vector& identifier_field_names, GetId get_id = {}); diff --git a/src/iceberg/test/schema_json_test.cc b/src/iceberg/test/schema_json_test.cc index 5067191bf9..622e6535d3 100644 --- a/src/iceberg/test/schema_json_test.cc +++ b/src/iceberg/test/schema_json_test.cc @@ -327,6 +327,36 @@ TEST(SchemaJsonTest, NestedUnknownFieldsRoundTrip) { ASSERT_EQ(schema_json, parsed_json); } +TEST(SchemaJsonTest, RejectsRequiredUnknownFields) { + constexpr std::string_view required_field = + R"({"fields":[{"id":1,"name":"mystery","required":true,"type":"unknown"}],"type":"struct"})"; + auto result = SchemaFromJson(nlohmann::json::parse(required_field)); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); + + constexpr std::string_view required_nested_field = + R"({"fields":[{"id":1,"name":"profile","required":false,"type":{"type":"struct","fields":[ + {"id":2,"name":"mystery","required":true,"type":"unknown"}]}}],"type":"struct"})"; + result = SchemaFromJson(nlohmann::json::parse(required_nested_field)); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional")); + + constexpr std::string_view required_element = + R"({"fields":[{"id":1,"name":"mysteries","required":false,"type":{"type":"list", + "element-id":2,"element-required":true,"element":"unknown"}}],"type":"struct"})"; + result = SchemaFromJson(nlohmann::json::parse(required_element)); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(result, HasErrorMessage("Unknown type field 'element' must be optional")); + + constexpr std::string_view unknown_key = + R"({"fields":[{"id":1,"name":"properties","required":false,"type":{"type":"map", + "key-id":2,"key":"unknown","value-id":3,"value-required":false,"value":"string"}}], + "type":"struct"})"; + result = SchemaFromJson(nlohmann::json::parse(unknown_key)); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(result, HasErrorMessage("Map 'key' cannot be unknown type")); +} + TEST(SchemaJsonTest, IdentifierFieldIds) { // Test schema with identifier-field-ids constexpr std::string_view json_with_identifier_str = diff --git a/src/iceberg/test/schema_test.cc b/src/iceberg/test/schema_test.cc index ed0c5f5a1f..109643b6af 100644 --- a/src/iceberg/test/schema_test.cc +++ b/src/iceberg/test/schema_test.cc @@ -311,6 +311,22 @@ TEST(SchemaTest, ValidateRejectsInvalidUnknownFields) { EXPECT_THAT(status, iceberg::HasErrorMessage("Map 'key' cannot be unknown type")); } +TEST(SchemaTest, MakeRejectsRequiredUnknownField) { + auto by_ids = iceberg::Schema::Make( + {iceberg::SchemaField::MakeRequired(1, "mystery", iceberg::unknown())}, + iceberg::Schema::kInitialSchemaId, std::vector{}); + ASSERT_THAT(by_ids, iceberg::IsError(iceberg::ErrorKind::kInvalidArgument)); + EXPECT_THAT(by_ids, + iceberg::HasErrorMessage("Unknown type field 'mystery' must be optional")); + + auto by_names = iceberg::Schema::Make( + {iceberg::SchemaField::MakeRequired(1, "mystery", iceberg::unknown())}, + iceberg::Schema::kInitialSchemaId, std::vector{}); + ASSERT_THAT(by_names, iceberg::IsError(iceberg::ErrorKind::kInvalidArgument)); + EXPECT_THAT(by_names, + iceberg::HasErrorMessage("Unknown type field 'mystery' must be optional")); +} + TEST(SchemaTest, IdentifierFields) { using iceberg::ErrorKind; using iceberg::Schema;