Skip to content

Commit 07df136

Browse files
committed
fix: reject required unknown fields in Schema::Make
The spec requires a field of type unknown to be optional and forbids an unknown map key. Schema::Validate already checked this, but only table metadata parsing and schema updates call it, so SchemaFromJson and other Schema::Make callers accepted such schemas. Run the nullability check in both Schema::Make overloads. Java does the same check when it constructs a NestedField. Closes #1141.
1 parent dc57c8f commit 07df136

4 files changed

Lines changed: 52 additions & 2 deletions

File tree

‎src/iceberg/schema.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ Result<std::unique_ptr<Schema>> Schema::Make(std::vector<SchemaField> fields,
191191
std::vector<int32_t> identifier_field_ids,
192192
GetId get_id) {
193193
auto schema = std::make_unique<Schema>(std::move(fields), schema_id, std::move(get_id));
194+
ICEBERG_RETURN_UNEXPECTED(ValidateFieldNullability(*schema));
194195

195196
if (!identifier_field_ids.empty()) {
196197
auto id_to_parent = IndexParents(*schema);
@@ -208,6 +209,7 @@ Result<std::unique_ptr<Schema>> Schema::Make(
208209
std::vector<SchemaField> fields, int32_t schema_id,
209210
const std::vector<std::string>& identifier_field_names, GetId get_id) {
210211
auto schema = std::make_unique<Schema>(std::move(fields), schema_id, std::move(get_id));
212+
ICEBERG_RETURN_UNEXPECTED(ValidateFieldNullability(*schema));
211213

212214
std::vector<int32_t> fresh_identifier_ids;
213215
for (const auto& name : identifier_field_names) {

‎src/iceberg/schema.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ class ICEBERG_EXPORT Schema : public StructType {
7474
/// \param schema_id The unique identifier for this schema (default:kInitialSchemaId).
7575
/// \param identifier_field_ids Field IDs that uniquely identify rows in the table.
7676
/// \param get_id Function mapping each original field ID to its reassigned ID.
77-
/// \return A new Schema instance or Status if failed.
77+
/// \return A new Schema instance, or an error if a field of type unknown is
78+
/// required, a map key is of type unknown, or an identifier field is invalid.
7879
static Result<std::unique_ptr<Schema>> Make(std::vector<SchemaField> fields,
7980
int32_t schema_id,
8081
std::vector<int32_t> identifier_field_ids,
@@ -87,7 +88,8 @@ class ICEBERG_EXPORT Schema : public StructType {
8788
/// \param identifier_field_names Canonical names of fields that uniquely identify rows
8889
/// in the table.
8990
/// \param get_id Function mapping each original field ID to its reassigned ID.
90-
/// \return A new Schema instance or Status if failed.
91+
/// \return A new Schema instance, or an error if a field of type unknown is
92+
/// required, a map key is of type unknown, or an identifier field is invalid.
9193
static Result<std::unique_ptr<Schema>> Make(
9294
std::vector<SchemaField> fields, int32_t schema_id,
9395
const std::vector<std::string>& identifier_field_names, GetId get_id = {});

‎src/iceberg/test/schema_json_test.cc‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,36 @@ TEST(SchemaJsonTest, NestedUnknownFieldsRoundTrip) {
327327
ASSERT_EQ(schema_json, parsed_json);
328328
}
329329

330+
TEST(SchemaJsonTest, RejectsRequiredUnknownFields) {
331+
constexpr std::string_view required_field =
332+
R"({"fields":[{"id":1,"name":"mystery","required":true,"type":"unknown"}],"type":"struct"})";
333+
auto result = SchemaFromJson(nlohmann::json::parse(required_field));
334+
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
335+
EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional"));
336+
337+
constexpr std::string_view required_nested_field =
338+
R"({"fields":[{"id":1,"name":"profile","required":false,"type":{"type":"struct","fields":[
339+
{"id":2,"name":"mystery","required":true,"type":"unknown"}]}}],"type":"struct"})";
340+
result = SchemaFromJson(nlohmann::json::parse(required_nested_field));
341+
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
342+
EXPECT_THAT(result, HasErrorMessage("Unknown type field 'mystery' must be optional"));
343+
344+
constexpr std::string_view required_element =
345+
R"({"fields":[{"id":1,"name":"mysteries","required":false,"type":{"type":"list",
346+
"element-id":2,"element-required":true,"element":"unknown"}}],"type":"struct"})";
347+
result = SchemaFromJson(nlohmann::json::parse(required_element));
348+
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
349+
EXPECT_THAT(result, HasErrorMessage("Unknown type field 'element' must be optional"));
350+
351+
constexpr std::string_view unknown_key =
352+
R"({"fields":[{"id":1,"name":"properties","required":false,"type":{"type":"map",
353+
"key-id":2,"key":"unknown","value-id":3,"value-required":false,"value":"string"}}],
354+
"type":"struct"})";
355+
result = SchemaFromJson(nlohmann::json::parse(unknown_key));
356+
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
357+
EXPECT_THAT(result, HasErrorMessage("Map 'key' cannot be unknown type"));
358+
}
359+
330360
TEST(SchemaJsonTest, IdentifierFieldIds) {
331361
// Test schema with identifier-field-ids
332362
constexpr std::string_view json_with_identifier_str =

‎src/iceberg/test/schema_test.cc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,22 @@ TEST(SchemaTest, ValidateRejectsInvalidUnknownFields) {
311311
EXPECT_THAT(status, iceberg::HasErrorMessage("Map 'key' cannot be unknown type"));
312312
}
313313

314+
TEST(SchemaTest, MakeRejectsRequiredUnknownField) {
315+
auto by_ids = iceberg::Schema::Make(
316+
{iceberg::SchemaField::MakeRequired(1, "mystery", iceberg::unknown())},
317+
iceberg::Schema::kInitialSchemaId, std::vector<int32_t>{});
318+
ASSERT_THAT(by_ids, iceberg::IsError(iceberg::ErrorKind::kInvalidArgument));
319+
EXPECT_THAT(by_ids,
320+
iceberg::HasErrorMessage("Unknown type field 'mystery' must be optional"));
321+
322+
auto by_names = iceberg::Schema::Make(
323+
{iceberg::SchemaField::MakeRequired(1, "mystery", iceberg::unknown())},
324+
iceberg::Schema::kInitialSchemaId, std::vector<std::string>{});
325+
ASSERT_THAT(by_names, iceberg::IsError(iceberg::ErrorKind::kInvalidArgument));
326+
EXPECT_THAT(by_names,
327+
iceberg::HasErrorMessage("Unknown type field 'mystery' must be optional"));
328+
}
329+
314330
TEST(SchemaTest, IdentifierFields) {
315331
using iceberg::ErrorKind;
316332
using iceberg::Schema;

0 commit comments

Comments
 (0)