From 7803766fcc49fe7b6d23daa4e5006164102fab20 Mon Sep 17 00:00:00 2001 From: Kam Cheung Ting Date: Thu, 17 Sep 2026 08:46:27 +0000 Subject: [PATCH 1/2] fix(serde): validate schema and partition IDs as integers --- src/iceberg/json_serde.cc | 37 ++++--- src/iceberg/test/json_serde_test.cc | 136 ++++++++++++++++++++++++ src/iceberg/test/metadata_serde_test.cc | 18 ++++ src/iceberg/util/json_util_internal.h | 12 +++ 4 files changed, 186 insertions(+), 17 deletions(-) diff --git a/src/iceberg/json_serde.cc b/src/iceberg/json_serde.cc index c7d0cb4344..c1c3ffb73f 100644 --- a/src/iceberg/json_serde.cc +++ b/src/iceberg/json_serde.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -768,7 +769,7 @@ Result> ListTypeFromJson(const nlohmann::json& json) { ICEBERG_ASSIGN_OR_RAISE( auto element_type, GetJsonValue(json, kElement).and_then(TypeFromJson)); - ICEBERG_ASSIGN_OR_RAISE(auto element_id, GetJsonValue(json, kElementId)); + ICEBERG_ASSIGN_OR_RAISE(auto element_id, GetJsonInteger(json, kElementId)); ICEBERG_ASSIGN_OR_RAISE(auto element_required, GetJsonValue(json, kElementRequired)); @@ -784,8 +785,8 @@ Result> MapTypeFromJson(const nlohmann::json& json) { ICEBERG_ASSIGN_OR_RAISE( auto value_type, GetJsonValue(json, kValue).and_then(TypeFromJson)); - ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonValue(json, kKeyId)); - ICEBERG_ASSIGN_OR_RAISE(auto value_id, GetJsonValue(json, kValueId)); + ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonInteger(json, kKeyId)); + ICEBERG_ASSIGN_OR_RAISE(auto value_id, GetJsonInteger(json, kValueId)); ICEBERG_ASSIGN_OR_RAISE(auto value_required, GetJsonValue(json, kValueRequired)); SchemaField key_field(key_id, std::string(MapType::kKeyName), std::move(key_type), @@ -946,7 +947,7 @@ Status ValidateTimestamptzDefaultIsUtc(const Type& type, const nlohmann::json& v Result> FieldFromJson(const nlohmann::json& json) { ICEBERG_ASSIGN_OR_RAISE( auto type, GetJsonValue(json, kType).and_then(TypeFromJson)); - ICEBERG_ASSIGN_OR_RAISE(auto field_id, GetJsonValue(json, kId)); + ICEBERG_ASSIGN_OR_RAISE(auto field_id, GetJsonInteger(json, kId)); ICEBERG_ASSIGN_OR_RAISE(auto name, GetJsonValue(json, kName)); ICEBERG_ASSIGN_OR_RAISE(auto required, GetJsonValue(json, kRequired)); ICEBERG_ASSIGN_OR_RAISE(auto doc, GetJsonValueOrDefault(json, kDoc)); @@ -979,7 +980,7 @@ Result> FieldFromJson(const nlohmann::json& json) { Result> SchemaFromJson(const nlohmann::json& json) { ICEBERG_ASSIGN_OR_RAISE(auto schema_id_opt, - GetJsonValueOptional(json, kSchemaId)); + GetJsonIntegerOptional(json, kSchemaId)); ICEBERG_ASSIGN_OR_RAISE(auto type, TypeFromJson(json)); if (type->type_id() != TypeId::kStruct) [[unlikely]] { @@ -993,9 +994,11 @@ Result> SchemaFromJson(const nlohmann::json& json) { fields.emplace_back(std::move(field)); } - ICEBERG_ASSIGN_OR_RAISE( - auto identifier_field_ids, - GetJsonValueOrDefault>(json, kIdentifierFieldIds)); + std::vector identifier_field_ids; + if (json.contains(kIdentifierFieldIds) && !json.at(kIdentifierFieldIds).is_null()) { + ICEBERG_ASSIGN_OR_RAISE(identifier_field_ids, + IntegerVectorFromJson(json, kIdentifierFieldIds)); + } return Schema::Make(std::move(fields), schema_id_opt.value_or(Schema::kInitialSchemaId), std::move(identifier_field_ids)); @@ -1028,14 +1031,13 @@ Result ToJsonString(const PartitionSpec& partition_spec) { Result> PartitionFieldFromJson( const nlohmann::json& json, bool allow_field_id_missing) { - ICEBERG_ASSIGN_OR_RAISE(auto source_id, GetJsonValue(json, kSourceId)); + ICEBERG_ASSIGN_OR_RAISE(auto source_id, GetJsonInteger(json, kSourceId)); int32_t field_id; - if (allow_field_id_missing) { + if (allow_field_id_missing && !json.contains(kFieldId)) { // Partition field id in v1 is not tracked, so we use -1 to indicate that. - ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonValueOrDefault( - json, kFieldId, SchemaField::kInvalidFieldId)); + field_id = SchemaField::kInvalidFieldId; } else { - ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonValue(json, kFieldId)); + ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonInteger(json, kFieldId)); } ICEBERG_ASSIGN_OR_RAISE( auto transform, @@ -1067,13 +1069,13 @@ Result> ParseBoundPartitionSpec( Result> PartitionSpecFromJson( const std::shared_ptr& schema, const nlohmann::json& json, int32_t default_spec_id) { - ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonValue(json, kSpecId)); + ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonInteger(json, kSpecId)); return ParseBoundPartitionSpec(*schema, json, spec_id, /*allow_missing_fields=*/spec_id != default_spec_id); } Result> PartitionSpecFromJson(const nlohmann::json& json) { - ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonValue(json, kSpecId)); + ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonInteger(json, kSpecId)); ICEBERG_ASSIGN_OR_RAISE(auto fields, GetJsonValue(json, kFields)); std::vector partition_fields; @@ -1438,7 +1440,7 @@ Result> ParseSchemas( } ICEBERG_ASSIGN_OR_RAISE(current_schema_id, - GetJsonValue(json, kCurrentSchemaId)); + GetJsonInteger(json, kCurrentSchemaId)); for (const auto& schema_json : schema_array) { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr schema, SchemaFromJson(schema_json)); @@ -1484,7 +1486,8 @@ Status ParsePartitionSpecs(const nlohmann::json& json, int8_t format_version, return JsonParseError("Cannot parse partition specs from non-array: {}", SafeDumpJson(spec_array)); } - ICEBERG_ASSIGN_OR_RAISE(default_spec_id, GetJsonValue(json, kDefaultSpecId)); + ICEBERG_ASSIGN_OR_RAISE(default_spec_id, + GetJsonInteger(json, kDefaultSpecId)); for (const auto& spec_json : spec_array) { ICEBERG_ASSIGN_OR_RAISE( diff --git a/src/iceberg/test/json_serde_test.cc b/src/iceberg/test/json_serde_test.cc index be363ea3b0..67656fe753 100644 --- a/src/iceberg/test/json_serde_test.cc +++ b/src/iceberg/test/json_serde_test.cc @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include @@ -209,6 +210,70 @@ TEST(JsonInternalTest, ListTypeFromJsonMissingElement) { EXPECT_THAT(result, HasErrorMessage("Missing 'element'")); } +TEST(JsonInternalTest, SchemaIdsRejectInvalidJsonIntegers) { + const nlohmann::json valid_schema = R"({ + "type": "struct", + "schema-id": 10, + "identifier-field-ids": [20], + "fields": [ + {"id": 20, "name": "id", "required": true, "type": "int"}, + { + "id": 30, + "name": "items", + "required": false, + "type": { + "type": "list", + "element-id": 40, + "element-required": false, + "element": { + "type": "map", + "key-id": 50, + "key": "string", + "value-id": 60, + "value": "long", + "value-required": false + } + } + } + ] + })"_json; + + struct IdPath { + nlohmann::json::json_pointer pointer; + std::string_view key; + }; + const std::array id_paths = {{ + {.pointer = nlohmann::json::json_pointer("/schema-id"), .key = "schema-id"}, + {.pointer = nlohmann::json::json_pointer("/identifier-field-ids/0"), + .key = "identifier-field-ids"}, + {.pointer = nlohmann::json::json_pointer("/fields/0/id"), .key = "id"}, + {.pointer = nlohmann::json::json_pointer("/fields/1/type/element-id"), + .key = "element-id"}, + {.pointer = nlohmann::json::json_pointer("/fields/1/type/element/key-id"), + .key = "key-id"}, + {.pointer = nlohmann::json::json_pointer("/fields/1/type/element/value-id"), + .key = "value-id"}, + }}; + const std::array, 3> invalid_ids = {{ + {nlohmann::json(1.5), "must be an integer"}, + {nlohmann::json(true), "must be an integer"}, + {nlohmann::json(2147483648ULL), "out of range"}, + }}; + + for (const auto& id_path : id_paths) { + for (const auto& [invalid_id, expected_error] : invalid_ids) { + SCOPED_TRACE(id_path.pointer.to_string() + "=" + invalid_id.dump()); + auto invalid_schema = valid_schema; + invalid_schema[id_path.pointer] = invalid_id; + + auto result = SchemaFromJson(invalid_schema); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage(id_path.key)); + EXPECT_THAT(result, HasErrorMessage(expected_error)); + } + } +} + TEST(JsonInternalTest, SortField) { auto identity_transform = Transform::Identity(); @@ -322,6 +387,77 @@ TEST(JsonInternalTest, PartitionSpecFromJson) { EXPECT_EQ(*spec, *parsed); } +TEST(JsonInternalTest, PartitionIdsRejectInvalidJsonIntegers) { + const nlohmann::json valid_spec = R"({ + "spec-id": 10, + "fields": [{ + "source-id": 20, + "field-id": 1000, + "transform": "identity", + "name": "id" + }] + })"_json; + auto schema = std::make_shared( + std::vector{SchemaField(20, "id", int32(), false)}, + /*schema_id=*/10); + + struct IdPath { + nlohmann::json::json_pointer pointer; + std::string_view key; + }; + const std::array id_paths = {{ + {.pointer = nlohmann::json::json_pointer("/spec-id"), .key = "spec-id"}, + {.pointer = nlohmann::json::json_pointer("/fields/0/source-id"), + .key = "source-id"}, + {.pointer = nlohmann::json::json_pointer("/fields/0/field-id"), .key = "field-id"}, + }}; + const std::array, 3> invalid_ids = {{ + {nlohmann::json(1.5), "must be an integer"}, + {nlohmann::json(true), "must be an integer"}, + {nlohmann::json(2147483648ULL), "out of range"}, + }}; + + for (const auto& id_path : id_paths) { + for (const auto& [invalid_id, expected_error] : invalid_ids) { + SCOPED_TRACE(id_path.pointer.to_string() + "=" + invalid_id.dump()); + auto invalid_spec = valid_spec; + invalid_spec[id_path.pointer] = invalid_id; + + auto unbound_result = PartitionSpecFromJson(invalid_spec); + EXPECT_THAT(unbound_result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(unbound_result, HasErrorMessage(id_path.key)); + EXPECT_THAT(unbound_result, HasErrorMessage(expected_error)); + + auto bound_result = PartitionSpecFromJson(schema, invalid_spec, 10); + EXPECT_THAT(bound_result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(bound_result, HasErrorMessage(id_path.key)); + EXPECT_THAT(bound_result, HasErrorMessage(expected_error)); + } + } + + auto legacy_field = valid_spec["fields"][0]; + legacy_field.erase("field-id"); + ICEBERG_UNWRAP_OR_FAIL( + auto parsed_legacy_field, + PartitionFieldFromJson(legacy_field, /*allow_field_id_missing=*/true)); + EXPECT_EQ(parsed_legacy_field->field_id(), SchemaField::kInvalidFieldId); + + for (const auto& [invalid_id, expected_error] : invalid_ids) { + SCOPED_TRACE("legacy field-id=" + invalid_id.dump()); + legacy_field["field-id"] = invalid_id; + auto result = PartitionFieldFromJson(legacy_field, /*allow_field_id_missing=*/true); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage("field-id")); + EXPECT_THAT(result, HasErrorMessage(expected_error)); + } + + legacy_field["field-id"] = nullptr; + auto null_result = + PartitionFieldFromJson(legacy_field, /*allow_field_id_missing=*/true); + EXPECT_THAT(null_result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(null_result, HasErrorMessage("field-id")); +} + TEST(JsonInternalTest, SnapshotRefBranch) { SnapshotRef ref(1234567890, SnapshotRef::Branch{.min_snapshots_to_keep = 10, .max_snapshot_age_ms = 123456789, diff --git a/src/iceberg/test/metadata_serde_test.cc b/src/iceberg/test/metadata_serde_test.cc index 10529d1285..88272866a5 100644 --- a/src/iceberg/test/metadata_serde_test.cc +++ b/src/iceberg/test/metadata_serde_test.cc @@ -327,6 +327,24 @@ TEST(MetadataSerdeTest, DeserializeV2ValidMinimal) { ASSERT_FALSE(metadata->Snapshot().has_value()); } +TEST(MetadataSerdeTest, RejectsFractionalSchemaAndDefaultSpecIds) { + ICEBERG_UNWRAP_OR_FAIL( + auto metadata, ReadTableMetadataFromResource("TableMetadataV2ValidMinimal.json")); + ICEBERG_UNWRAP_OR_FAIL(auto valid_json, ToJson(*metadata)); + ASSERT_THAT(TableMetadataFromJson(valid_json), IsOk()); + + for (const auto* key : {"current-schema-id", "default-spec-id"}) { + SCOPED_TRACE(key); + auto invalid_json = valid_json; + invalid_json[key] = 0.5; + + auto result = TableMetadataFromJson(invalid_json); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage(key)); + EXPECT_THAT(result, HasErrorMessage("must be an integer")); + } +} + TEST(MetadataSerdeTest, DeserializeStatisticsFiles) { ICEBERG_UNWRAP_OR_FAIL( auto metadata, ReadTableMetadataFromResource("TableMetadataStatisticsFiles.json")); diff --git a/src/iceberg/util/json_util_internal.h b/src/iceberg/util/json_util_internal.h index 0c30b4493a..b1edf583d9 100644 --- a/src/iceberg/util/json_util_internal.h +++ b/src/iceberg/util/json_util_internal.h @@ -145,6 +145,18 @@ Result GetJsonInteger(const nlohmann::json& json, std::string_view key) return IntegerFromJson(value, std::format("'{}'", key)); } +/// \brief Parses an optional JSON integer field with `IntegerFromJson` strictness, +/// returning `std::nullopt` when the key is missing or null. +template +Result> GetJsonIntegerOptional(const nlohmann::json& json, + std::string_view key) { + if (!json.contains(key) || json.at(key).is_null()) { + return std::nullopt; + } + ICEBERG_ASSIGN_OR_RAISE(auto value, GetJsonInteger(json, key)); + return value; +} + /// \brief Parses an optional `int64_t` field, returning `default_value` when the key is /// missing or null. inline Result GetJsonIntegerOrDefault(const nlohmann::json& json, From c6d5cf43079e9eb1dfff47d8fd72608cdf4104e2 Mon Sep 17 00:00:00 2001 From: Kam Cheung Ting Date: Wed, 30 Sep 2026 10:44:57 +0000 Subject: [PATCH 2/2] test(serde): cover signed schema and partition ID bounds --- src/iceberg/test/json_serde_test.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/iceberg/test/json_serde_test.cc b/src/iceberg/test/json_serde_test.cc index 67656fe753..51bc2af11c 100644 --- a/src/iceberg/test/json_serde_test.cc +++ b/src/iceberg/test/json_serde_test.cc @@ -254,9 +254,11 @@ TEST(JsonInternalTest, SchemaIdsRejectInvalidJsonIntegers) { {.pointer = nlohmann::json::json_pointer("/fields/1/type/element/value-id"), .key = "value-id"}, }}; - const std::array, 3> invalid_ids = {{ + const std::array, 5> invalid_ids = {{ {nlohmann::json(1.5), "must be an integer"}, {nlohmann::json(true), "must be an integer"}, + {nlohmann::json(-2147483649LL), "out of range"}, + {nlohmann::json(2147483648LL), "out of range"}, {nlohmann::json(2147483648ULL), "out of range"}, }}; @@ -411,9 +413,11 @@ TEST(JsonInternalTest, PartitionIdsRejectInvalidJsonIntegers) { .key = "source-id"}, {.pointer = nlohmann::json::json_pointer("/fields/0/field-id"), .key = "field-id"}, }}; - const std::array, 3> invalid_ids = {{ + const std::array, 5> invalid_ids = {{ {nlohmann::json(1.5), "must be an integer"}, {nlohmann::json(true), "must be an integer"}, + {nlohmann::json(-2147483649LL), "out of range"}, + {nlohmann::json(2147483648LL), "out of range"}, {nlohmann::json(2147483648ULL), "out of range"}, }};