Skip to content

Commit 54f4541

Browse files
fix(serde): validate schema and partition IDs as integers
1 parent 01447df commit 54f4541

2 files changed

Lines changed: 159 additions & 15 deletions

File tree

‎src/iceberg/json_serde.cc‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <format>
2424
#include <limits>
2525
#include <map>
26+
#include <optional>
2627
#include <regex>
2728
#include <string>
2829
#include <unordered_set>
@@ -313,6 +314,16 @@ Result<Integer> GetJsonInteger(const nlohmann::json& json, std::string_view key)
313314
return IntegerFromJson<Integer>(value, std::format("'{}'", key));
314315
}
315316

317+
template <std::signed_integral Integer>
318+
Result<std::optional<Integer>> GetJsonIntegerOptional(const nlohmann::json& json,
319+
std::string_view key) {
320+
if (!json.contains(key) || json.at(key).is_null()) {
321+
return std::nullopt;
322+
}
323+
ICEBERG_ASSIGN_OR_RAISE(auto value, GetJsonInteger<Integer>(json, key));
324+
return value;
325+
}
326+
316327
template <std::signed_integral Integer>
317328
Result<std::vector<Integer>> IntegerVectorFromJson(const nlohmann::json& json,
318329
std::string_view key) {
@@ -818,7 +829,7 @@ Result<std::unique_ptr<Type>> StructTypeFromJson(const nlohmann::json& json) {
818829

819830
Result<std::unique_ptr<Type>> ListTypeFromJson(const nlohmann::json& json) {
820831
ICEBERG_ASSIGN_OR_RAISE(auto element_type, TypeFromJson(json[kElement]));
821-
ICEBERG_ASSIGN_OR_RAISE(auto element_id, GetJsonValue<int32_t>(json, kElementId));
832+
ICEBERG_ASSIGN_OR_RAISE(auto element_id, GetJsonInteger<int32_t>(json, kElementId));
822833
ICEBERG_ASSIGN_OR_RAISE(auto element_required,
823834
GetJsonValue<bool>(json, kElementRequired));
824835

@@ -834,8 +845,8 @@ Result<std::unique_ptr<Type>> MapTypeFromJson(const nlohmann::json& json) {
834845
ICEBERG_ASSIGN_OR_RAISE(
835846
auto value_type, GetJsonValue<nlohmann::json>(json, kValue).and_then(TypeFromJson));
836847

837-
ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonValue<int32_t>(json, kKeyId));
838-
ICEBERG_ASSIGN_OR_RAISE(auto value_id, GetJsonValue<int32_t>(json, kValueId));
848+
ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonInteger<int32_t>(json, kKeyId));
849+
ICEBERG_ASSIGN_OR_RAISE(auto value_id, GetJsonInteger<int32_t>(json, kValueId));
839850
ICEBERG_ASSIGN_OR_RAISE(auto value_required, GetJsonValue<bool>(json, kValueRequired));
840851

841852
SchemaField key_field(key_id, std::string(MapType::kKeyName), std::move(key_type),
@@ -996,7 +1007,7 @@ Status ValidateTimestamptzDefaultIsUtc(const Type& type, const nlohmann::json& v
9961007
Result<std::unique_ptr<SchemaField>> FieldFromJson(const nlohmann::json& json) {
9971008
ICEBERG_ASSIGN_OR_RAISE(
9981009
auto type, GetJsonValue<nlohmann::json>(json, kType).and_then(TypeFromJson));
999-
ICEBERG_ASSIGN_OR_RAISE(auto field_id, GetJsonValue<int32_t>(json, kId));
1010+
ICEBERG_ASSIGN_OR_RAISE(auto field_id, GetJsonInteger<int32_t>(json, kId));
10001011
ICEBERG_ASSIGN_OR_RAISE(auto name, GetJsonValue<std::string>(json, kName));
10011012
ICEBERG_ASSIGN_OR_RAISE(auto required, GetJsonValue<bool>(json, kRequired));
10021013
ICEBERG_ASSIGN_OR_RAISE(auto doc, GetJsonValueOrDefault<std::string>(json, kDoc));
@@ -1029,7 +1040,7 @@ Result<std::unique_ptr<SchemaField>> FieldFromJson(const nlohmann::json& json) {
10291040

10301041
Result<std::unique_ptr<Schema>> SchemaFromJson(const nlohmann::json& json) {
10311042
ICEBERG_ASSIGN_OR_RAISE(auto schema_id_opt,
1032-
GetJsonValueOptional<int32_t>(json, kSchemaId));
1043+
GetJsonIntegerOptional<int32_t>(json, kSchemaId));
10331044
ICEBERG_ASSIGN_OR_RAISE(auto type, TypeFromJson(json));
10341045

10351046
if (type->type_id() != TypeId::kStruct) [[unlikely]] {
@@ -1043,9 +1054,11 @@ Result<std::unique_ptr<Schema>> SchemaFromJson(const nlohmann::json& json) {
10431054
fields.emplace_back(std::move(field));
10441055
}
10451056

1046-
ICEBERG_ASSIGN_OR_RAISE(
1047-
auto identifier_field_ids,
1048-
GetJsonValueOrDefault<std::vector<int32_t>>(json, kIdentifierFieldIds));
1057+
std::vector<int32_t> identifier_field_ids;
1058+
if (json.contains(kIdentifierFieldIds) && !json.at(kIdentifierFieldIds).is_null()) {
1059+
ICEBERG_ASSIGN_OR_RAISE(identifier_field_ids,
1060+
IntegerVectorFromJson<int32_t>(json, kIdentifierFieldIds));
1061+
}
10491062

10501063
return Schema::Make(std::move(fields), schema_id_opt.value_or(Schema::kInitialSchemaId),
10511064
std::move(identifier_field_ids));
@@ -1078,14 +1091,14 @@ Result<std::string> ToJsonString(const PartitionSpec& partition_spec) {
10781091

10791092
Result<std::unique_ptr<PartitionField>> PartitionFieldFromJson(
10801093
const nlohmann::json& json, bool allow_field_id_missing) {
1081-
ICEBERG_ASSIGN_OR_RAISE(auto source_id, GetJsonValue<int32_t>(json, kSourceId));
1094+
ICEBERG_ASSIGN_OR_RAISE(auto source_id, GetJsonInteger<int32_t>(json, kSourceId));
10821095
int32_t field_id;
1083-
if (allow_field_id_missing) {
1096+
if (allow_field_id_missing &&
1097+
(!json.contains(kFieldId) || json.at(kFieldId).is_null())) {
10841098
// Partition field id in v1 is not tracked, so we use -1 to indicate that.
1085-
ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonValueOrDefault<int32_t>(
1086-
json, kFieldId, SchemaField::kInvalidFieldId));
1099+
field_id = SchemaField::kInvalidFieldId;
10871100
} else {
1088-
ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonValue<int32_t>(json, kFieldId));
1101+
ICEBERG_ASSIGN_OR_RAISE(field_id, GetJsonInteger<int32_t>(json, kFieldId));
10891102
}
10901103
ICEBERG_ASSIGN_OR_RAISE(
10911104
auto transform,
@@ -1098,7 +1111,7 @@ Result<std::unique_ptr<PartitionField>> PartitionFieldFromJson(
10981111
Result<std::unique_ptr<PartitionSpec>> PartitionSpecFromJson(
10991112
const std::shared_ptr<Schema>& schema, const nlohmann::json& json,
11001113
int32_t default_spec_id) {
1101-
ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonValue<int32_t>(json, kSpecId));
1114+
ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonInteger<int32_t>(json, kSpecId));
11021115
ICEBERG_ASSIGN_OR_RAISE(auto fields, GetJsonValue<nlohmann::json>(json, kFields));
11031116

11041117
std::vector<PartitionField> partition_fields;
@@ -1121,7 +1134,7 @@ Result<std::unique_ptr<PartitionSpec>> PartitionSpecFromJson(
11211134
}
11221135

11231136
Result<std::unique_ptr<PartitionSpec>> PartitionSpecFromJson(const nlohmann::json& json) {
1124-
ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonValue<int32_t>(json, kSpecId));
1137+
ICEBERG_ASSIGN_OR_RAISE(auto spec_id, GetJsonInteger<int32_t>(json, kSpecId));
11251138
ICEBERG_ASSIGN_OR_RAISE(auto fields, GetJsonValue<nlohmann::json>(json, kFields));
11261139

11271140
std::vector<PartitionField> partition_fields;

‎src/iceberg/test/json_serde_test.cc‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
* under the License.
1818
*/
1919

20+
#include <array>
2021
#include <memory>
2122
#include <string>
2223
#include <string_view>
@@ -198,6 +199,70 @@ TEST(JsonInternalTest, SchemaFieldRejectsNonUtcTimestamptzDefault) {
198199
EXPECT_TRUE(FieldFromJson(utc).has_value());
199200
}
200201

202+
TEST(JsonInternalTest, SchemaIdsRejectInvalidJsonIntegers) {
203+
const nlohmann::json valid_schema = R"({
204+
"type": "struct",
205+
"schema-id": 10,
206+
"identifier-field-ids": [20],
207+
"fields": [
208+
{"id": 20, "name": "id", "required": true, "type": "int"},
209+
{
210+
"id": 30,
211+
"name": "items",
212+
"required": false,
213+
"type": {
214+
"type": "list",
215+
"element-id": 40,
216+
"element-required": false,
217+
"element": {
218+
"type": "map",
219+
"key-id": 50,
220+
"key": "string",
221+
"value-id": 60,
222+
"value": "long",
223+
"value-required": false
224+
}
225+
}
226+
}
227+
]
228+
})"_json;
229+
230+
struct IdPath {
231+
nlohmann::json::json_pointer pointer;
232+
std::string_view key;
233+
};
234+
const std::array<IdPath, 6> id_paths = {{
235+
{.pointer = nlohmann::json::json_pointer("/schema-id"), .key = "schema-id"},
236+
{.pointer = nlohmann::json::json_pointer("/identifier-field-ids/0"),
237+
.key = "identifier-field-ids"},
238+
{.pointer = nlohmann::json::json_pointer("/fields/0/id"), .key = "id"},
239+
{.pointer = nlohmann::json::json_pointer("/fields/1/type/element-id"),
240+
.key = "element-id"},
241+
{.pointer = nlohmann::json::json_pointer("/fields/1/type/element/key-id"),
242+
.key = "key-id"},
243+
{.pointer = nlohmann::json::json_pointer("/fields/1/type/element/value-id"),
244+
.key = "value-id"},
245+
}};
246+
const std::array<std::pair<nlohmann::json, std::string_view>, 3> invalid_ids = {{
247+
{nlohmann::json(1.5), "must be an integer"},
248+
{nlohmann::json(true), "must be an integer"},
249+
{nlohmann::json(2147483648ULL), "out of range"},
250+
}};
251+
252+
for (const auto& id_path : id_paths) {
253+
for (const auto& [invalid_id, expected_error] : invalid_ids) {
254+
SCOPED_TRACE(id_path.pointer.to_string() + "=" + invalid_id.dump());
255+
auto invalid_schema = valid_schema;
256+
invalid_schema[id_path.pointer] = invalid_id;
257+
258+
auto result = SchemaFromJson(invalid_schema);
259+
EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError));
260+
EXPECT_THAT(result, HasErrorMessage(id_path.key));
261+
EXPECT_THAT(result, HasErrorMessage(expected_error));
262+
}
263+
}
264+
}
265+
201266
TEST(JsonInternalTest, SortField) {
202267
auto identity_transform = Transform::Identity();
203268

@@ -311,6 +376,72 @@ TEST(JsonInternalTest, PartitionSpecFromJson) {
311376
EXPECT_EQ(*spec, *parsed);
312377
}
313378

379+
TEST(JsonInternalTest, PartitionIdsRejectInvalidJsonIntegers) {
380+
const nlohmann::json valid_spec = R"({
381+
"spec-id": 10,
382+
"fields": [{
383+
"source-id": 20,
384+
"field-id": 1000,
385+
"transform": "identity",
386+
"name": "id"
387+
}]
388+
})"_json;
389+
auto schema = std::make_shared<Schema>(
390+
std::vector<SchemaField>{SchemaField(20, "id", int32(), false)},
391+
/*schema_id=*/10);
392+
393+
struct IdPath {
394+
nlohmann::json::json_pointer pointer;
395+
std::string_view key;
396+
};
397+
const std::array<IdPath, 3> id_paths = {{
398+
{.pointer = nlohmann::json::json_pointer("/spec-id"), .key = "spec-id"},
399+
{.pointer = nlohmann::json::json_pointer("/fields/0/source-id"),
400+
.key = "source-id"},
401+
{.pointer = nlohmann::json::json_pointer("/fields/0/field-id"),
402+
.key = "field-id"},
403+
}};
404+
const std::array<std::pair<nlohmann::json, std::string_view>, 3> invalid_ids = {{
405+
{nlohmann::json(1.5), "must be an integer"},
406+
{nlohmann::json(true), "must be an integer"},
407+
{nlohmann::json(2147483648ULL), "out of range"},
408+
}};
409+
410+
for (const auto& id_path : id_paths) {
411+
for (const auto& [invalid_id, expected_error] : invalid_ids) {
412+
SCOPED_TRACE(id_path.pointer.to_string() + "=" + invalid_id.dump());
413+
auto invalid_spec = valid_spec;
414+
invalid_spec[id_path.pointer] = invalid_id;
415+
416+
auto unbound_result = PartitionSpecFromJson(invalid_spec);
417+
EXPECT_THAT(unbound_result, IsError(ErrorKind::kJsonParseError));
418+
EXPECT_THAT(unbound_result, HasErrorMessage(id_path.key));
419+
EXPECT_THAT(unbound_result, HasErrorMessage(expected_error));
420+
421+
auto bound_result = PartitionSpecFromJson(schema, invalid_spec, 10);
422+
EXPECT_THAT(bound_result, IsError(ErrorKind::kJsonParseError));
423+
EXPECT_THAT(bound_result, HasErrorMessage(id_path.key));
424+
EXPECT_THAT(bound_result, HasErrorMessage(expected_error));
425+
}
426+
}
427+
428+
auto legacy_field = valid_spec["fields"][0];
429+
legacy_field.erase("field-id");
430+
ICEBERG_UNWRAP_OR_FAIL(
431+
auto parsed_legacy_field,
432+
PartitionFieldFromJson(legacy_field, /*allow_field_id_missing=*/true));
433+
EXPECT_EQ(parsed_legacy_field->field_id(), SchemaField::kInvalidFieldId);
434+
435+
for (const auto& [invalid_id, expected_error] : invalid_ids) {
436+
SCOPED_TRACE("legacy field-id=" + invalid_id.dump());
437+
legacy_field["field-id"] = invalid_id;
438+
auto result = PartitionFieldFromJson(legacy_field, /*allow_field_id_missing=*/true);
439+
EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError));
440+
EXPECT_THAT(result, HasErrorMessage("field-id"));
441+
EXPECT_THAT(result, HasErrorMessage(expected_error));
442+
}
443+
}
444+
314445
TEST(JsonInternalTest, SnapshotRefBranch) {
315446
SnapshotRef ref(1234567890, SnapshotRef::Branch{.min_snapshots_to_keep = 10,
316447
.max_snapshot_age_ms = 123456789,

0 commit comments

Comments
 (0)