Skip to content

Commit 66d7ef5

Browse files
authored
fix: respect nullability in nested struct accessors (#967)
1 parent 39c04b9 commit 66d7ef5

5 files changed

Lines changed: 235 additions & 30 deletions

File tree

‎src/iceberg/row/struct_like.cc‎

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -76,39 +76,73 @@ Result<Scalar> LiteralToScalar(const Literal& literal) {
7676
}
7777
}
7878

79+
namespace {
80+
81+
Result<Scalar> GetCheckedField(const StructLike& row, size_t pos, bool optional) {
82+
ICEBERG_ASSIGN_OR_RAISE(auto field, row.GetField(pos));
83+
if (!optional && std::holds_alternative<std::monostate>(field)) {
84+
return InvalidArgument("Required field at position {} is null", pos);
85+
}
86+
return field;
87+
}
88+
89+
Result<std::shared_ptr<StructLike>> GetParentStruct(const StructLike& row, size_t pos,
90+
bool optional) {
91+
ICEBERG_ASSIGN_OR_RAISE(auto field, GetCheckedField(row, pos, optional));
92+
if (std::holds_alternative<std::monostate>(field)) return std::shared_ptr<StructLike>{};
93+
if (!std::holds_alternative<std::shared_ptr<StructLike>>(field)) {
94+
return InvalidSchema("Encountered non-struct at position {}", pos);
95+
}
96+
auto parent = std::get<std::shared_ptr<StructLike>>(std::move(field));
97+
if (!parent && !optional) {
98+
return InvalidArgument("Required field at position {} is null", pos);
99+
}
100+
return parent;
101+
}
102+
103+
} // namespace
104+
79105
StructLikeAccessor::StructLikeAccessor(std::shared_ptr<Type> type,
80-
std::span<const size_t> position_path)
81-
: type_(std::move(type)), position_path_(position_path.begin(), position_path.end()) {
82-
if (position_path.size() == 1) {
83-
accessor_ = [pos =
84-
position_path[0]](const StructLike& struct_like) -> Result<Scalar> {
85-
return struct_like.GetField(pos);
106+
std::span<const size_t> position_path,
107+
std::vector<bool> is_optional)
108+
: type_(std::move(type)),
109+
position_path_(position_path.begin(), position_path.end()),
110+
is_optional_(std::move(is_optional)) {
111+
if (position_path_.size() != is_optional_.size()) {
112+
accessor_ = [](const StructLike&) -> Result<Scalar> {
113+
return InvalidArgument("Optionality count does not match position path");
86114
};
87-
} else if (position_path.size() == 2) {
88-
accessor_ = [pos0 = position_path[0], pos1 = position_path[1]](
115+
return;
116+
}
117+
if (position_path_.size() == 1) {
118+
accessor_ = [pos = position_path_[0], optional = static_cast<bool>(is_optional_[0])](
89119
const StructLike& struct_like) -> Result<Scalar> {
90-
ICEBERG_ASSIGN_OR_RAISE(auto first_level_field, struct_like.GetField(pos0));
91-
if (!std::holds_alternative<std::shared_ptr<StructLike>>(first_level_field)) {
92-
return InvalidSchema("Encountered non-struct in the position path [{},{}]", pos0,
93-
pos1);
94-
}
95-
return std::get<std::shared_ptr<StructLike>>(first_level_field)->GetField(pos1);
120+
return GetCheckedField(struct_like, pos, optional);
121+
};
122+
} else if (position_path_.size() == 2) {
123+
accessor_ = [pos0 = position_path_[0], pos1 = position_path_[1],
124+
optional = static_cast<bool>(is_optional_[0]),
125+
leaf_optional = static_cast<bool>(is_optional_[1])](
126+
const StructLike& struct_like) -> Result<Scalar> {
127+
ICEBERG_ASSIGN_OR_RAISE(auto nested, GetParentStruct(struct_like, pos0, optional));
128+
if (!nested) return Scalar{std::monostate{}};
129+
return GetCheckedField(*nested, pos1, leaf_optional);
96130
};
97-
} else if (!position_path.empty()) {
131+
} else if (!position_path_.empty()) {
98132
accessor_ = [this](const StructLike& struct_like) -> Result<Scalar> {
99133
std::vector<std::shared_ptr<StructLike>> backups;
134+
backups.reserve(position_path_.size() - 1);
100135
const StructLike* current_struct_like = &struct_like;
101136
for (size_t i = 0; i < position_path_.size() - 1; ++i) {
102-
ICEBERG_ASSIGN_OR_RAISE(auto field,
103-
current_struct_like->GetField(position_path_[i]));
104-
if (!std::holds_alternative<std::shared_ptr<StructLike>>(field)) {
105-
return InvalidSchema("Encountered non-struct in the position path [{}]",
106-
position_path_);
107-
}
108-
backups.push_back(std::get<std::shared_ptr<StructLike>>(field));
137+
ICEBERG_ASSIGN_OR_RAISE(
138+
auto parent,
139+
GetParentStruct(*current_struct_like, position_path_[i], is_optional_[i]));
140+
if (!parent) return Scalar{std::monostate{}};
141+
backups.push_back(std::move(parent));
109142
current_struct_like = backups.back().get();
110143
}
111-
return current_struct_like->GetField(position_path_.back());
144+
return GetCheckedField(*current_struct_like, position_path_.back(),
145+
is_optional_.back());
112146
};
113147
} else {
114148
accessor_ = [](const StructLike&) -> Result<Scalar> {

‎src/iceberg/row/struct_like.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include <span>
3232
#include <string_view>
3333
#include <variant>
34+
#include <vector>
3435

3536
#include "iceberg/expression/literal.h"
3637
#include "iceberg/result.h"
@@ -105,9 +106,11 @@ class ICEBERG_EXPORT MapLike {
105106
class ICEBERG_EXPORT StructLikeAccessor {
106107
public:
107108
explicit StructLikeAccessor(std::shared_ptr<Type> type,
108-
std::span<const size_t> position_path);
109+
std::span<const size_t> position_path,
110+
std::vector<bool> is_optional);
109111

110112
/// \brief Get the scalar value at the given position.
113+
/// A null optional parent returns monostate; a null required field is an error.
111114
Result<Scalar> Get(const StructLike& struct_like) const {
112115
return accessor_(struct_like);
113116
}
@@ -121,13 +124,15 @@ class ICEBERG_EXPORT StructLikeAccessor {
121124
/// \brief Get the type of the value that this accessor is bound to.
122125
const Type& type() const { return *type_; }
123126

124-
/// \brief Get the position path of the value that this accessor bounded to.
127+
/// \brief Get the position path of the value that this accessor is bound to.
125128
const std::vector<size_t>& position_path() const { return position_path_; }
126129

127130
private:
128131
std::shared_ptr<Type> type_;
129132
std::function<Result<Scalar>(const StructLike&)> accessor_;
130133
std::vector<size_t> position_path_;
134+
// One entry per position, including the leaf field.
135+
std::vector<bool> is_optional_;
131136
};
132137

133138
} // namespace iceberg

‎src/iceberg/schema.cc‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,18 @@ Result<std::unique_ptr<StructLikeAccessor>> Schema::GetAccessorById(
357357
if (!field.has_value()) {
358358
return NotFound("Cannot get accessor for field id: {}", field_id);
359359
}
360-
return std::make_unique<StructLikeAccessor>(field.value().get().type(), it->second);
360+
std::vector<bool> is_optional;
361+
is_optional.reserve(it->second.size());
362+
const StructType* current = this;
363+
for (size_t i = 0; i < it->second.size(); ++i) {
364+
const auto& field_at_position = current->fields()[it->second[i]];
365+
is_optional.push_back(field_at_position.optional());
366+
if (i + 1 < it->second.size()) {
367+
current = static_cast<const StructType*>(field_at_position.type().get());
368+
}
369+
}
370+
return std::make_unique<StructLikeAccessor>(field.value().get().type(), it->second,
371+
std::move(is_optional));
361372
}
362373
return NotFound("Cannot get accessor for field id: {}", field_id);
363374
}

‎src/iceberg/test/evaluator_test.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ TEST_F(EvaluatorTest, NotEqual) {
330330

331331
TEST_F(EvaluatorTest, StartsWith) {
332332
auto string_schema = std::make_unique<Schema>(
333-
std::vector<SchemaField>{SchemaField::MakeRequired(24, "s", string())});
333+
std::vector<SchemaField>{SchemaField::MakeOptional(24, "s", string())});
334334
ICEBERG_UNWRAP_OR_FAIL(
335335
auto evaluator,
336336
Evaluator::Make(*string_schema, Expressions::StartsWith("s", "abc")));

‎src/iceberg/test/struct_like_test.cc‎

Lines changed: 158 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
#include "iceberg/row/struct_like.h"
2121

2222
#include <array>
23+
#include <memory>
24+
#include <optional>
2325
#include <utility>
26+
#include <vector>
2427

2528
#include <arrow/c/bridge.h>
2629
#include <arrow/json/from_string.h>
@@ -71,6 +74,7 @@ namespace {
7174
class SingleFieldStructLike : public StructLike {
7275
public:
7376
explicit SingleFieldStructLike(Scalar value) : value_(std::move(value)) {}
77+
explicit SingleFieldStructLike(Result<Scalar> value) : value_(std::move(value)) {}
7478

7579
Result<Scalar> GetField(size_t pos) const override {
7680
if (pos != 0) {
@@ -82,9 +86,35 @@ class SingleFieldStructLike : public StructLike {
8286
size_t num_fields() const override { return 1; }
8387

8488
private:
85-
Scalar value_;
89+
Result<Scalar> value_;
8690
};
8791

92+
std::shared_ptr<StructLike> WrapSingleFieldRow(Result<Scalar> field, size_t depth) {
93+
std::shared_ptr<StructLike> row =
94+
std::make_shared<SingleFieldStructLike>(std::move(field));
95+
for (size_t i = 1; i < depth; ++i) {
96+
row = std::make_shared<SingleFieldStructLike>(Scalar{row});
97+
}
98+
return row;
99+
}
100+
101+
std::unique_ptr<Schema> MakeNestedSchema(
102+
size_t depth, std::optional<size_t> optional_parent = std::nullopt,
103+
bool optional_leaf = false) {
104+
auto field =
105+
optional_leaf
106+
? SchemaField::MakeOptional(static_cast<int32_t>(depth), "value", int32())
107+
: SchemaField::MakeRequired(static_cast<int32_t>(depth), "value", int32());
108+
for (size_t level = depth - 1; level > 0; --level) {
109+
auto nested = struct_({field});
110+
field =
111+
optional_parent == level - 1
112+
? SchemaField::MakeOptional(static_cast<int32_t>(level), "nested", nested)
113+
: SchemaField::MakeRequired(static_cast<int32_t>(level), "nested", nested);
114+
}
115+
return std::make_unique<Schema>(std::vector<SchemaField>{std::move(field)});
116+
}
117+
88118
} // namespace
89119

90120
TEST(ManifestFileStructLike, BasicFields) {
@@ -152,7 +182,7 @@ TEST(StructLikeAccessorTest, GetLiteralUuid) {
152182
std::string_view uuid_data(reinterpret_cast<const char*>(bytes.data()), bytes.size());
153183
SingleFieldStructLike row(Scalar{uuid_data});
154184
std::array<size_t, 1> path = {0};
155-
StructLikeAccessor accessor(iceberg::uuid(), path);
185+
StructLikeAccessor accessor(iceberg::uuid(), path, {false});
156186

157187
ICEBERG_UNWRAP_OR_FAIL(auto literal, accessor.GetLiteral(row));
158188
EXPECT_EQ(literal.type()->type_id(), TypeId::kUuid);
@@ -163,13 +193,138 @@ TEST(StructLikeAccessorTest, GetLiteralUuid) {
163193
TEST(StructLikeAccessorTest, GetLiteralUuidRejectsWrongLength) {
164194
SingleFieldStructLike row(Scalar{std::string_view("not-a-uuid")});
165195
std::array<size_t, 1> path = {0};
166-
StructLikeAccessor accessor(iceberg::uuid(), path);
196+
StructLikeAccessor accessor(iceberg::uuid(), path, {false});
167197

168198
auto result = accessor.GetLiteral(row);
169199
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
170200
EXPECT_THAT(result, HasErrorMessage("UUID byte array must be exactly 16 bytes"));
171201
}
172202

203+
TEST(StructLikeAccessorTest, RequiredAndOptionalLeaf) {
204+
SingleFieldStructLike row(Scalar{std::monostate{}});
205+
for (bool optional : {false, true}) {
206+
auto schema = MakeNestedSchema(1, std::nullopt, optional);
207+
ICEBERG_UNWRAP_OR_FAIL(auto accessor, schema->GetAccessorById(1));
208+
if (optional) {
209+
EXPECT_SCALAR_NULL(accessor->Get(row));
210+
ICEBERG_UNWRAP_OR_FAIL(auto literal, accessor->GetLiteral(row));
211+
EXPECT_TRUE(literal.IsNull());
212+
} else {
213+
EXPECT_THAT(accessor->Get(row), IsError(ErrorKind::kInvalidArgument));
214+
EXPECT_THAT(accessor->GetLiteral(row), IsError(ErrorKind::kInvalidArgument));
215+
}
216+
}
217+
}
218+
219+
class NestedStructLikeAccessorTest : public ::testing::TestWithParam<size_t> {};
220+
221+
TEST_P(NestedStructLikeAccessorTest, NullParents) {
222+
const std::vector<size_t> path(GetParam(), 0);
223+
StructLikeAccessor accessor(int32(), path, std::vector<bool>(path.size(), true));
224+
for (const Scalar null :
225+
{Scalar{std::monostate{}}, Scalar{std::shared_ptr<StructLike>{}}}) {
226+
for (size_t parent = 0; parent + 1 < path.size(); ++parent) {
227+
SCOPED_TRACE(parent);
228+
auto row = WrapSingleFieldRow(null, parent + 1);
229+
ICEBERG_UNWRAP_OR_FAIL(auto scalar, accessor.Get(*row));
230+
EXPECT_TRUE(std::holds_alternative<std::monostate>(scalar));
231+
ICEBERG_UNWRAP_OR_FAIL(auto literal, accessor.GetLiteral(*row));
232+
EXPECT_TRUE(literal.IsNull());
233+
EXPECT_EQ(literal.type()->type_id(), TypeId::kInt);
234+
}
235+
}
236+
}
237+
238+
TEST_P(NestedStructLikeAccessorTest, SchemaRequiredAndOptionalParents) {
239+
const auto depth = GetParam();
240+
auto required_schema = MakeNestedSchema(depth);
241+
ICEBERG_UNWRAP_OR_FAIL(auto required_accessor,
242+
required_schema->GetAccessorById(static_cast<int32_t>(depth)));
243+
EXPECT_EQ(required_accessor->position_path(), std::vector<size_t>(depth, 0));
244+
245+
for (const Scalar null :
246+
{Scalar{std::monostate{}}, Scalar{std::shared_ptr<StructLike>{}}}) {
247+
for (size_t parent = 0; parent + 1 < depth; ++parent) {
248+
SCOPED_TRACE(parent);
249+
auto row = WrapSingleFieldRow(null, parent + 1);
250+
auto schema = MakeNestedSchema(depth, parent);
251+
ICEBERG_UNWRAP_OR_FAIL(auto accessor,
252+
schema->GetAccessorById(static_cast<int32_t>(depth)));
253+
EXPECT_SCALAR_NULL(accessor->Get(*row));
254+
ICEBERG_UNWRAP_OR_FAIL(auto literal, accessor->GetLiteral(*row));
255+
EXPECT_TRUE(literal.IsNull());
256+
EXPECT_EQ(literal.type()->type_id(), TypeId::kInt);
257+
258+
EXPECT_THAT(required_accessor->Get(*row), IsError(ErrorKind::kInvalidArgument));
259+
EXPECT_THAT(required_accessor->GetLiteral(*row),
260+
IsError(ErrorKind::kInvalidArgument));
261+
}
262+
}
263+
}
264+
265+
TEST_P(NestedStructLikeAccessorTest, ValueAndNullLeaf) {
266+
const std::vector<size_t> path(GetParam(), 0);
267+
StructLikeAccessor accessor(int32(), path, std::vector<bool>(path.size(), true));
268+
auto row = WrapSingleFieldRow(Scalar{int32_t{42}}, path.size());
269+
ICEBERG_UNWRAP_OR_FAIL(auto value, accessor.GetLiteral(*row));
270+
EXPECT_EQ(value, Literal::Int(42));
271+
272+
auto null_row = WrapSingleFieldRow(Scalar{std::monostate{}}, path.size());
273+
ICEBERG_UNWRAP_OR_FAIL(auto null, accessor.GetLiteral(*null_row));
274+
EXPECT_TRUE(null.IsNull());
275+
EXPECT_EQ(null.type()->type_id(), TypeId::kInt);
276+
}
277+
278+
TEST_P(NestedStructLikeAccessorTest, RequiredAndOptionalLeaf) {
279+
const auto depth = GetParam();
280+
auto row = WrapSingleFieldRow(Scalar{std::monostate{}}, depth);
281+
for (bool optional : {false, true}) {
282+
auto schema = MakeNestedSchema(depth, std::nullopt, optional);
283+
ICEBERG_UNWRAP_OR_FAIL(auto accessor,
284+
schema->GetAccessorById(static_cast<int32_t>(depth)));
285+
if (optional) {
286+
EXPECT_SCALAR_NULL(accessor->Get(*row));
287+
ICEBERG_UNWRAP_OR_FAIL(auto literal, accessor->GetLiteral(*row));
288+
EXPECT_TRUE(literal.IsNull());
289+
} else {
290+
EXPECT_THAT(accessor->Get(*row), IsError(ErrorKind::kInvalidArgument));
291+
EXPECT_THAT(accessor->GetLiteral(*row), IsError(ErrorKind::kInvalidArgument));
292+
}
293+
}
294+
295+
auto schema = MakeNestedSchema(depth, 0);
296+
ICEBERG_UNWRAP_OR_FAIL(auto accessor,
297+
schema->GetAccessorById(static_cast<int32_t>(depth)));
298+
EXPECT_THAT(accessor->Get(*row), IsError(ErrorKind::kInvalidArgument));
299+
auto absent_parent = WrapSingleFieldRow(Scalar{std::monostate{}}, 1);
300+
EXPECT_SCALAR_NULL(accessor->Get(*absent_parent));
301+
}
302+
303+
TEST_P(NestedStructLikeAccessorTest, RejectsNonStructParents) {
304+
const std::vector<size_t> path(GetParam(), 0);
305+
StructLikeAccessor accessor(int32(), path, std::vector<bool>(path.size(), true));
306+
for (size_t parent = 0; parent + 1 < path.size(); ++parent) {
307+
SCOPED_TRACE(parent);
308+
auto row = WrapSingleFieldRow(Scalar{int32_t{42}}, parent + 1);
309+
EXPECT_THAT(accessor.Get(*row), IsError(ErrorKind::kInvalidSchema));
310+
}
311+
}
312+
313+
TEST_P(NestedStructLikeAccessorTest, PreservesFieldErrors) {
314+
const std::vector<size_t> path(GetParam(), 0);
315+
StructLikeAccessor accessor(int32(), path, std::vector<bool>(path.size(), true));
316+
for (size_t level = 0; level < path.size(); ++level) {
317+
SCOPED_TRACE(level);
318+
auto row = WrapSingleFieldRow(IOError("Cannot read field"), level + 1);
319+
auto result = accessor.GetLiteral(*row);
320+
EXPECT_THAT(result, IsError(ErrorKind::kIOError));
321+
EXPECT_THAT(result, HasErrorMessage("Cannot read field"));
322+
}
323+
}
324+
325+
INSTANTIATE_TEST_SUITE_P(NestedPaths, NestedStructLikeAccessorTest,
326+
::testing::Values(size_t{2}, size_t{3}, size_t{4}));
327+
173328
TEST(ManifestFileStructLike, OptionalFields) {
174329
ManifestFile manifest_file{.manifest_path = "/path/to/manifest2.avro",
175330
.manifest_length = 54321,

0 commit comments

Comments
 (0)