diff --git a/src/iceberg/data/delete_filter.cc b/src/iceberg/data/delete_filter.cc index 5f21a32de1..1f023ad46b 100644 --- a/src/iceberg/data/delete_filter.cc +++ b/src/iceberg/data/delete_filter.cc @@ -460,50 +460,50 @@ Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, bool need_row_pos_col, - std::shared_ptr counter) { + std::shared_ptr counter, bool validate_batch) { ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(table_schema)); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_metadata, std::shared_ptr requested_schema, std::shared_ptr io, - bool need_row_pos_col, std::shared_ptr counter) { + bool need_row_pos_col, std::shared_ptr counter, bool validate_batch) { ICEBERG_PRECHECK(table_metadata != nullptr, "Table metadata must not be null"); ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(std::move(table_metadata))); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, std::span> schemas, - bool need_row_pos_col, std::shared_ptr counter) { + bool need_row_pos_col, std::shared_ptr counter, bool validate_batch) { ICEBERG_ASSIGN_OR_RAISE(auto field_lookup, MakeFieldLookup(table_schema, schemas)); return Make(std::move(file_path), delete_files, std::move(requested_schema), std::move(io), std::move(field_lookup), need_row_pos_col, - std::move(counter)); + std::move(counter), validate_batch); } Result> DeleteFilter::Make( std::string file_path, std::span> delete_files, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, bool need_row_pos_col, - std::shared_ptr counter) { + std::shared_ptr counter, bool validate_batch) { ICEBERG_PRECHECK(requested_schema != nullptr, "Requested schema must not be null"); ICEBERG_PRECHECK(field_lookup != nullptr, "Field lookup must not be null"); ICEBERG_PRECHECK(delete_files.empty() || io != nullptr, "FileIO must not be null when delete files are present"); - auto filter = std::unique_ptr( - new DeleteFilter(std::move(file_path), std::move(requested_schema), std::move(io), - std::move(field_lookup), need_row_pos_col, std::move(counter))); + auto filter = std::unique_ptr(new DeleteFilter( + std::move(file_path), std::move(requested_schema), std::move(io), + std::move(field_lookup), need_row_pos_col, std::move(counter), validate_batch)); ICEBERG_RETURN_UNEXPECTED(filter->Init(delete_files)); return filter; } @@ -511,13 +511,15 @@ Result> DeleteFilter::Make( DeleteFilter::DeleteFilter(std::string file_path, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, - bool need_row_pos_col, std::shared_ptr counter) + bool need_row_pos_col, std::shared_ptr counter, + bool validate_batch) : file_path_(std::move(file_path)), requested_schema_(std::move(requested_schema)), field_lookup_(std::move(field_lookup)), need_row_pos_col_(need_row_pos_col), + validate_batch_(validate_batch), counter_(std::move(counter)), - delete_loader_(std::move(io)) {} + delete_loader_(std::move(io), validate_batch) {} DeleteFilter::~DeleteFilter() = default; @@ -750,7 +752,9 @@ Result DeleteFilter::ComputeAliveRows(const ArrowSchema& batc batch.length <= static_cast(std::numeric_limits::max()), "Batch length {} exceeds int32_t row index capacity", batch.length); result.indices.reserve(static_cast(batch.length)); - ICEBERG_ASSIGN_OR_RAISE(auto row, ArrowArrayStructLike::Make(batch_schema, batch)); + ICEBERG_ASSIGN_OR_RAISE( + auto row, + ArrowArrayStructLike::Make(batch_schema, batch, /*row_index=*/0, validate_batch_)); for (int64_t i = 0; i < batch.length; ++i) { if (i > 0) { diff --git a/src/iceberg/data/delete_filter.h b/src/iceberg/data/delete_filter.h index 4cb9bace66..527290c150 100644 --- a/src/iceberg/data/delete_filter.h +++ b/src/iceberg/data/delete_filter.h @@ -110,28 +110,30 @@ class ICEBERG_DATA_EXPORT DeleteFilter { std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, bool need_row_pos_col = true, - std::shared_ptr counter = nullptr); + std::shared_ptr counter = nullptr, bool validate_batch = true); /// \brief Create a DeleteFilter using table metadata for schema-aware field lookup. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr table_metadata, std::shared_ptr requested_schema, std::shared_ptr io, - bool need_row_pos_col = true, std::shared_ptr counter = nullptr); + bool need_row_pos_col = true, std::shared_ptr counter = nullptr, + bool validate_batch = true); /// \brief Create a DeleteFilter with table schemas for dropped equality fields. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr table_schema, std::shared_ptr requested_schema, std::shared_ptr io, std::span> schemas, - bool need_row_pos_col = true, std::shared_ptr counter = nullptr); + bool need_row_pos_col = true, std::shared_ptr counter = nullptr, + bool validate_batch = true); /// \brief Create a DeleteFilter with a custom field lookup. static Result> Make( std::string file_path, std::span> delete_files, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, bool need_row_pos_col = true, - std::shared_ptr counter = nullptr); + std::shared_ptr counter = nullptr, bool validate_batch = true); ~DeleteFilter(); @@ -191,7 +193,8 @@ class ICEBERG_DATA_EXPORT DeleteFilter { DeleteFilter(std::string file_path, std::shared_ptr requested_schema, std::shared_ptr io, FieldLookup field_lookup, - bool need_row_pos_col, std::shared_ptr counter); + bool need_row_pos_col, std::shared_ptr counter, + bool validate_batch); Status Init(std::span> delete_files); Result> ComputeRequiredSchema() const; @@ -207,6 +210,7 @@ class ICEBERG_DATA_EXPORT DeleteFilter { FieldLookup field_lookup_; const bool need_row_pos_col_; + const bool validate_batch_; // Position of `_pos` in required_schema_ when existent std::optional pos_field_position_; std::shared_ptr counter_; diff --git a/src/iceberg/data/delete_loader.cc b/src/iceberg/data/delete_loader.cc index fa020c09e2..73242032ce 100644 --- a/src/iceberg/data/delete_loader.cc +++ b/src/iceberg/data/delete_loader.cc @@ -106,7 +106,8 @@ Status ValidateDV(const DataFile& dv, std::string_view data_file_path) { } // namespace -DeleteLoader::DeleteLoader(std::shared_ptr io) : io_(std::move(io)) {} +DeleteLoader::DeleteLoader(std::shared_ptr io, bool validate_batch) + : io_(std::move(io)), validate_batch_(validate_batch) {} DeleteLoader::~DeleteLoader() = default; @@ -255,7 +256,8 @@ Result> DeleteLoader::LoadEqualityDelete internal::ArrowArrayGuard batch_guard(&batch); ICEBERG_ASSIGN_OR_RAISE( - auto row, ArrowArrayStructLike::Make(arrow_schema, batch, /*row_index=*/0)); + auto row, ArrowArrayStructLike::Make(arrow_schema, batch, /*row_index=*/0, + validate_batch_)); for (int64_t i = 0; i < batch.length; ++i) { if (i > 0) { diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h index 360914e855..df3da5736d 100644 --- a/src/iceberg/data/delete_loader.h +++ b/src/iceberg/data/delete_loader.h @@ -37,7 +37,8 @@ class ICEBERG_DATA_EXPORT DeleteLoader { public: /// \brief Create a DeleteLoader. /// \param io FileIO instance for reading delete files - explicit DeleteLoader(std::shared_ptr io); + /// \param validate_batch Whether equality-delete Arrow batches are fully validated + explicit DeleteLoader(std::shared_ptr io, bool validate_batch = true); ~DeleteLoader(); @@ -71,6 +72,7 @@ class ICEBERG_DATA_EXPORT DeleteLoader { std::string_view data_file_path) const; std::shared_ptr io_; + const bool validate_batch_; }; } // namespace iceberg diff --git a/src/iceberg/data/file_scan_task_reader.cc b/src/iceberg/data/file_scan_task_reader.cc index f52a421449..77716510b6 100644 --- a/src/iceberg/data/file_scan_task_reader.cc +++ b/src/iceberg/data/file_scan_task_reader.cc @@ -80,14 +80,15 @@ class TaskStreamSource { std::unique_ptr residual_evaluator, std::shared_ptr<::iceberg::Schema> required_schema, std::shared_ptr<::iceberg::Schema> projected_schema, - ProjectionContext projection_context) + ProjectionContext projection_context, bool validate_batch) : reader_(std::move(reader)), delete_filter_(std::move(delete_filter)), residual_evaluator_(std::move(residual_evaluator)), required_schema_(std::move(required_schema)), projected_schema_(std::move(projected_schema)), project_all_columns_(required_schema_->SameSchema(*projected_schema_)), - projection_context_(std::move(projection_context)) {} + projection_context_(std::move(projection_context)), + validate_batch_(validate_batch) {} ~TaskStreamSource() { if (cached_schema_.has_value() && cached_schema_->release != nullptr) { @@ -185,7 +186,8 @@ class TaskStreamSource { input_batch.length); // DeleteFilter preserves existing field positions when adding delete keys. ICEBERG_ASSIGN_OR_RAISE(auto row, - ArrowArrayStructLike::Make(input_arrow_schema, input_batch)); + ArrowArrayStructLike::Make(input_arrow_schema, input_batch, + /*row_index=*/0, validate_batch_)); std::vector kept; auto keep = [&](int64_t index) -> Status { ICEBERG_RETURN_UNEXPECTED(row->Reset(index)); @@ -216,6 +218,7 @@ class TaskStreamSource { std::shared_ptr<::iceberg::Schema> projected_schema_; bool project_all_columns_ = false; ProjectionContext projection_context_; + const bool validate_batch_; std::optional cached_schema_; }; @@ -296,7 +299,8 @@ class FileScanTaskReader::Impl { ICEBERG_ASSIGN_OR_RAISE( delete_filter, DeleteFilter::Make(data_file->file_path, task.delete_files(), projected_schema_, - io_, field_lookup_, has_position_deletes, delete_counter_)); + io_, field_lookup_, has_position_deletes, delete_counter_, + properties_.Get(ReaderProperties::kArrowValidateBatch))); required_schema = delete_filter->RequiredSchema(); } @@ -322,7 +326,8 @@ class FileScanTaskReader::Impl { auto task_source = std::make_unique( std::move(reader), std::move(delete_filter), std::move(residual_evaluator), - std::move(required_schema), projected_schema_, std::move(projection_context)); + std::move(required_schema), projected_schema_, std::move(projection_context), + properties_.Get(ReaderProperties::kArrowValidateBatch)); return MakeArrowArrayStream(std::move(task_source)); } diff --git a/src/iceberg/file_reader.h b/src/iceberg/file_reader.h index eded9191c6..218e85aed2 100644 --- a/src/iceberg/file_reader.h +++ b/src/iceberg/file_reader.h @@ -80,7 +80,8 @@ class ICEBERG_EXPORT ReaderProperties : public ConfigBase { /// Only the Parquet reader honors this option; other readers ignore it. /// Default: false (use 32-bit offset list). inline static Entry kArrowUseLargeList{"read.arrow.use-large-list", false}; - /// \brief Validate Arrow batches before merge-on-read projection. + /// \brief Fully validate Arrow batches for delete filtering, equality-delete + /// loading, residual evaluation, and projection. /// Default: true. Disable only when the reader is trusted to produce valid arrays. inline static Entry kArrowValidateBatch{"read.arrow.validate-batch", true}; /// \brief Use footer statistics to prune Parquet row groups. diff --git a/src/iceberg/row/arrow_array_wrapper.cc b/src/iceberg/row/arrow_array_wrapper.cc index 5a9cb0894d..a68a22c16f 100644 --- a/src/iceberg/row/arrow_array_wrapper.cc +++ b/src/iceberg/row/arrow_array_wrapper.cc @@ -85,18 +85,22 @@ Result ExtractValue(const ArrowSchema* schema, const ArrowArray* array, return Decimal(int_value); } case NANOARROW_TYPE_STRUCT: { + // The parent view validates nested children recursively; avoid rescanning them. ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr struct_like, - ArrowArrayStructLike::Make(*schema, *array, index)); + ArrowArrayStructLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return struct_like; } case NANOARROW_TYPE_LIST: { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr array_like, - ArrowArrayArrayLike::Make(*schema, *array, index)); + ArrowArrayArrayLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return array_like; } case NANOARROW_TYPE_MAP: { ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr map_like, - ArrowArrayMapLike::Make(*schema, *array, index)); + ArrowArrayMapLike::Make(*schema, *array, index, + /*validate_batch=*/false)); return map_like; } case NANOARROW_TYPE_EXTENSION: @@ -113,8 +117,12 @@ Result ExtractValue(const ArrowSchema* schema, const ArrowArray* array, class ArrowArrayStructLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -147,6 +155,10 @@ class ArrowArrayStructLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return {}; } @@ -161,6 +173,10 @@ class ArrowArrayStructLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return {}; } @@ -171,11 +187,13 @@ class ArrowArrayStructLike::Impl { const ArrowSchema& schema_; std::reference_wrapper array_; int64_t row_index_; + const bool validate_batch_; }; Result> ArrowArrayStructLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayStructLike(std::move(impl))); } @@ -201,8 +219,12 @@ Status ArrowArrayStructLike::Reset(const ArrowArray& array, int64_t row_index) { class ArrowArrayArrayLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -229,6 +251,10 @@ class ArrowArrayArrayLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -243,6 +269,10 @@ class ArrowArrayArrayLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -268,11 +298,13 @@ class ArrowArrayArrayLike::Impl { int64_t offset_ = 0; int64_t length_ = 0; + const bool validate_batch_; }; Result> ArrowArrayArrayLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayArrayLike(std::move(impl))); } @@ -298,8 +330,12 @@ Status ArrowArrayArrayLike::Reset(const ArrowArray& array, int64_t row_index) { class ArrowArrayMapLike::Impl { public: - Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) - : schema_(schema), array_(std::cref(array)), row_index_(row_index) {} + Impl(const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) + : schema_(schema), + array_(std::cref(array)), + row_index_(row_index), + validate_batch_(validate_batch) {} ~Impl() = default; @@ -340,6 +376,10 @@ class ArrowArrayMapLike::Impl { ArrowError error; NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -354,6 +394,10 @@ class ArrowArrayMapLike::Impl { ArrowArrayViewInitFromSchema(&array_view_, &schema_, &error)); NANOARROW_RETURN_IF_NOT_OK( ArrowArrayViewSetArray(&array_view_, &array_.get(), &error)); + if (validate_batch_) { + NANOARROW_RETURN_IF_NOT_OK( + ArrowArrayViewValidate(&array_view_, NANOARROW_VALIDATION_LEVEL_FULL, &error)); + } return UpdateOffsets(); } @@ -387,11 +431,13 @@ class ArrowArrayMapLike::Impl { int64_t offset_ = 0; int64_t length_ = 0; + const bool validate_batch_; }; Result> ArrowArrayMapLike::Make( - const ArrowSchema& schema, const ArrowArray& array, int64_t row_index) { - auto impl = std::make_unique(schema, array, row_index); + const ArrowSchema& schema, const ArrowArray& array, int64_t row_index, + bool validate_batch) { + auto impl = std::make_unique(schema, array, row_index, validate_batch); ICEBERG_RETURN_UNEXPECTED(impl->Init()); return std::unique_ptr(new ArrowArrayMapLike(std::move(impl))); } diff --git a/src/iceberg/row/arrow_array_wrapper.h b/src/iceberg/row/arrow_array_wrapper.h index 7316baddf5..140a98c05b 100644 --- a/src/iceberg/row/arrow_array_wrapper.h +++ b/src/iceberg/row/arrow_array_wrapper.h @@ -41,9 +41,12 @@ class ICEBERG_EXPORT ArrowArrayStructLike : public StructLike { Status Reset(const ArrowArray& array, int64_t row_index = 0); + /// \param validate_batch Fully validate the array and children on creation and + /// Reset to a new array (default true). Disable only for validated or trusted input. static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayStructLike(const ArrowArrayStructLike&) = delete; ArrowArrayStructLike& operator=(const ArrowArrayStructLike&) = delete; @@ -68,9 +71,12 @@ class ICEBERG_EXPORT ArrowArrayArrayLike : public ArrayLike { Status Reset(const ArrowArray& array, int64_t row_index = 0); + /// \param validate_batch Fully validate the array and children on creation and + /// Reset to a new array (default true). Disable only for validated or trusted input. static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayArrayLike(const ArrowArrayArrayLike& other) = delete; ArrowArrayArrayLike& operator=(const ArrowArrayArrayLike& other) = delete; @@ -97,9 +103,12 @@ class ICEBERG_EXPORT ArrowArrayMapLike : public MapLike { Status Reset(const ArrowArray& array, int64_t row_index = 0); + /// \param validate_batch Fully validate the array and children on creation and + /// Reset to a new array (default true). Disable only for validated or trusted input. static Result> Make(const ArrowSchema& schema, const ArrowArray& array, - int64_t row_index = 0); + int64_t row_index = 0, + bool validate_batch = true); ArrowArrayMapLike(const ArrowArrayMapLike& other) = delete; ArrowArrayMapLike& operator=(const ArrowArrayMapLike& other) = delete; diff --git a/src/iceberg/test/delete_filter_test.cc b/src/iceberg/test/delete_filter_test.cc index cfacf1a2c1..18a15aa17b 100644 --- a/src/iceberg/test/delete_filter_test.cc +++ b/src/iceberg/test/delete_filter_test.cc @@ -1362,6 +1362,33 @@ TEST_F(DeleteFilterTest, NullPosInBatchErrors) { EXPECT_THAT(alive, IsError(ErrorKind::kInvalidArrowData)); } +TEST_F(DeleteFilterTest, ConfiguresBatchValidation) { + auto requested_schema = Project({2}); + std::vector> delete_files; + + for (bool validate_batch : {true, false}) { + ICEBERG_UNWRAP_OR_FAIL( + auto filter, + DeleteFilter::Make(std::string(kDataPath), delete_files, table_schema_, + requested_schema, file_io_, true, nullptr, validate_batch)); + ICEBERG_UNWRAP_OR_FAIL( + auto batch, + MakeBatch(*filter->RequiredSchema(), R"([["first"], ["second"], ["third"]])")); + + auto* offsets = const_cast( + reinterpret_cast(batch.array.children[0]->buffers[1])); + offsets[2] = offsets[1] - 1; + + auto alive = filter->ComputeAliveRows(batch.schema, batch.array); + if (validate_batch) { + EXPECT_THAT(alive, IsError(ErrorKind::kInvalidArrowData)); + } else { + ASSERT_THAT(alive, IsOk()); + EXPECT_THAT(alive.value().indices, ::testing::ElementsAre(0, 1, 2)); + } + } +} + TEST_F(DeleteFilterTest, ExpectedSchemaIsRequestedSchema) { auto requested_schema = Project({1}); auto eq_by_name = std::make_shared(DataFile{ diff --git a/src/iceberg/test/file_scan_task_reader_test.cc b/src/iceberg/test/file_scan_task_reader_test.cc index 69e2dbf2cd..02aa3febaf 100644 --- a/src/iceberg/test/file_scan_task_reader_test.cc +++ b/src/iceberg/test/file_scan_task_reader_test.cc @@ -1186,6 +1186,93 @@ TEST_F(FileScanTaskReaderTest, ReadsArrowValidateBatchProperty) { } } +TEST_F(FileScanTaskReaderTest, ConfiguresWrapperValidationThroughOpen) { + ICEBERG_UNWRAP_OR_FAIL(auto data_file, + MakeDataFile(table_schema_, std::string(kResidualDataJson))); + auto delete_schema = + std::make_shared(std::vector{table_schema_->fields()[0]}); + ICEBERG_UNWRAP_OR_FAIL( + auto eq_delete, MakeEqualityDeleteFile(CreateNewTempFilePathWithSuffix(".parquet"), + delete_schema, "[[999]]", {1})); + + class CorruptingReader : public Reader { + public: + explicit CorruptingReader(std::unique_ptr reader) + : reader_(std::move(reader)) {} + + Status Open(const ReaderOptions& options) override { return reader_->Open(options); } + Status Close() override { return reader_->Close(); } + Result Schema() override { return reader_->Schema(); } + Result> Metadata() override { + return reader_->Metadata(); + } + Result> Next() override { + ICEBERG_ASSIGN_OR_RAISE(auto batch, reader_->Next()); + if (batch.has_value() && batch->length == 3 && batch->n_children == 3) { + // Only full validation should inspect the unused column's offsets. + auto* offsets = const_cast( + static_cast(batch->children[1]->buffers[1])); + offsets[2] = offsets[1] - 1; + } + return batch; + } + + private: + std::unique_ptr reader_; + }; + + struct RestoreFactory { + ReaderFactory factory = ReaderFactoryRegistry::GetFactory(FileFormatType::kParquet); + ~RestoreFactory() { + ReaderFactoryRegistry::GetFactory(FileFormatType::kParquet) = std::move(factory); + } + } restore; + ReaderFactoryRegistry::GetFactory(FileFormatType::kParquet) = + [&restore]() -> Result> { + ICEBERG_ASSIGN_OR_RAISE(auto reader, restore.factory()); + return std::make_unique(std::move(reader)); + }; + + for (auto [with_deletes, apply_residuals] : + {std::pair{false, true}, std::pair{true, false}, std::pair{true, true}}) { + SCOPED_TRACE(with_deletes); + SCOPED_TRACE(apply_residuals); + for (bool validate_batch : {true, false}) { + SCOPED_TRACE(validate_batch); + FileScanTask task(data_file, + with_deletes ? std::vector{eq_delete} + : std::vector>{}, + Expressions::GreaterThanOrEqual("id", Literal::Int(1))); + FileScanTaskReader::Options options{ + .io = file_io_, + .table_schema = table_schema_, + .projected_schema = table_schema_, + .apply_residuals = apply_residuals, + }; + if (!validate_batch) { + options.properties[ReaderProperties::kArrowValidateBatch.key()] = "false"; + } + ICEBERG_UNWRAP_OR_FAIL(auto reader, FileScanTaskReader::Make(std::move(options))); + ICEBERG_UNWRAP_OR_FAIL(auto stream, reader->Open(task)); + auto release_stream = [](ArrowArrayStream* stream) { stream->release(stream); }; + std::unique_ptr stream_guard( + &stream, release_stream); + ArrowArray batch{}; + internal::ArrowArrayGuard batch_guard(&batch); + if (validate_batch) { + EXPECT_NE(stream.get_next(&stream, &batch), 0); + EXPECT_THAT(stream.get_last_error(&stream), + ::testing::HasSubstr("Expected element size >= 0")); + } else { + ASSERT_EQ(stream.get_next(&stream, &batch), 0) << stream.get_last_error(&stream); + ASSERT_EQ(batch.length, 3); + const auto* ids = static_cast(batch.children[0]->buffers[1]); + EXPECT_THAT(std::span(ids, 3), ::testing::ElementsAre(1, 2, 3)); + } + } + } +} + TEST_F(FileScanTaskReaderTest, OpenWithEqualityDeletesAddsAndPrunesDeleteOnlyColumns) { ICEBERG_UNWRAP_OR_FAIL( auto data_file, diff --git a/src/iceberg/test/struct_like_test.cc b/src/iceberg/test/struct_like_test.cc index e04d2c089b..0e84a7cc43 100644 --- a/src/iceberg/test/struct_like_test.cc +++ b/src/iceberg/test/struct_like_test.cc @@ -33,6 +33,7 @@ #include #include #include +#include #include "iceberg/arrow_c_data_guard_internal.h" #include "iceberg/expression/literal.h" @@ -618,6 +619,107 @@ TEST(ArrowArrayStructLike, PrimitiveMap) { } } +TEST(ArrowArrayStructLike, ConfigurableFullValidation) { + { + auto type = ::arrow::struct_({::arrow::field("id", ::arrow::int64(), false), + ::arrow::field("name", ::arrow::utf8())}); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[1,"first"], [2,"second"], [3,"third"]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = const_cast( + reinterpret_cast(c_array.children[1]->buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array, 0, false), IsOk()); + } + + { + auto type = ::arrow::list(::arrow::field("item", ::arrow::int32())); + auto array_result = ::arrow::json::ArrayFromJSONString(type, R"([[1, 2], [3], [4]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = + const_cast(reinterpret_cast(c_array.buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayArrayLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayArrayLike::Make(c_schema, c_array, 0, false), IsOk()); + } + + { + auto type = std::make_shared<::arrow::MapType>( + ::arrow::field("key", ::arrow::utf8(), false), + ::arrow::field("value", ::arrow::int32(), false)); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[ ["a", 1], ["b", 2] ], [ ["c", 3] ], []])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = + const_cast(reinterpret_cast(c_array.buffers[1])); + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayMapLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + EXPECT_THAT(ArrowArrayMapLike::Make(c_schema, c_array, 0, false), IsOk()); + } +} + +TEST(ArrowArrayStructLike, NestedValidationPolicy) { + auto type = ::arrow::struct_({::arrow::field( + "nested", ::arrow::struct_({::arrow::field("value", ::arrow::utf8())}))}); + auto array_result = ::arrow::json::ArrayFromJSONString( + type, R"([[{"value":"first"}], [{"value":"second"}]])"); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + auto array = array_result.ValueOrDie(); + + ArrowSchema c_schema; + ArrowArray c_array; + internal::ArrowSchemaGuard schema_guard(&c_schema); + internal::ArrowArrayGuard array_guard(&c_array); + ASSERT_TRUE(::arrow::ExportType(*type, &c_schema).ok()); + ASSERT_TRUE(::arrow::ExportArray(*array, &c_array).ok()); + + auto* offsets = const_cast( + reinterpret_cast(c_array.children[0]->children[0]->buffers[1])); + offsets[1] = offsets[0] + 1; + offsets[2] = offsets[1] - 1; + + EXPECT_THAT(ArrowArrayStructLike::Make(c_schema, c_array), + IsError(ErrorKind::kInvalidArrowData)); + ICEBERG_UNWRAP_OR_FAIL( + auto struct_like, + ArrowArrayStructLike::Make(c_schema, c_array, 0, /*validate_batch=*/false)); + EXPECT_THAT(struct_like->GetField(0), IsOk()); +} + TEST(ArrowArrayStructLike, Accessor) { Schema schema{std::vector{ SchemaField::MakeOptional(1, "c1", int32()),