diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index f1fb08cf96..77b758bf2c 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -47,3 +47,9 @@ target_link_libraries(demo_example PRIVATE "$,iceberg::iceberg_bundle_shared,iceberg::iceberg_bundle_static>" "$,iceberg::iceberg_rest_shared,iceberg::iceberg_rest_static>" ) + +add_executable(metadata_table_example metadata_table_example.cc) + +target_link_libraries(metadata_table_example + PRIVATE "$,iceberg::iceberg_bundle_shared,iceberg::iceberg_bundle_static>" +) diff --git a/example/metadata_table_example.cc b/example/metadata_table_example.cc new file mode 100644 index 0000000000..18c5bd4d5e --- /dev/null +++ b/example/metadata_table_example.cc @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Reads the snapshots and history metadata tables of an existing Iceberg table: the +// snapshots table as Arrow batches, and the history table as rows without Arrow. + +#include +#include +#include +#include + +#include "iceberg/arrow/arrow_io_util.h" +#include "iceberg/catalog/memory/in_memory_catalog.h" +#include "iceberg/inspect/history_table.h" +#include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_reader.h" +#include "iceberg/inspect/snapshots_table.h" +#include "iceberg/result.h" +#include "iceberg/row/struct_like.h" +#include "iceberg/table.h" +#include "iceberg/table_scan.h" +#include "iceberg/util/macros.h" + +namespace { + +// ICEBERG_ASSIGN_OR_RAISE names Error without its namespace. +using iceberg::Error; + +// Read the snapshots table as an ArrowArrayStream. Any Arrow consumer can take the +// stream; this one counts rows through the Arrow C stream interface. +iceberg::Status PrintSnapshotCount(const iceberg::SnapshotsTable& snapshots) { + ICEBERG_ASSIGN_OR_RAISE( + auto scan, snapshots.NewScan()->Select({"snapshot_id", "operation"}).Build()); + ICEBERG_ASSIGN_OR_RAISE(ArrowArrayStream stream, iceberg::ReadMetadataTable(*scan)); + + int64_t num_rows = 0; + while (true) { + ArrowArray batch{}; + if (stream.get_next(&stream, &batch) != 0) { + const char* message = stream.get_last_error(&stream); + std::string error = message != nullptr ? message : "unknown error"; + stream.release(&stream); + return iceberg::IOError("Failed to read the snapshots table: {}", error); + } + if (batch.release == nullptr) { + break; + } + num_rows += batch.length; + batch.release(&batch); + } + stream.release(&stream); + + std::cout << "Snapshots: " << num_rows << std::endl; + return {}; +} + +// Read the history table row by row, without converting to Arrow. +iceberg::Status PrintHistory(const iceberg::HistoryTable& history) { + ICEBERG_ASSIGN_OR_RAISE( + auto scan, + history.NewScan()->Select({"snapshot_id", "is_current_ancestor"}).Build()); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles()); + + std::cout << "History:" << std::endl; + for (const auto& task : tasks) { + ICEBERG_ASSIGN_OR_RAISE(auto rows, task->Rows()); + while (true) { + ICEBERG_ASSIGN_OR_RAISE(auto row, rows->Next()); + if (!row.has_value()) { + break; + } + // Fields follow the scan's schema, which keeps the table's column order. + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, (*row)->GetField(0)); + ICEBERG_ASSIGN_OR_RAISE(auto is_current_ancestor, (*row)->GetField(1)); + std::cout << " - " << std::get(snapshot_id) + << (std::get(is_current_ancestor) ? "" + : " (not a current ancestor)") + << std::endl; + } + } + return {}; +} + +iceberg::Status Run(const std::string& warehouse_location, const std::string& table_name, + const std::string& metadata_location) { + ICEBERG_ASSIGN_OR_RAISE(auto catalog, iceberg::InMemoryCatalog::Make( + "example", iceberg::arrow::MakeLocalFileIO(), + warehouse_location, /*properties=*/{})); + ICEBERG_ASSIGN_OR_RAISE( + auto table, catalog->RegisterTable({.name = table_name}, metadata_location)); + + ICEBERG_ASSIGN_OR_RAISE(auto snapshots, + iceberg::MetadataTable::Make(table)); + ICEBERG_RETURN_UNEXPECTED(PrintSnapshotCount(*snapshots)); + + ICEBERG_ASSIGN_OR_RAISE(auto history, + iceberg::MetadataTable::Make(table)); + return PrintHistory(*history); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { + std::cerr << "Usage: " << argv[0] + << " " << std::endl; + return 0; + } + + auto status = Run(argv[1], argv[2], argv[3]); + if (!status.has_value()) { + std::cerr << status.error().message << std::endl; + return 1; + } + return 0; +} diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index c283b8af9d..0ac040b150 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -27,6 +27,7 @@ set(ICEBERG_SOURCES catalog/session_catalog.cc catalog/session_context.cc data_table_scan.cc + data_task_reader.cc delete_file_index.cc deletes/dv_util.cc deletes/dv_writer.cc @@ -59,7 +60,10 @@ set(ICEBERG_SOURCES incremental_changelog_scan.cc inspect/history_table.cc inspect/metadata_table.cc + inspect/metadata_table_reader.cc + inspect/metadata_table_scan.cc inspect/snapshots_table.cc + inspect/static_data_task_internal.cc inheritable_metadata.cc json_serde.cc location_provider.cc diff --git a/src/iceberg/cpp20_compatibility_internal.h b/src/iceberg/cpp20_compatibility_internal.h index 359e5ddff8..870769b721 100644 --- a/src/iceberg/cpp20_compatibility_internal.h +++ b/src/iceberg/cpp20_compatibility_internal.h @@ -43,6 +43,7 @@ #include "iceberg/data/position_delete_writer.h" #include "iceberg/data/writer.h" #include "iceberg/data_table_scan.h" +#include "iceberg/data_task_reader.h" #include "iceberg/delete_file_index.h" #include "iceberg/deletes/dv_writer.h" #include "iceberg/deletes/position_delete_index.h" @@ -81,6 +82,8 @@ #include "iceberg/inheritable_metadata.h" #include "iceberg/inspect/history_table.h" #include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_reader.h" +#include "iceberg/inspect/metadata_table_scan.h" #include "iceberg/inspect/snapshots_table.h" #include "iceberg/labels.h" #include "iceberg/location_provider.h" diff --git a/src/iceberg/data_task_reader.cc b/src/iceberg/data_task_reader.cc new file mode 100644 index 0000000000..e3f6e8fadd --- /dev/null +++ b/src/iceberg/data_task_reader.cc @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data_task_reader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/arrow_c_data_util_internal.h" +#include "iceberg/arrow_row_builder_internal.h" +#include "iceberg/nanoarrow_status_internal.h" +#include "iceberg/row/struct_like.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/schema_internal.h" +#include "iceberg/table_scan.h" +#include "iceberg/type.h" +#include "iceberg/util/decimal.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +Status AppendScalar(ArrowArray* array, const SchemaField& field, const Scalar& value); + +Status AppendNullField(ArrowArray* array, const SchemaField& field) { + ICEBERG_PRECHECK(field.optional(), "Required data task field {} is null", field.name()); + return AppendNull(array); +} + +template +bool ScalarMatchesType(TypeId type_id) { + if constexpr (std::is_same_v) { + return type_id == TypeId::kBoolean; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kInt || type_id == TypeId::kDate; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kLong || type_id == TypeId::kTime || + type_id == TypeId::kTimestamp || type_id == TypeId::kTimestampTz || + type_id == TypeId::kTimestampNs || type_id == TypeId::kTimestampTzNs; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kFloat; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kDouble; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kString || type_id == TypeId::kBinary || + type_id == TypeId::kFixed || type_id == TypeId::kUuid || + type_id == TypeId::kGeometry || type_id == TypeId::kGeography; + } else if constexpr (std::is_same_v) { + return type_id == TypeId::kDecimal; + } else if constexpr (std::is_same_v>) { + return type_id == TypeId::kStruct; + } else if constexpr (std::is_same_v>) { + return type_id == TypeId::kList; + } else { + static_assert(std::is_same_v>); + return type_id == TypeId::kMap; + } +} + +Status AppendStruct(ArrowArray* array, const StructType& type, const StructLike& value) { + if (std::cmp_not_equal(value.num_fields(), array->n_children)) { + return InvalidArgument("Cannot append a struct with {} fields to {} Arrow children", + value.num_fields(), array->n_children); + } + for (int64_t i = 0; i < array->n_children; ++i) { + ICEBERG_ASSIGN_OR_RAISE(auto field, value.GetField(static_cast(i))); + ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[i], type.fields()[i], field)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendList(ArrowArray* array, const ListType& type, const ArrayLike& value) { + if (array->n_children != 1) { + return InvalidArrowData("List array must have exactly 1 child"); + } + for (size_t i = 0; i < value.size(); ++i) { + ICEBERG_ASSIGN_OR_RAISE(auto element, value.GetElement(i)); + ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[0], type.element(), element)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendMap(ArrowArray* array, const MapType& type, const MapLike& value) { + // A map array is a list of struct. + if (array->n_children != 1 || array->children[0]->n_children != 2) { + return InvalidArrowData("Map array must have one entries child with 2 children"); + } + ArrowArray* entries = array->children[0]; + for (size_t i = 0; i < value.size(); ++i) { + ICEBERG_ASSIGN_OR_RAISE(auto key, value.GetKey(i)); + ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[0], type.key(), key)); + ICEBERG_ASSIGN_OR_RAISE(auto item, value.GetValue(i)); + ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[1], type.value(), item)); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(entries)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +// Validate the row's scalar alternative before letting nanoarrow encode it. +Status AppendScalar(ArrowArray* array, const SchemaField& field, const Scalar& value) { + return std::visit( + [array, &field](const T& v) -> Status { + if constexpr (std::is_same_v) { + return AppendNullField(array, field); + } else { + if constexpr (std::is_same_v> || + std::is_same_v> || + std::is_same_v>) { + if (v == nullptr) { + return AppendNullField(array, field); + } + } + ICEBERG_PRECHECK(ScalarMatchesType(field.type()->type_id()), + "Data task field {} has a scalar that does not match type {}", + field.name(), field.type()->ToString()); + if constexpr (std::is_same_v) { + return AppendBoolean(array, v); + } else if constexpr (std::is_same_v || std::is_same_v) { + return AppendInt(array, v); + } else if constexpr (std::is_same_v || std::is_same_v) { + return AppendDouble(array, v); + } else if constexpr (std::is_same_v) { + return AppendBytes(array, + std::span( + reinterpret_cast(v.data()), // NOLINT + v.size())); + } else if constexpr (std::is_same_v) { + return AppendDecimal(array, v); + } else if constexpr (std::is_same_v>) { + return AppendStruct(array, static_cast(*field.type()), *v); + } else if constexpr (std::is_same_v>) { + return AppendList(array, static_cast(*field.type()), *v); + } else { + static_assert(std::is_same_v>); + return AppendMap(array, static_cast(*field.type()), *v); + } + } + }, + value); +} + +Status AppendRow(ArrowRowBuilder& builder, const Schema& schema, const StructLike& row) { + if (std::cmp_not_equal(row.num_fields(), builder.num_columns())) { + return InvalidArgument("Data task row has {} fields but the schema has {} columns", + row.num_fields(), builder.num_columns()); + } + for (int64_t i = 0; i < builder.num_columns(); ++i) { + ICEBERG_ASSIGN_OR_RAISE(auto field, row.GetField(static_cast(i))); + ICEBERG_RETURN_UNEXPECTED(AppendScalar(builder.column(i), schema.fields()[i], field)); + } + return builder.FinishRow(); +} + +class DataTaskArrowSource { + public: + static Result> Make( + std::shared_ptr schema, + std::vector> tasks, int64_t batch_size) { + ArrowSchema arrow_schema{}; + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*schema, &arrow_schema)); + return std::unique_ptr(new DataTaskArrowSource( + std::move(schema), std::move(tasks), arrow_schema, batch_size)); + } + + ~DataTaskArrowSource() { std::ignore = Close(); } + + Status Close() { + rows_.reset(); + tasks_.clear(); + if (arrow_schema_.release != nullptr) { + ArrowSchemaRelease(&arrow_schema_); + } + return {}; + } + + Result> Next() { + if (arrow_schema_.release == nullptr) [[unlikely]] { + return InvalidArgument("Cannot read from a closed data task stream"); + } + + // Fetch a row before allocating a builder, so an exhausted stream allocates none. + ICEBERG_ASSIGN_OR_RAISE(auto row, NextRow()); + if (row == nullptr) { + return std::nullopt; + } + + ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_)); + while (true) { + ICEBERG_RETURN_UNEXPECTED(AppendRow(builder, *schema_, *row)); + if (builder.num_rows() == batch_size_) { + break; + } + ICEBERG_ASSIGN_OR_RAISE(row, NextRow()); + if (row == nullptr) { + break; + } + } + + ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish()); + return array; + } + + Result Schema() { + if (arrow_schema_.release == nullptr) [[unlikely]] { + return InvalidArgument("Cannot read schema from a closed data task stream"); + } + ArrowSchema schema_copy{}; + ICEBERG_NANOARROW_RETURN_UNEXPECTED( + ArrowSchemaDeepCopy(&arrow_schema_, &schema_copy)); + return schema_copy; + } + + private: + DataTaskArrowSource(std::shared_ptr schema, + std::vector> tasks, + ArrowSchema arrow_schema, int64_t batch_size) + : schema_(std::move(schema)), + tasks_(std::move(tasks)), + arrow_schema_(arrow_schema), + batch_size_(batch_size) {} + + // Return the next row across all tasks, or nullptr once every task is exhausted. + Result> NextRow() { + while (true) { + if (rows_ == nullptr) { + if (next_task_ == tasks_.size()) { + return nullptr; + } + ICEBERG_ASSIGN_OR_RAISE(rows_, tasks_[next_task_++]->Rows()); + ICEBERG_PRECHECK(rows_ != nullptr, "Data task returned a null row stream"); + } + ICEBERG_ASSIGN_OR_RAISE(auto row, rows_->Next()); + if (!row.has_value()) { + rows_.reset(); + continue; + } + if (*row == nullptr) [[unlikely]] { + return InvalidArgument("Data task returned a null row"); + } + return std::move(*row); + } + } + + std::shared_ptr schema_; + std::vector> tasks_; + size_t next_task_ = 0; + std::unique_ptr rows_; + ArrowSchema arrow_schema_{}; + int64_t batch_size_; +}; + +} // namespace + +Result ReadDataTasks(std::shared_ptr schema, + std::vector> tasks, + int64_t batch_size) { + ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); + ICEBERG_PRECHECK(batch_size > 0, "Batch size must be positive, got {}", batch_size); + for (const auto& task : tasks) { + ICEBERG_PRECHECK(task != nullptr, "Data task cannot be null"); + ICEBERG_PRECHECK(task->schema() != nullptr, "Data task schema cannot be null"); + ICEBERG_PRECHECK(task->schema() == schema || *task->schema() == *schema, + "Data task schema does not match the requested schema"); + } + + ICEBERG_ASSIGN_OR_RAISE( + auto source, + DataTaskArrowSource::Make(std::move(schema), std::move(tasks), batch_size)); + return MakeArrowArrayStream(std::move(source)); +} + +} // namespace iceberg diff --git a/src/iceberg/data_task_reader.h b/src/iceberg/data_task_reader.h new file mode 100644 index 0000000000..8e490cf174 --- /dev/null +++ b/src/iceberg/data_task_reader.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data_task_reader.h +/// \brief Read DataTask rows as Arrow C Data batches. + +#include +#include +#include + +#include "iceberg/arrow_c_data.h" +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief Default maximum number of rows in each batch read from data tasks. +inline constexpr int64_t kDefaultDataTaskBatchSize = 1024; + +/// \brief Read the rows of data tasks as one Arrow stream. +/// +/// Tasks are read in order and a batch may hold rows from more than one task, so a +/// caller that wants to read tasks in parallel passes one task per call. +/// +/// \param schema Schema of the returned stream. Every task must return rows in it. +/// \param tasks Tasks to read. The stream keeps them alive until it is released. +/// \param batch_size Maximum number of rows in each batch. Must be positive. +/// \return A stream that the caller owns and must release by invoking its `release` +/// callback, or an error. +ICEBERG_EXPORT Result ReadDataTasks( + std::shared_ptr schema, std::vector> tasks, + int64_t batch_size = kDefaultDataTaskBatchSize); + +} // namespace iceberg diff --git a/src/iceberg/inspect/history_table.cc b/src/iceberg/inspect/history_table.cc index 03bbcca97f..72ec61ab54 100644 --- a/src/iceberg/inspect/history_table.cc +++ b/src/iceberg/inspect/history_table.cc @@ -19,17 +19,116 @@ #include "iceberg/inspect/history_table.h" +#include +#include +#include #include +#include +#include +#include #include #include +#include "iceberg/constants.h" +#include "iceberg/inspect/static_data_task_internal.h" +#include "iceberg/row/struct_like.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" -#include "iceberg/table.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_metadata.h" #include "iceberg/type.h" #include "iceberg/util/macros.h" +#include "iceberg/util/snapshot_util.h" namespace iceberg { +namespace { + +// A snapshot log entry viewed as a row of the history table. +class HistoryRow : public StructLike { + public: + HistoryRow(int64_t made_current_at_micros, int64_t snapshot_id, + std::optional parent_id, bool is_current_ancestor) + : made_current_at_micros_(made_current_at_micros), + snapshot_id_(snapshot_id), + parent_id_(parent_id), + is_current_ancestor_(is_current_ancestor) {} + + Result GetField(size_t pos) const override { + switch (pos) { + case 0: + return made_current_at_micros_; + case 1: + return snapshot_id_; + case 2: + if (parent_id_.has_value()) { + return *parent_id_; + } + return std::monostate{}; + case 3: + return is_current_ancestor_; + default: + return InvalidArgument("Invalid history table field index: {}", pos); + } + } + + size_t num_fields() const override { return 4; } + + private: + int64_t made_current_at_micros_; + int64_t snapshot_id_; + std::optional parent_id_; + bool is_current_ancestor_; +}; + +class HistoryTableScan : public MetadataTableScan { + public: + HistoryTableScan(std::shared_ptr metadata, + std::shared_ptr schema, std::shared_ptr io, + internal::TableScanContext context) + : MetadataTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context)) {} + + Result>> PlanFiles() const override { + ICEBERG_ASSIGN_OR_RAISE(auto projected_schema, schema()); + + std::unordered_map> parent_ids; + for (const auto& snapshot : metadata_->snapshots) { + if (snapshot != nullptr) { + parent_ids.emplace(snapshot->snapshot_id, snapshot->parent_snapshot_id); + } + } + + // A table without a current snapshot has no current ancestors. + std::unordered_set ancestor_ids; + if (metadata_->current_snapshot_id != kInvalidSnapshotId) { + ICEBERG_ASSIGN_OR_RAISE(auto ancestors, SnapshotUtil::CurrentAncestors(*metadata_)); + for (const auto& ancestor : ancestors) { + ancestor_ids.insert(ancestor->snapshot_id); + } + } + + std::vector> rows; + rows.reserve(metadata_->snapshot_log.size()); + for (const auto& entry : metadata_->snapshot_log) { + std::optional parent_id; + if (auto iter = parent_ids.find(entry.snapshot_id); iter != parent_ids.end()) { + parent_id = iter->second; + } + rows.push_back(std::make_shared( + static_cast(std::chrono::duration_cast( + entry.timestamp_ms.time_since_epoch()) + .count()), + entry.snapshot_id, parent_id, ancestor_ids.contains(entry.snapshot_id))); + } + + ICEBERG_ASSIGN_OR_RAISE( + auto task, + StaticDataTask::Make(*schema_, std::move(projected_schema), std::move(rows))); + return std::vector>{std::move(task)}; + } +}; + +} // namespace HistoryTable::HistoryTable(std::shared_ptr table) : MetadataTable(std::move(table)) {} @@ -50,8 +149,11 @@ Result> HistoryTable::Make(std::shared_ptr
return std::unique_ptr(new HistoryTable(std::move(table))); } -Result HistoryTable::Scan() { - return NotSupported("Scan is not supported for the history table"); +Result> HistoryTable::MakeScan( + std::shared_ptr metadata, std::shared_ptr io, + internal::TableScanContext context) const { + return std::make_unique(std::move(metadata), schema(), std::move(io), + std::move(context)); } } // namespace iceberg diff --git a/src/iceberg/inspect/history_table.h b/src/iceberg/inspect/history_table.h index 7d863fc2c0..f036d81ba6 100644 --- a/src/iceberg/inspect/history_table.h +++ b/src/iceberg/inspect/history_table.h @@ -32,6 +32,12 @@ namespace iceberg { /// \brief History metadata table. +/// +/// Scans return one row per snapshot log entry, in log order. The snapshot log records +/// each change to the table's current snapshot, so a snapshot appears again when a +/// rollback makes it current again. `is_current_ancestor` is true for the current +/// snapshot and its ancestors, and false for snapshots a rollback left behind. A log +/// entry whose snapshot is no longer in the metadata has a null `parent_id`. class ICEBERG_EXPORT HistoryTable : public MetadataTable { public: static Result> Make(std::shared_ptr
table); @@ -42,10 +48,12 @@ class ICEBERG_EXPORT HistoryTable : public MetadataTable { const std::shared_ptr& schema() const override; - Result Scan() override; - private: explicit HistoryTable(std::shared_ptr
table); + + Result> MakeScan( + std::shared_ptr metadata, std::shared_ptr io, + internal::TableScanContext context) const override; }; } // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table.cc b/src/iceberg/inspect/metadata_table.cc index 7bc94c5115..5b4992f603 100644 --- a/src/iceberg/inspect/metadata_table.cc +++ b/src/iceberg/inspect/metadata_table.cc @@ -22,6 +22,8 @@ #include #include +#include "iceberg/table.h" + namespace iceberg { MetadataTable::MetadataTable(std::shared_ptr
source_table) @@ -35,20 +37,9 @@ const std::shared_ptr
& MetadataTable::source_table() const { return source_table_; } -TimeTravelMetadataTable::TimeTravelMetadataTable(std::shared_ptr
source_table) - : MetadataTable(std::move(source_table)) {} - -TimeTravelMetadataTable::~TimeTravelMetadataTable() = default; - -bool TimeTravelMetadataTable::supports_time_travel() const noexcept { return true; } - -Result TimeTravelMetadataTable::Scan() { - return ScanSnapshot(SnapshotSelection{}); -} - -Result TimeTravelMetadataTable::Scan( - const SnapshotSelection& snapshot_selection) { - return ScanSnapshot(snapshot_selection); +std::unique_ptr MetadataTable::NewScan() const { + return std::unique_ptr(new MetadataTableScanBuilder( + *this, source_table_->metadata(), source_table_->io())); } } // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table.h b/src/iceberg/inspect/metadata_table.h index 85000ee1e7..a262fc92e2 100644 --- a/src/iceberg/inspect/metadata_table.h +++ b/src/iceberg/inspect/metadata_table.h @@ -24,19 +24,21 @@ #include #include -#include #include -#include -#include "iceberg/arrow_c_data.h" #include "iceberg/iceberg_export.h" +#include "iceberg/inspect/metadata_table_scan.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" -#include "iceberg/util/timepoint.h" namespace iceberg { /// \brief Base interface for an Iceberg metadata table. +/// +/// A metadata table exposes the source table's metadata as rows, read through a scan: +/// NewScan() configures the columns and, for tables that support time travel, the +/// snapshot; the scan plans DataTasks; and each task returns StructLike rows that a +/// caller consumes directly. ReadMetadataTable() reads a whole scan as Arrow batches. class ICEBERG_EXPORT MetadataTable { public: /// \brief Supported metadata table kinds. @@ -45,9 +47,6 @@ class ICEBERG_EXPORT MetadataTable { kHistory, }; - /// \brief Maximum number of rows emitted in each Arrow batch. - static constexpr int64_t kBatchSize = 1024; - /// \brief Create a metadata table of the requested concrete type. /// /// \tparam MetadataTableType Concrete class derived from MetadataTable. @@ -70,61 +69,31 @@ class ICEBERG_EXPORT MetadataTable { /// \brief Return the source table whose metadata is exposed. const std::shared_ptr
& source_table() const; - /// \brief Return whether this metadata table supports time travel. + /// \brief Return whether scans of this table accept a snapshot selection. virtual bool supports_time_travel() const noexcept; - /// \brief Scan the metadata table without time travel. + /// \brief Create a builder for a scan of this metadata table. /// - /// The caller owns the returned stream and must invoke its `release` callback - /// when the stream is no longer needed. - virtual Result Scan() = 0; + /// The builder captures the source table's current metadata. This metadata table must + /// outlive the builder. + std::unique_ptr NewScan() const; protected: explicit MetadataTable(std::shared_ptr
source_table); private: - std::shared_ptr
source_table_; -}; + friend class MetadataTableScanBuilder; -/// \brief Snapshot selection parameters for a time-travel scan. -struct SnapshotSelection { - /// \brief Select the current snapshot, a snapshot ID, or an as-of timestamp. + /// \brief Create a scan of this table. /// - /// std::monostate selects the current snapshot. - std::variant snapshot; + /// \param metadata Source table metadata captured by NewScan(). + /// \param io FileIO of the source table. + /// \param context Scan options validated by the builder. + virtual Result> MakeScan( + std::shared_ptr metadata, std::shared_ptr io, + internal::TableScanContext context) const = 0; - /// \brief Resolve the snapshot relative to this branch or tag. - /// - /// An empty string uses the main branch. - std::string ref_name; -}; - -/// \brief Base interface for metadata tables that support time travel. -/// -/// Keeping snapshot selection on this capability-specific interface prevents it from -/// being exposed by metadata tables that do not support time travel. -class ICEBERG_EXPORT TimeTravelMetadataTable : public MetadataTable { - public: - ~TimeTravelMetadataTable() override; - - /// \brief Return true because this interface supports time travel. - bool supports_time_travel() const noexcept final; - - /// \brief Scan using the current snapshot on the main branch. - Result Scan() final; - - /// \brief Scan using the requested snapshot selection. - /// - /// \param snapshot_selection Snapshot ID, timestamp, and optional ref selection. - /// \return An Arrow stream containing the metadata table rows, or an error. - Result Scan(const SnapshotSelection& snapshot_selection); - - protected: - explicit TimeTravelMetadataTable(std::shared_ptr
source_table); - - /// \brief Implement a scan for the requested snapshot selection. - virtual Result ScanSnapshot( - const SnapshotSelection& snapshot_selection) = 0; + std::shared_ptr
source_table_; }; } // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table_reader.cc b/src/iceberg/inspect/metadata_table_reader.cc new file mode 100644 index 0000000000..508638e726 --- /dev/null +++ b/src/iceberg/inspect/metadata_table_reader.cc @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/inspect/metadata_table_reader.h" + +#include + +#include "iceberg/inspect/metadata_table_scan.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +Result ReadMetadataTable(const MetadataTableScan& scan, + int64_t batch_size) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, scan.schema()); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan.PlanFiles()); + return ReadDataTasks(std::move(schema), std::move(tasks), batch_size); +} + +} // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table_reader.h b/src/iceberg/inspect/metadata_table_reader.h new file mode 100644 index 0000000000..3448125ef7 --- /dev/null +++ b/src/iceberg/inspect/metadata_table_reader.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/inspect/metadata_table_reader.h +/// \brief Read metadata table scans as Arrow C Data batches. + +#include + +#include "iceberg/arrow_c_data.h" +#include "iceberg/data_task_reader.h" +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief Plan a metadata table scan and read all of its rows as one Arrow stream. +/// +/// The stream has the scan's schema and reads the tasks planned by +/// MetadataTableScan::PlanFiles() in order. To read tasks in parallel, plan them with +/// PlanFiles() and pass one task to each ReadDataTasks() call instead. +/// +/// \param scan The scan to read. +/// \param batch_size Maximum number of rows in each batch. Must be positive. +/// \return A stream that the caller owns and must release by invoking its `release` +/// callback, or an error. +ICEBERG_EXPORT Result ReadMetadataTable( + const MetadataTableScan& scan, int64_t batch_size = kDefaultDataTaskBatchSize); + +} // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table_scan.cc b/src/iceberg/inspect/metadata_table_scan.cc new file mode 100644 index 0000000000..9f3521c030 --- /dev/null +++ b/src/iceberg/inspect/metadata_table_scan.cc @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/inspect/metadata_table_scan.h" + +#include +#include +#include + +#include "iceberg/inspect/metadata_table.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/snapshot_util.h" +#include "iceberg/util/timepoint.h" + +namespace iceberg { + +MetadataTableScan::~MetadataTableScan() = default; + +MetadataTableScanBuilder::MetadataTableScanBuilder( + const MetadataTable& table, std::shared_ptr metadata, + std::shared_ptr io) + : table_(table), metadata_(std::move(metadata)), io_(std::move(io)) {} + +MetadataTableScanBuilder& MetadataTableScanBuilder::Select( + const std::vector& column_names) { + context_.selected_columns = column_names; + return *this; +} + +MetadataTableScanBuilder& MetadataTableScanBuilder::CaseSensitive(bool case_sensitive) { + context_.case_sensitive = case_sensitive; + return *this; +} + +MetadataTableScanBuilder& MetadataTableScanBuilder::UseSnapshot(int64_t snapshot_id) { + ICEBERG_BUILDER_CHECK( + table_.supports_time_travel(), + "Cannot select snapshot {}: metadata table does not support time travel", + snapshot_id); + ICEBERG_BUILDER_CHECK(!context_.snapshot_id.has_value(), + "Cannot override snapshot, already set snapshot id={}", + context_.snapshot_id.value()); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::ignore, metadata_->SnapshotById(snapshot_id)); + context_.snapshot_id = snapshot_id; + return *this; +} + +MetadataTableScanBuilder& MetadataTableScanBuilder::UseRef(const std::string& ref) { + if (ref == SnapshotRef::kMainBranch) { + context_.snapshot_id.reset(); + return *this; + } + ICEBERG_BUILDER_CHECK( + table_.supports_time_travel(), + "Cannot select ref {}: metadata table does not support time travel", ref); + ICEBERG_BUILDER_CHECK(!context_.snapshot_id.has_value(), + "Cannot override ref, already set snapshot id={}", + context_.snapshot_id.value()); + auto iter = metadata_->refs.find(ref); + ICEBERG_BUILDER_CHECK(iter != metadata_->refs.end(), "Cannot find ref {}", ref); + ICEBERG_BUILDER_CHECK(iter->second != nullptr, "Ref {} is null", ref); + const int64_t snapshot_id = iter->second->snapshot_id; + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::ignore, metadata_->SnapshotById(snapshot_id)); + context_.snapshot_id = snapshot_id; + return *this; +} + +MetadataTableScanBuilder& MetadataTableScanBuilder::AsOfTime(int64_t timestamp_millis) { + ICEBERG_BUILDER_CHECK( + table_.supports_time_travel(), + "Cannot select a snapshot as of {}: metadata table does not support time travel", + timestamp_millis); + ICEBERG_BUILDER_ASSIGN_OR_RETURN( + auto snapshot_id, SnapshotUtil::SnapshotIdAsOfTime( + *metadata_, TimePointMsFromUnixMs(timestamp_millis))); + return UseSnapshot(snapshot_id); +} + +Result> MetadataTableScanBuilder::Build() { + ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(context_.Validate()); + return table_.MakeScan(metadata_, io_, std::move(context_)); +} + +} // namespace iceberg diff --git a/src/iceberg/inspect/metadata_table_scan.h b/src/iceberg/inspect/metadata_table_scan.h new file mode 100644 index 0000000000..fe1a0ebb8a --- /dev/null +++ b/src/iceberg/inspect/metadata_table_scan.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/inspect/metadata_table_scan.h +/// \brief Scan APIs for Iceberg metadata tables. + +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/table_scan.h" +#include "iceberg/type_fwd.h" +#include "iceberg/util/error_collector.h" + +namespace iceberg { + +/// \brief A configured scan of a metadata table. +/// +/// The scan holds the source table metadata captured by MetadataTable::NewScan(), so +/// planning and reading see the same metadata even if the source table refreshes. +/// TableScan::schema() returns the projected schema that every planned task uses. +class ICEBERG_EXPORT MetadataTableScan : public TableScan { + public: + ~MetadataTableScan() override; + + /// \brief Plan the tasks that produce this scan's rows. + /// + /// Each task returns rows in schema(). Tasks can be read independently, for example + /// one per manifest for metadata tables backed by manifests. + virtual Result>> PlanFiles() const = 0; + + protected: + using TableScan::TableScan; +}; + +/// \brief Builder for metadata table scans. +/// +/// Create one with MetadataTable::NewScan(). The metadata table must outlive the +/// builder; scans built from it do not reference the metadata table. +class ICEBERG_EXPORT MetadataTableScanBuilder : public ErrorCollector { + public: + /// \brief Read only the given columns. + /// + /// Names resolve as in Schema::Select. If this method is not called, all columns are + /// selected. + /// + /// \param column_names Column names from the metadata table's schema. + MetadataTableScanBuilder& Select(const std::vector& column_names); + + /// \brief Set whether Select() matches column names case-sensitively. Defaults to + /// true. + MetadataTableScanBuilder& CaseSensitive(bool case_sensitive); + + /// \brief Read the metadata of the given snapshot. + /// + /// Only metadata tables that support time travel accept a snapshot. + /// + /// \param snapshot_id A snapshot ID of the source table. + MetadataTableScanBuilder& UseSnapshot(int64_t snapshot_id); + + /// \brief Read the metadata of the snapshot that a branch or tag references. + /// + /// The main branch is always accepted and selects the current snapshot. Other refs + /// require a metadata table that supports time travel. + /// + /// \param ref A branch or tag name of the source table. + MetadataTableScanBuilder& UseRef(const std::string& ref); + + /// \brief Read the metadata of the most recent snapshot as of the given time. + /// + /// Only metadata tables that support time travel accept a timestamp. + /// + /// \param timestamp_millis Milliseconds since the Unix epoch. + MetadataTableScanBuilder& AsOfTime(int64_t timestamp_millis); + + /// \brief Build the configured scan. + Result> Build(); + + private: + friend class MetadataTable; + + MetadataTableScanBuilder(const MetadataTable& table, + std::shared_ptr metadata, + std::shared_ptr io); + + const MetadataTable& table_; + std::shared_ptr metadata_; + std::shared_ptr io_; + internal::TableScanContext context_; +}; + +} // namespace iceberg diff --git a/src/iceberg/inspect/snapshots_table.cc b/src/iceberg/inspect/snapshots_table.cc index 9e9d7906a1..f5b8243045 100644 --- a/src/iceberg/inspect/snapshots_table.cc +++ b/src/iceberg/inspect/snapshots_table.cc @@ -21,21 +21,18 @@ #include #include +#include #include -#include +#include +#include #include #include -#include - -#include "iceberg/arrow_c_data_util_internal.h" -#include "iceberg/arrow_row_builder_internal.h" -#include "iceberg/nanoarrow_status_internal.h" +#include "iceberg/inspect/static_data_task_internal.h" +#include "iceberg/row/struct_like.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" -#include "iceberg/schema_internal.h" #include "iceberg/snapshot.h" -#include "iceberg/table.h" #include "iceberg/table_metadata.h" #include "iceberg/type.h" #include "iceberg/util/macros.h" @@ -43,107 +40,107 @@ namespace iceberg { namespace { -Status AppendSnapshot(ArrowRowBuilder& builder, const Snapshot& snapshot) { - ICEBERG_RETURN_UNEXPECTED( - AppendInt(builder.column(0), std::chrono::duration_cast( - snapshot.timestamp_ms.time_since_epoch()) - .count())); - ICEBERG_RETURN_UNEXPECTED(AppendInt(builder.column(1), snapshot.snapshot_id)); - - if (snapshot.parent_snapshot_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendInt(builder.column(2), *snapshot.parent_snapshot_id)); - } else { - ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(2))); +// A snapshot summary without the operation entry, which has its own column. +class SummaryMapLike : public MapLike { + public: + explicit SummaryMapLike(std::shared_ptr snapshot) + : snapshot_(std::move(snapshot)) { + entries_.reserve(snapshot_->summary.size()); + for (const auto& entry : snapshot_->summary) { + if (entry.first != SnapshotSummaryFields::kOperation) { + entries_.push_back(&entry); + } + } } - auto operation = snapshot.Operation(); - if (operation.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(3), *operation)); - } else { - ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(3))); + Result GetKey(size_t pos) const override { + if (pos >= entries_.size()) { + return InvalidArgument("Invalid map index: {}", pos); + } + return std::string_view(entries_[pos]->first); } - ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(4), snapshot.manifest_list)); - - if (!snapshot.summary.empty()) { - if (snapshot.summary.contains(SnapshotSummaryFields::kOperation)) { - auto summary = snapshot.summary; - summary.erase(SnapshotSummaryFields::kOperation); - ICEBERG_RETURN_UNEXPECTED(AppendStringMap(builder.column(5), summary)); - } else { - ICEBERG_RETURN_UNEXPECTED(AppendStringMap(builder.column(5), snapshot.summary)); + Result GetValue(size_t pos) const override { + if (pos >= entries_.size()) { + return InvalidArgument("Invalid map index: {}", pos); } - } else { - ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(5))); + return std::string_view(entries_[pos]->second); } - return builder.FinishRow(); -} - -class SnapshotsTableStream { - public: - static Result> Make( - std::shared_ptr metadata, const iceberg::Schema& schema) { - ArrowSchema arrow_schema{}; - ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema)); - return std::unique_ptr( - new SnapshotsTableStream(std::move(metadata), std::move(arrow_schema))); - } + size_t size() const override { return entries_.size(); } - ~SnapshotsTableStream() = default; + private: + std::shared_ptr snapshot_; + std::vector*> entries_; +}; - Status Close() { - metadata_.reset(); - if (arrow_schema_.release != nullptr) { - ArrowSchemaRelease(&arrow_schema_); +// A snapshot viewed as a row of the snapshots table. +class SnapshotRow : public StructLike { + public: + explicit SnapshotRow(std::shared_ptr snapshot) + : snapshot_(std::move(snapshot)) {} + + Result GetField(size_t pos) const override { + switch (pos) { + case 0: + return static_cast(std::chrono::duration_cast( + snapshot_->timestamp_ms.time_since_epoch()) + .count()); + case 1: + return snapshot_->snapshot_id; + case 2: + if (snapshot_->parent_snapshot_id.has_value()) { + return *snapshot_->parent_snapshot_id; + } + return std::monostate{}; + case 3: + if (auto operation = snapshot_->Operation(); operation.has_value()) { + return *operation; + } + return std::monostate{}; + case 4: + return std::string_view(snapshot_->manifest_list); + case 5: + // A missing summary is null. A summary that held only the operation is an + // empty map. + if (snapshot_->summary.empty()) { + return std::monostate{}; + } + return std::make_shared(snapshot_); + default: + return InvalidArgument("Invalid snapshots table field index: {}", pos); } - return {}; } - Result> Next() { - if (metadata_ == nullptr) [[unlikely]] { - return InvalidArgument("Cannot read from a closed snapshots table stream"); - } - const auto& snapshots = metadata_->snapshots; - if (next_snapshot_ == snapshots.size()) { - return std::nullopt; - } + size_t num_fields() const override { return 6; } + + private: + std::shared_ptr snapshot_; +}; - ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_)); - while (next_snapshot_ < snapshots.size() && - builder.num_rows() < MetadataTable::kBatchSize) { - const auto& snapshot = snapshots[next_snapshot_++]; +class SnapshotsTableScan : public MetadataTableScan { + public: + SnapshotsTableScan(std::shared_ptr metadata, + std::shared_ptr schema, std::shared_ptr io, + internal::TableScanContext context) + : MetadataTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context)) {} + + Result>> PlanFiles() const override { + ICEBERG_ASSIGN_OR_RAISE(auto projected_schema, schema()); + std::vector> rows; + rows.reserve(metadata_->snapshots.size()); + for (const auto& snapshot : metadata_->snapshots) { if (snapshot == nullptr) [[unlikely]] { continue; } - ICEBERG_RETURN_UNEXPECTED(AppendSnapshot(builder, *snapshot)); - } - if (builder.num_rows() == 0) { - return std::nullopt; + rows.push_back(std::make_shared(snapshot)); } - - ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish()); - return array; + ICEBERG_ASSIGN_OR_RAISE( + auto task, + StaticDataTask::Make(*schema_, std::move(projected_schema), std::move(rows))); + return std::vector>{std::move(task)}; } - - Result Schema() { - if (arrow_schema_.release == nullptr) [[unlikely]] { - return InvalidArgument("Cannot read schema from a closed snapshots table stream"); - } - ArrowSchema schema_copy{}; - ICEBERG_NANOARROW_RETURN_UNEXPECTED( - ArrowSchemaDeepCopy(&arrow_schema_, &schema_copy)); - return schema_copy; - } - - private: - SnapshotsTableStream(std::shared_ptr metadata, - ArrowSchema arrow_schema) - : metadata_(std::move(metadata)), arrow_schema_(std::move(arrow_schema)) {} - - std::shared_ptr metadata_; - ArrowSchema arrow_schema_{}; - size_t next_snapshot_ = 0; }; } // namespace @@ -173,10 +170,11 @@ Result> SnapshotsTable::Make( return std::unique_ptr(new SnapshotsTable(std::move(table))); } -Result SnapshotsTable::Scan() { - ICEBERG_ASSIGN_OR_RAISE( - auto stream, SnapshotsTableStream::Make(source_table()->metadata(), *schema())); - return MakeArrowArrayStream(std::move(stream)); +Result> SnapshotsTable::MakeScan( + std::shared_ptr metadata, std::shared_ptr io, + internal::TableScanContext context) const { + return std::make_unique(std::move(metadata), schema(), + std::move(io), std::move(context)); } } // namespace iceberg diff --git a/src/iceberg/inspect/snapshots_table.h b/src/iceberg/inspect/snapshots_table.h index d2f0ddf909..abb02dfbdd 100644 --- a/src/iceberg/inspect/snapshots_table.h +++ b/src/iceberg/inspect/snapshots_table.h @@ -32,6 +32,9 @@ namespace iceberg { /// \brief Snapshots metadata table. +/// +/// Scans return every known snapshot as one row, so this table does not support time +/// travel. class ICEBERG_EXPORT SnapshotsTable : public MetadataTable { public: static Result> Make(std::shared_ptr
table); @@ -42,13 +45,12 @@ class ICEBERG_EXPORT SnapshotsTable : public MetadataTable { const std::shared_ptr& schema() const override; - /// \brief Scan all snapshots as rows. - /// - /// The snapshots table always returns every known snapshot. - Result Scan() override; - private: explicit SnapshotsTable(std::shared_ptr
table); + + Result> MakeScan( + std::shared_ptr metadata, std::shared_ptr io, + internal::TableScanContext context) const override; }; } // namespace iceberg diff --git a/src/iceberg/inspect/static_data_task_internal.cc b/src/iceberg/inspect/static_data_task_internal.cc new file mode 100644 index 0000000000..c248227f4d --- /dev/null +++ b/src/iceberg/inspect/static_data_task_internal.cc @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/inspect/static_data_task_internal.h" + +#include +#include +#include +#include +#include + +#include "iceberg/row/struct_like.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/type.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +class ProjectedStructLike : public StructLike { + public: + ProjectedStructLike(std::shared_ptr row, + std::shared_ptr> positions) + : row_(std::move(row)), positions_(std::move(positions)) {} + + Result GetField(size_t pos) const override { + if (pos >= positions_->size()) { + return InvalidArgument("Invalid projected field index: {}", pos); + } + return row_->GetField((*positions_)[pos]); + } + + size_t num_fields() const override { return positions_->size(); } + + private: + std::shared_ptr row_; + std::shared_ptr> positions_; +}; + +class StaticRowStream : public StructLikeStream { + public: + StaticRowStream(const std::vector>& rows, + std::shared_ptr> positions) + : rows_(rows), positions_(std::move(positions)) {} + + protected: + Result>> NextImpl() override { + if (next_row_ == rows_.size()) { + return std::nullopt; + } + const auto& row = rows_[next_row_++]; + if (positions_ == nullptr) { + return row; + } + return std::make_shared(row, positions_); + } + + private: + const std::vector>& rows_; + std::shared_ptr> positions_; + size_t next_row_ = 0; +}; + +} // namespace + +StaticDataTask::StaticDataTask(std::shared_ptr projected_schema, + std::shared_ptr> positions, + std::vector> rows) + : projected_schema_(std::move(projected_schema)), + positions_(std::move(positions)), + rows_(std::move(rows)) {} + +StaticDataTask::~StaticDataTask() = default; + +Result> StaticDataTask::Make( + const Schema& table_schema, std::shared_ptr projected_schema, + std::vector> rows) { + ICEBERG_PRECHECK(projected_schema != nullptr, "Projected schema cannot be null"); + + const auto table_fields = table_schema.fields(); + std::vector positions; + positions.reserve(projected_schema->fields().size()); + for (const auto& projected_field : projected_schema->fields()) { + std::optional position; + for (size_t i = 0; i < table_fields.size(); ++i) { + if (table_fields[i].field_id() == projected_field.field_id()) { + position = i; + break; + } + } + ICEBERG_PRECHECK(position.has_value(), "Cannot find projected field {} (id {})", + projected_field.name(), projected_field.field_id()); + if (*table_fields[*position].type() != *projected_field.type()) { + return NotSupported("Cannot project only part of nested field {}", + projected_field.name()); + } + positions.push_back(*position); + } + + bool is_identity = positions.size() == table_fields.size(); + for (size_t i = 0; is_identity && i < positions.size(); ++i) { + is_identity = positions[i] == i; + } + + std::shared_ptr> shared_positions; + if (!is_identity) { + shared_positions = std::make_shared>(std::move(positions)); + } + return std::shared_ptr(new StaticDataTask( + std::move(projected_schema), std::move(shared_positions), std::move(rows))); +} + +Result> StaticDataTask::Rows() const { + return std::make_unique(rows_, positions_); +} + +} // namespace iceberg diff --git a/src/iceberg/inspect/static_data_task_internal.h b/src/iceberg/inspect/static_data_task_internal.h new file mode 100644 index 0000000000..cf48fcfd50 --- /dev/null +++ b/src/iceberg/inspect/static_data_task_internal.h @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/inspect/static_data_task_internal.h +/// \brief A data task over metadata table rows that are already in memory. + +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/table_scan.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief A data task over rows that are already in memory. +/// +/// A metadata table builds every row in its full schema, and Rows() returns them +/// projected to the scan's schema, so each table needs no projection logic of its own. +class ICEBERG_EXPORT StaticDataTask : public DataTask { + public: + /// \brief Create a task over in-memory rows. + /// + /// \param table_schema Schema of `rows`. + /// \param projected_schema Schema of the rows returned by Rows(). It may select and + /// reorder complete top-level fields of `table_schema`; selecting only part of a + /// nested struct is not supported. + /// \param rows Rows whose fields follow `table_schema` by position. + static Result> Make( + const Schema& table_schema, std::shared_ptr projected_schema, + std::vector> rows); + + ~StaticDataTask() override; + + const std::shared_ptr& schema() const override { return projected_schema_; } + + Result> Rows() const override; + + /// \brief Return zero because the rows are already in memory. + int64_t size_bytes() const override { return 0; } + + /// \brief Return zero because reading the rows opens no file. + int32_t files_count() const override { return 0; } + + int64_t estimated_row_count() const override { + return static_cast(rows_.size()); + } + + private: + StaticDataTask(std::shared_ptr projected_schema, + std::shared_ptr> positions, + std::vector> rows); + + std::shared_ptr projected_schema_; + // Position in the table schema of each projected field; null when the projection + // keeps every field in order. + std::shared_ptr> positions_; + std::vector> rows_; +}; + +} // namespace iceberg diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 7afaa79d3b..d6a0d93e9c 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -47,6 +47,7 @@ class ICEBERG_EXPORT ScanTask { enum class Kind : uint8_t { kFileScanTask, kChangelogScanTask, + kDataTask, }; /// \brief The kind of scan task. @@ -139,6 +140,29 @@ using FileScanTaskStream = Stream>; /// \brief Owning pointer to a file scan task stream. using FileScanTaskStreamPtr = std::unique_ptr; +/// \brief Stream of rows viewed through StructLike. +using StructLikeStream = Stream>; + +/// \brief A scan task whose rows come from table metadata instead of a data file. +/// +/// Metadata tables plan data tasks. Rows() exposes the rows without committing to an +/// in-memory format, so a caller can consume them directly or convert them, for example +/// to Arrow with ReadDataTasks() in iceberg/data_task_reader.h. +class ICEBERG_EXPORT DataTask : public ScanTask { + public: + Kind kind() const override { return Kind::kDataTask; } + + /// \brief The schema of rows returned by Rows(). + virtual const std::shared_ptr& schema() const = 0; + + /// \brief Open a stream over this task's rows. + /// + /// Each row's fields follow schema() by position. A row, and every value read from + /// it, stays valid while the row is alive, even after the stream advances. This task + /// must outlive the returned stream. + virtual Result> Rows() const = 0; +}; + enum class ChangelogOperation : uint8_t { kInsert, kDelete, diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index a86bc9087b..2b956a81f4 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -193,6 +193,7 @@ if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(metadata_table_test USE_BUNDLE SOURCES + data_task_test.cc history_table_test.cc metadata_table_test.cc snapshots_table_test.cc) diff --git a/src/iceberg/test/data_task_test.cc b/src/iceberg/test/data_task_test.cc new file mode 100644 index 0000000000..b8ff1f1386 --- /dev/null +++ b/src/iceberg/test/data_task_test.cc @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "iceberg/data_task_reader.h" +#include "iceberg/inspect/static_data_task_internal.h" +#include "iceberg/row/struct_like.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/table_scan.h" +#include "iceberg/test/matchers.h" +#include "iceberg/type.h" +#include "iceberg/util/decimal.h" + +namespace iceberg { +namespace { + +class TestStruct : public StructLike { + public: + explicit TestStruct(std::vector fields) : fields_(std::move(fields)) {} + + Result GetField(size_t pos) const override { + if (pos >= fields_.size()) { + return InvalidArgument("Invalid field index: {}", pos); + } + return fields_[pos]; + } + + size_t num_fields() const override { return fields_.size(); } + + private: + std::vector fields_; +}; + +class TestList : public ArrayLike { + public: + explicit TestList(std::vector elements) : elements_(std::move(elements)) {} + + Result GetElement(size_t pos) const override { return elements_.at(pos); } + + size_t size() const override { return elements_.size(); } + + private: + std::vector elements_; +}; + +class TestMap : public MapLike { + public: + explicit TestMap(std::vector> entries) + : entries_(std::move(entries)) {} + + Result GetKey(size_t pos) const override { return entries_.at(pos).first; } + + Result GetValue(size_t pos) const override { return entries_.at(pos).second; } + + size_t size() const override { return entries_.size(); } + + private: + std::vector> entries_; +}; + +std::shared_ptr MakeRow(std::vector fields) { + return std::make_shared(std::move(fields)); +} + +std::shared_ptr MakeIdSchema() { + return std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32())}); +} + +Result>> ReadAllBatches( + ArrowArrayStream&& stream) { + auto reader = ::arrow::ImportRecordBatchReader(&stream); + if (!reader.ok()) { + if (stream.release != nullptr) { + stream.release(&stream); + } + return InvalidArrowData(reader.status().ToString()); + } + auto batches = (*reader)->ToRecordBatches(); + if (!batches.ok()) { + return InvalidArrowData(batches.status().ToString()); + } + return std::move(batches).MoveValueUnsafe(); +} + +Result>> ReadSingleField( + SchemaField field, Scalar value) { + auto schema = std::make_shared(std::vector{std::move(field)}); + ICEBERG_ASSIGN_OR_RAISE( + auto task, StaticDataTask::Make(*schema, schema, {MakeRow({std::move(value)})})); + ICEBERG_ASSIGN_OR_RAISE(auto stream, ReadDataTasks(schema, {task})); + return ReadAllBatches(std::move(stream)); +} + +class NullRowStreamTask : public DataTask { + public: + explicit NullRowStreamTask(std::shared_ptr schema) + : schema_(std::move(schema)) {} + + const std::shared_ptr& schema() const override { return schema_; } + Result> Rows() const override { return nullptr; } + int64_t size_bytes() const override { return 0; } + int32_t files_count() const override { return 0; } + int64_t estimated_row_count() const override { return 0; } + + private: + std::shared_ptr schema_; +}; + +} // namespace + +TEST(DataTaskReaderTest, ReadsNestedValues) { + auto schema = std::make_shared(std::vector{ + SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeOptional(2, "name", string()), + SchemaField::MakeOptional(3, "point", + std::make_shared(std::vector{ + SchemaField::MakeRequired(4, "x", float64()), + SchemaField::MakeOptional(5, "label", binary())})), + SchemaField::MakeOptional( + 6, "tags", + std::make_shared(SchemaField::MakeRequired(7, "element", int64()))), + SchemaField::MakeOptional( + 8, "props", + std::make_shared(SchemaField::MakeRequired(9, "key", string()), + SchemaField::MakeOptional(10, "value", int32()))), + SchemaField::MakeOptional(11, "hash", fixed(4)), + SchemaField::MakeOptional(12, "price", decimal(9, 2)), + }); + + std::vector> rows{ + MakeRow({int32_t{1}, std::string_view("a"), MakeRow({1.5, std::string_view("xy")}), + std::make_shared(std::vector{int64_t{10}, int64_t{20}}), + std::make_shared(std::vector>{ + {std::string_view("k1"), int32_t{1}}, + {std::string_view("k2"), std::monostate{}}}), + std::string_view("abcd"), Decimal(12345)}), + MakeRow({int32_t{2}, std::monostate{}, std::monostate{}, std::monostate{}, + std::make_shared(std::vector>{}), + std::monostate{}, std::monostate{}}), + }; + ICEBERG_UNWRAP_OR_FAIL(auto task, StaticDataTask::Make(*schema, schema, rows)); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadDataTasks(schema, {task})); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + const auto& batch = batches.front(); + ASSERT_EQ(batch->num_rows(), 2); + + auto ids = std::static_pointer_cast<::arrow::Int32Array>(batch->column(0)); + EXPECT_EQ(ids->Value(0), 1); + EXPECT_EQ(ids->Value(1), 2); + + auto names = std::static_pointer_cast<::arrow::StringArray>(batch->column(1)); + EXPECT_EQ(names->GetString(0), "a"); + EXPECT_TRUE(names->IsNull(1)); + + auto points = std::static_pointer_cast<::arrow::StructArray>(batch->column(2)); + EXPECT_FALSE(points->IsNull(0)); + EXPECT_TRUE(points->IsNull(1)); + EXPECT_EQ(std::static_pointer_cast<::arrow::DoubleArray>(points->field(0))->Value(0), + 1.5); + EXPECT_EQ( + std::static_pointer_cast<::arrow::BinaryArray>(points->field(1))->GetString(0), + "xy"); + + auto tags = std::static_pointer_cast<::arrow::ListArray>(batch->column(3)); + EXPECT_TRUE(tags->IsNull(1)); + ASSERT_EQ(tags->value_length(0), 2); + auto tag_values = std::static_pointer_cast<::arrow::Int64Array>(tags->values()); + EXPECT_EQ(tag_values->Value(0), 10); + EXPECT_EQ(tag_values->Value(1), 20); + + auto props = std::static_pointer_cast<::arrow::MapArray>(batch->column(4)); + ASSERT_EQ(props->value_length(0), 2); + EXPECT_FALSE(props->IsNull(1)); + EXPECT_EQ(props->value_length(1), 0); + auto prop_keys = std::static_pointer_cast<::arrow::StringArray>(props->keys()); + auto prop_values = std::static_pointer_cast<::arrow::Int32Array>(props->items()); + EXPECT_EQ(prop_keys->GetString(0), "k1"); + EXPECT_EQ(prop_values->Value(0), 1); + EXPECT_EQ(prop_keys->GetString(1), "k2"); + EXPECT_TRUE(prop_values->IsNull(1)); + + auto hashes = std::static_pointer_cast<::arrow::FixedSizeBinaryArray>(batch->column(5)); + EXPECT_EQ(hashes->GetString(0), "abcd"); + EXPECT_TRUE(hashes->IsNull(1)); + + auto prices = std::static_pointer_cast<::arrow::Decimal128Array>(batch->column(6)); + EXPECT_EQ(prices->FormatValue(0), "123.45"); + EXPECT_TRUE(prices->IsNull(1)); +} + +TEST(DataTaskReaderTest, BatchesSpanTasks) { + auto schema = MakeIdSchema(); + ICEBERG_UNWRAP_OR_FAIL(auto first, + StaticDataTask::Make(*schema, schema, {MakeRow({int32_t{1}})})); + ICEBERG_UNWRAP_OR_FAIL( + auto second, StaticDataTask::Make(*schema, schema, + {MakeRow({int32_t{2}}), MakeRow({int32_t{3}})})); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, + ReadDataTasks(schema, {first, second}, /*batch_size=*/2)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 2); + ASSERT_EQ(batches[0]->num_rows(), 2); + ASSERT_EQ(batches[1]->num_rows(), 1); + auto first_ids = std::static_pointer_cast<::arrow::Int32Array>(batches[0]->column(0)); + EXPECT_EQ(first_ids->Value(0), 1); + EXPECT_EQ(first_ids->Value(1), 2); + auto second_ids = std::static_pointer_cast<::arrow::Int32Array>(batches[1]->column(0)); + EXPECT_EQ(second_ids->Value(0), 3); +} + +TEST(DataTaskReaderTest, NoTasksReturnsNoBatches) { + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadDataTasks(MakeIdSchema(), {})); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + EXPECT_TRUE(batches.empty()); +} + +TEST(DataTaskReaderTest, RejectsInvalidArguments) { + auto schema = MakeIdSchema(); + ICEBERG_UNWRAP_OR_FAIL(auto task, StaticDataTask::Make(*schema, schema, {})); + + EXPECT_THAT(ReadDataTasks(schema, {task}, /*batch_size=*/0), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(ReadDataTasks(schema, {nullptr}), IsError(ErrorKind::kInvalidArgument)); + + auto other_schema = std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int64())}); + EXPECT_THAT(ReadDataTasks(other_schema, {task}), + HasErrorMessage("Data task schema does not match")); +} + +TEST(DataTaskReaderTest, ReportsRowsThatDoNotMatchSchema) { + auto schema = MakeIdSchema(); + ICEBERG_UNWRAP_OR_FAIL( + auto task, + StaticDataTask::Make(*schema, schema, {MakeRow({int32_t{1}, int32_t{2}})})); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadDataTasks(schema, {task})); + EXPECT_THAT(ReadAllBatches(std::move(stream)), + HasErrorMessage("Data task row has 2 fields but the schema has 1 columns")); +} + +TEST(DataTaskReaderTest, RejectsNullRowStream) { + auto schema = MakeIdSchema(); + auto task = std::make_shared(schema); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadDataTasks(schema, {task})); + EXPECT_THAT(ReadAllBatches(std::move(stream)), + HasErrorMessage("Data task returned a null row stream")); +} + +TEST(DataTaskReaderTest, RejectsNullRequiredFields) { + EXPECT_THAT( + ReadSingleField(SchemaField::MakeRequired(1, "id", int32()), std::monostate{}), + HasErrorMessage("Required data task field id is null")); + + std::vector, Scalar>> cases{ + {std::make_shared(std::vector{}), + std::shared_ptr{}}, + {std::make_shared(SchemaField::MakeRequired(2, "element", int32())), + std::shared_ptr{}}, + {std::make_shared(SchemaField::MakeRequired(2, "key", string()), + SchemaField::MakeOptional(3, "value", int32())), + std::shared_ptr{}}}; + for (const auto& [type, value] : cases) { + SCOPED_TRACE(type->ToString()); + EXPECT_THAT(ReadSingleField(SchemaField::MakeRequired(1, "field", type), value), + HasErrorMessage("Required data task field field is null")); + ICEBERG_UNWRAP_OR_FAIL( + auto batches, + ReadSingleField(SchemaField::MakeOptional(1, "field", type), value)); + ASSERT_EQ(batches.size(), 1); + EXPECT_TRUE(batches[0]->column(0)->IsNull(0)); + EXPECT_TRUE(batches[0]->ValidateFull().ok()); + } +} + +TEST(DataTaskReaderTest, RejectsNullRequiredNestedFields) { + auto struct_type = std::make_shared( + std::vector{SchemaField::MakeRequired(2, "child", int32())}); + auto list_type = + std::make_shared(SchemaField::MakeRequired(2, "element", int32())); + auto map_type = + std::make_shared(SchemaField::MakeRequired(2, "key", string()), + SchemaField::MakeRequired(3, "value", int32())); + + EXPECT_THAT(ReadSingleField(SchemaField::MakeOptional(1, "field", struct_type), + MakeRow({std::monostate{}})), + HasErrorMessage("Required data task field child is null")); + EXPECT_THAT( + ReadSingleField(SchemaField::MakeOptional(1, "field", list_type), + std::make_shared(std::vector{std::monostate{}})), + HasErrorMessage("Required data task field element is null")); + EXPECT_THAT( + ReadSingleField(SchemaField::MakeOptional(1, "field", map_type), + std::make_shared(std::vector>{ + {std::monostate{}, int32_t{1}}})), + HasErrorMessage("Required data task field key is null")); + EXPECT_THAT( + ReadSingleField(SchemaField::MakeOptional(1, "field", map_type), + std::make_shared(std::vector>{ + {std::string_view("k"), std::monostate{}}})), + HasErrorMessage("Required data task field value is null")); +} + +TEST(DataTaskReaderTest, RejectsMismatchedScalarAlternatives) { + std::vector, Scalar>> cases{ + {int32(), true}, + {boolean(), int32_t{1}}, + {int32(), int64_t{1}}, + {int64(), int32_t{1}}, + {float32(), 1.0}, + {float64(), float{1.0}}, + {string(), int32_t{1}}, + {decimal(9, 2), int32_t{1}}, + {int32(), Decimal(1)}, + {std::make_shared(std::vector{}), + std::make_shared(std::vector{})}, + {std::make_shared(SchemaField::MakeRequired(2, "element", int32())), + MakeRow({})}, + {std::make_shared(SchemaField::MakeRequired(2, "key", string()), + SchemaField::MakeOptional(3, "value", int32())), + std::make_shared(std::vector{})}}; + for (const auto& [type, value] : cases) { + SCOPED_TRACE(type->ToString()); + EXPECT_THAT(ReadSingleField(SchemaField::MakeOptional(1, "field", type), value), + HasErrorMessage("does not match type")); + } +} + +TEST(DataTaskReaderTest, RejectsMismatchedNestedScalarAlternatives) { + std::vector, Scalar>> cases{ + {std::make_shared( + std::vector{SchemaField::MakeRequired(2, "child", int32())}), + MakeRow({true})}, + {std::make_shared(SchemaField::MakeRequired(2, "element", int32())), + std::make_shared(std::vector{true})}, + {std::make_shared(SchemaField::MakeRequired(2, "key", int32()), + SchemaField::MakeOptional(3, "value", int32())), + std::make_shared( + std::vector>{{true, int32_t{1}}})}, + {std::make_shared(SchemaField::MakeRequired(2, "key", string()), + SchemaField::MakeOptional(3, "value", int32())), + std::make_shared( + std::vector>{{std::string_view("k"), true}})}}; + for (const auto& [type, value] : cases) { + SCOPED_TRACE(type->ToString()); + EXPECT_THAT(ReadSingleField(SchemaField::MakeRequired(1, "field", type), value), + HasErrorMessage("does not match type")); + } +} + +TEST(StaticDataTaskTest, ProjectsTopLevelFields) { + auto table_schema = std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeOptional(2, "name", string()), + SchemaField::MakeOptional(3, "flag", boolean())}); + auto projected_schema = std::make_shared( + std::vector{SchemaField::MakeOptional(3, "flag", boolean()), + SchemaField::MakeRequired(1, "id", int32())}); + ICEBERG_UNWRAP_OR_FAIL( + auto task, + StaticDataTask::Make(*table_schema, projected_schema, + {MakeRow({int32_t{7}, std::string_view("a"), true})})); + EXPECT_EQ(task->kind(), ScanTask::Kind::kDataTask); + EXPECT_EQ(task->schema(), projected_schema); + EXPECT_EQ(task->estimated_row_count(), 1); + + ICEBERG_UNWRAP_OR_FAIL(auto rows, task->Rows()); + ICEBERG_UNWRAP_OR_FAIL(auto all_rows, rows->ToVector()); + ASSERT_EQ(all_rows.size(), 1); + ASSERT_EQ(all_rows[0]->num_fields(), 2); + ICEBERG_UNWRAP_OR_FAIL(auto flag, all_rows[0]->GetField(0)); + EXPECT_EQ(std::get(flag), true); + ICEBERG_UNWRAP_OR_FAIL(auto id, all_rows[0]->GetField(1)); + EXPECT_EQ(std::get(id), 7); +} + +TEST(StaticDataTaskTest, RejectsPartialNestedProjection) { + auto point_type = std::make_shared( + std::vector{SchemaField::MakeRequired(2, "x", float64()), + SchemaField::MakeRequired(3, "y", float64())}); + Schema table_schema( + std::vector{SchemaField::MakeOptional(1, "point", point_type)}); + auto projected_schema = std::make_shared(std::vector{ + SchemaField::MakeOptional(1, "point", + std::make_shared(std::vector{ + SchemaField::MakeRequired(2, "x", float64())}))}); + + EXPECT_THAT(StaticDataTask::Make(table_schema, projected_schema, {}), + IsError(ErrorKind::kNotSupported)); +} + +TEST(StaticDataTaskTest, RejectsUnknownProjectedField) { + auto schema = MakeIdSchema(); + auto projected_schema = std::make_shared( + std::vector{SchemaField::MakeRequired(99, "missing", int32())}); + + EXPECT_THAT(StaticDataTask::Make(*schema, projected_schema, {}), + HasErrorMessage("Cannot find projected field missing (id 99)")); +} + +} // namespace iceberg diff --git a/src/iceberg/test/history_table_test.cc b/src/iceberg/test/history_table_test.cc index 8da311c024..61fa09810b 100644 --- a/src/iceberg/test/history_table_test.cc +++ b/src/iceberg/test/history_table_test.cc @@ -22,15 +22,28 @@ #include "iceberg/inspect/history_table.h" +#include +#include +#include +#include + +#include +#include #include #include #include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_reader.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" #include "iceberg/test/metadata_table_test_base.h" #include "iceberg/type.h" +#include "iceberg/util/timepoint.h" namespace iceberg { namespace { @@ -45,11 +58,125 @@ std::shared_ptr MakeHistorySchema() { } // namespace -class HistoryTableTest : public MetadataTableTestBase {}; +class HistoryTableTest : public MetadataTableTestBase { + protected: + Result> MakeTableWithHistory( + std::vector> snapshots, int64_t current_snapshot_id, + std::vector snapshot_log) { + auto metadata = std::make_shared(*metadata_); + metadata->snapshots = std::move(snapshots); + metadata->current_snapshot_id = current_snapshot_id; + metadata->snapshot_log = std::move(snapshot_log); + TableIdentifier ident{.ns = Namespace{.levels = {"db"}}, .name = "history_table"}; + return Table::Make(ident, metadata, "s3://bucket/meta.json", io_, catalog_); + } + + static SnapshotLogEntry LogEntry(int64_t timestamp_ms, int64_t snapshot_id) { + return {.timestamp_ms = TimePointMsFromUnixMs(timestamp_ms), + .snapshot_id = snapshot_id}; + } +}; TEST_F(HistoryTableTest, SchemaMatchesIcebergSchema) { ICEBERG_UNWRAP_OR_FAIL(auto history_table, MetadataTable::Make(table_)); EXPECT_TRUE(*history_table->schema() == *MakeHistorySchema()); } +TEST_F(HistoryTableTest, ScanAfterRollback) { + auto [snap1, snap2] = MakeTestSnapshots(); + auto snap3 = std::make_shared(*snap2); + snap3->snapshot_id = 3; + ICEBERG_UNWRAP_OR_FAIL( + auto table, MakeTableWithHistory({snap1, snap2, snap3}, /*current_snapshot_id=*/3, + {LogEntry(1000, 1), LogEntry(2000, 2), + LogEntry(3000, 1), LogEntry(4000, 3)})); + ICEBERG_UNWRAP_OR_FAIL(auto history_table, MetadataTable::Make(table)); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*history_table)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + const auto& batch = batches.front(); + ASSERT_EQ(batch->num_rows(), 4); + ASSERT_EQ(batch->num_columns(), 4); + + auto made_current_at = + std::static_pointer_cast<::arrow::TimestampArray>(batch->column(0)); + EXPECT_EQ(made_current_at->Value(0), 1000 * 1000); + EXPECT_EQ(made_current_at->Value(1), 2000 * 1000); + EXPECT_EQ(made_current_at->Value(2), 3000 * 1000); + EXPECT_EQ(made_current_at->Value(3), 4000 * 1000); + + auto snapshot_ids = std::static_pointer_cast<::arrow::Int64Array>(batch->column(1)); + EXPECT_EQ(snapshot_ids->Value(0), 1); + EXPECT_EQ(snapshot_ids->Value(1), 2); + EXPECT_EQ(snapshot_ids->Value(2), 1); + EXPECT_EQ(snapshot_ids->Value(3), 3); + + auto parent_ids = std::static_pointer_cast<::arrow::Int64Array>(batch->column(2)); + EXPECT_TRUE(parent_ids->IsNull(0)); + EXPECT_EQ(parent_ids->Value(1), 1); + EXPECT_TRUE(parent_ids->IsNull(2)); + EXPECT_EQ(parent_ids->Value(3), 1); + + auto is_current_ancestor = + std::static_pointer_cast<::arrow::BooleanArray>(batch->column(3)); + EXPECT_TRUE(is_current_ancestor->Value(0)); + EXPECT_FALSE(is_current_ancestor->Value(1)); + EXPECT_TRUE(is_current_ancestor->Value(2)); + EXPECT_TRUE(is_current_ancestor->Value(3)); +} + +TEST_F(HistoryTableTest, ScanEntryForMissingSnapshot) { + auto snap1 = MakeTestSnapshots().first; + ICEBERG_UNWRAP_OR_FAIL(auto table, + MakeTableWithHistory({snap1}, /*current_snapshot_id=*/1, + {LogEntry(500, 99), LogEntry(1000, 1)})); + ICEBERG_UNWRAP_OR_FAIL(auto history_table, MetadataTable::Make(table)); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*history_table)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + const auto& batch = batches.front(); + ASSERT_EQ(batch->num_rows(), 2); + + auto snapshot_ids = std::static_pointer_cast<::arrow::Int64Array>(batch->column(1)); + EXPECT_EQ(snapshot_ids->Value(0), 99); + auto parent_ids = std::static_pointer_cast<::arrow::Int64Array>(batch->column(2)); + EXPECT_TRUE(parent_ids->IsNull(0)); + auto is_current_ancestor = + std::static_pointer_cast<::arrow::BooleanArray>(batch->column(3)); + EXPECT_FALSE(is_current_ancestor->Value(0)); + EXPECT_TRUE(is_current_ancestor->Value(1)); +} + +TEST_F(HistoryTableTest, ScanTableWithoutSnapshots) { + ICEBERG_UNWRAP_OR_FAIL(auto history_table, MetadataTable::Make(table_)); + + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*history_table)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + EXPECT_TRUE(batches.empty()); +} + +TEST_F(HistoryTableTest, ScanSelectedColumns) { + auto snap1 = MakeTestSnapshots().first; + ICEBERG_UNWRAP_OR_FAIL( + auto table, + MakeTableWithHistory({snap1}, /*current_snapshot_id=*/1, {LogEntry(1000, 1)})); + ICEBERG_UNWRAP_OR_FAIL(auto history_table, MetadataTable::Make(table)); + + ICEBERG_UNWRAP_OR_FAIL( + auto scan, + history_table->NewScan()->Select({"is_current_ancestor", "snapshot_id"}).Build()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadMetadataTable(*scan)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + const auto& batch = batches.front(); + ASSERT_EQ(batch->num_columns(), 2); + EXPECT_EQ(batch->schema()->field(0)->name(), "snapshot_id"); + EXPECT_EQ(batch->schema()->field(1)->name(), "is_current_ancestor"); + EXPECT_EQ(std::static_pointer_cast<::arrow::Int64Array>(batch->column(0))->Value(0), 1); + EXPECT_TRUE( + std::static_pointer_cast<::arrow::BooleanArray>(batch->column(1))->Value(0)); +} + } // namespace iceberg diff --git a/src/iceberg/test/metadata_table_test_base.h b/src/iceberg/test/metadata_table_test_base.h index b9f601cc1c..60abdb0884 100644 --- a/src/iceberg/test/metadata_table_test_base.h +++ b/src/iceberg/test/metadata_table_test_base.h @@ -20,7 +20,7 @@ /// \file metadata_table_test_base.h /// Shared test base for all metadata table tests. /// -/// Provides common helpers (ReadAllBatches, MakeTestSnapshots, +/// Provides common helpers (ScanToArrowStream, ReadAllBatches, MakeTestSnapshots, /// MakeTableWithSnapshots) and the MockFileIO + MockCatalog fixture that /// every metadata table test needs. @@ -39,6 +39,8 @@ #include #include "iceberg/constants.h" +#include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_reader.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" #include "iceberg/schema_internal.h" @@ -50,6 +52,7 @@ #include "iceberg/test/mock_catalog.h" #include "iceberg/test/mock_io.h" #include "iceberg/type.h" +#include "iceberg/util/macros.h" #include "iceberg/util/timepoint.h" namespace iceberg { @@ -80,7 +83,13 @@ class MetadataTableTestBase : public ::testing::Test { "s3://bucket/meta.json", io_, catalog_)); } - /// \brief Import and consume a Scan()-produced ArrowArrayStream. + /// \brief Read every column of a metadata table with a default scan. + static Result ScanToArrowStream(const MetadataTable& table) { + ICEBERG_ASSIGN_OR_RAISE(auto scan, table.NewScan()->Build()); + return ReadMetadataTable(*scan); + } + + /// \brief Import and consume an ArrowArrayStream. static Result>> ReadAllBatches( ArrowArrayStream&& stream) { auto reader_result = ::arrow::ImportRecordBatchReader(&stream); diff --git a/src/iceberg/test/snapshots_table_test.cc b/src/iceberg/test/snapshots_table_test.cc index cddfce69f8..34fd95d475 100644 --- a/src/iceberg/test/snapshots_table_test.cc +++ b/src/iceberg/test/snapshots_table_test.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,12 @@ #include #include "iceberg/constants.h" +#include "iceberg/data_task_reader.h" #include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_reader.h" +#include "iceberg/row/struct_like.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_scan.h" #include "iceberg/test/matchers.h" #include "iceberg/test/metadata_table_test_base.h" @@ -75,9 +81,80 @@ TEST_F(SnapshotsTableTest, Construct) { EXPECT_NE(snapshots_table_->schema(), nullptr); } +TEST_F(SnapshotsTableTest, PlanFilesReturnsOneDataTask) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, snapshots_table_->NewScan()->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + ASSERT_EQ(tasks.size(), 1); + EXPECT_EQ(tasks[0]->kind(), ScanTask::Kind::kDataTask); + EXPECT_EQ(tasks[0]->estimated_row_count(), 2); + EXPECT_EQ(tasks[0]->schema(), snapshots_table_->schema()); + + ICEBERG_UNWRAP_OR_FAIL(auto rows, tasks[0]->Rows()); + ICEBERG_UNWRAP_OR_FAIL(auto all_rows, rows->ToVector()); + ASSERT_EQ(all_rows.size(), 2); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot_id, all_rows[1]->GetField(1)); + EXPECT_EQ(std::get(snapshot_id), 2); + ICEBERG_UNWRAP_OR_FAIL(auto parent_id, all_rows[0]->GetField(2)); + EXPECT_TRUE(std::holds_alternative(parent_id)); + ICEBERG_UNWRAP_OR_FAIL(auto summary, all_rows[1]->GetField(5)); + EXPECT_EQ(std::get>(summary)->size(), 10); +} + +TEST_F(SnapshotsTableTest, ScanSelectedColumns) { + ICEBERG_UNWRAP_OR_FAIL( + auto scan, + snapshots_table_->NewScan()->Select({"operation", "snapshot_id"}).Build()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadMetadataTable(*scan)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + const auto& batch = batches.front(); + ASSERT_EQ(batch->num_columns(), 2); + EXPECT_EQ(batch->schema()->field(0)->name(), "snapshot_id"); + EXPECT_EQ(batch->schema()->field(1)->name(), "operation"); + + auto snapshot_ids = std::static_pointer_cast<::arrow::Int64Array>(batch->column(0)); + EXPECT_EQ(snapshot_ids->Value(0), 1); + EXPECT_EQ(snapshot_ids->Value(1), 2); + auto operations = std::static_pointer_cast<::arrow::StringArray>(batch->column(1)); + EXPECT_EQ(operations->GetString(0), "append"); + EXPECT_EQ(operations->GetString(1), "append"); +} + +TEST_F(SnapshotsTableTest, ScanSelectedMapColumn) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, + snapshots_table_->NewScan()->Select({"summary"}).Build()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadMetadataTable(*scan)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 1); + ASSERT_EQ(batches.front()->num_columns(), 1); + auto summaries = + std::static_pointer_cast<::arrow::MapArray>(batches.front()->column(0)); + EXPECT_THAT(GetMapEntries(summaries, 1), + ::testing::Contains(::testing::Pair("total-records", "2"))); +} + +TEST_F(SnapshotsTableTest, ScanRejectsTimeTravel) { + EXPECT_THAT(snapshots_table_->NewScan()->UseSnapshot(1).Build(), + HasErrorMessage("does not support time travel")); + EXPECT_THAT(snapshots_table_->NewScan()->AsOfTime(9876543210000).Build(), + HasErrorMessage("does not support time travel")); + EXPECT_THAT( + snapshots_table_->NewScan()->UseRef(std::string(SnapshotRef::kMainBranch)).Build(), + IsOk()); +} + +TEST_F(SnapshotsTableTest, ReadMetadataTableUsesBatchSize) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, snapshots_table_->NewScan()->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ReadMetadataTable(*scan, /*batch_size=*/1)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 2); + EXPECT_EQ(batches[0]->num_rows(), 1); + EXPECT_EQ(batches[1]->num_rows(), 1); +} + TEST_F(SnapshotsTableTest, Scan) { // Scan the snapshots table once and verify all columns of the result. - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table_->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table_)); ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); ASSERT_EQ(batches.size(), 1); const auto& batch = batches.front(); @@ -141,7 +218,7 @@ TEST_F(SnapshotsTableTest, ScanEmptySnapshotList) { ICEBERG_UNWRAP_OR_FAIL(snapshots_table_, MetadataTable::Make(empty_table)); - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table_->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table_)); ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); EXPECT_TRUE(batches.empty()); } @@ -153,7 +230,7 @@ TEST_F(SnapshotsTableTest, ScanSkipsNullSnapshots) { ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table, MetadataTable::Make(table)); - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table)); ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); ASSERT_EQ(batches.size(), 1); EXPECT_EQ(batches.front()->num_rows(), 2); @@ -170,7 +247,7 @@ TEST_F(SnapshotsTableTest, ScanEmptySummary) { ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table, MetadataTable::Make(table)); - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table)); ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); ASSERT_EQ(batches.size(), 1); auto summaries = @@ -182,30 +259,30 @@ TEST_F(SnapshotsTableTest, ScanEmptySummary) { TEST_F(SnapshotsTableTest, ScanReturnsMultipleBatches) { auto snapshot = MakeTestSnapshots().first; - std::vector> snapshots(MetadataTable::kBatchSize + 1, + std::vector> snapshots(kDefaultDataTaskBatchSize + 1, snapshot); ICEBERG_UNWRAP_OR_FAIL(auto table, MakeTableWithSnapshots(std::move(snapshots), /*current_snapshot_id=*/1)); ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table, MetadataTable::Make(table)); - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table)); ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); ASSERT_EQ(batches.size(), 2); - EXPECT_EQ(batches[0]->num_rows(), MetadataTable::kBatchSize); + EXPECT_EQ(batches[0]->num_rows(), kDefaultDataTaskBatchSize); EXPECT_EQ(batches[1]->num_rows(), 1); } TEST_F(SnapshotsTableTest, ScanUsesMetadataCapturedAtCreation) { auto snapshot = MakeTestSnapshots().first; - std::vector> snapshots(MetadataTable::kBatchSize + 1, + std::vector> snapshots(kDefaultDataTaskBatchSize + 1, snapshot); ICEBERG_UNWRAP_OR_FAIL(auto table, MakeTableWithSnapshots(std::move(snapshots), /*current_snapshot_id=*/1)); ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table, MetadataTable::Make(table)); - ICEBERG_UNWRAP_OR_FAIL(auto stream, snapshots_table->Scan()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, ScanToArrowStream(*snapshots_table)); auto reader_result = ::arrow::ImportRecordBatchReader(&stream); ASSERT_TRUE(reader_result.ok()) << reader_result.status().ToString(); auto reader = std::move(reader_result).ValueUnsafe(); @@ -213,7 +290,7 @@ TEST_F(SnapshotsTableTest, ScanUsesMetadataCapturedAtCreation) { auto first_batch_result = reader->Next(); ASSERT_TRUE(first_batch_result.ok()) << first_batch_result.status().ToString(); ASSERT_NE(*first_batch_result, nullptr); - EXPECT_EQ((*first_batch_result)->num_rows(), MetadataTable::kBatchSize); + EXPECT_EQ((*first_batch_result)->num_rows(), kDefaultDataTaskBatchSize); ICEBERG_UNWRAP_OR_FAIL( auto refreshed_table, diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 477774a173..b093c553fb 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -167,6 +167,7 @@ class StrictMetricsEvaluator; /// \brief Scan task. class ChangelogScanTask; +class DataTask; class FileScanTask; class ScanTask; @@ -282,6 +283,8 @@ class PositionDeleteIndex; /// \brief Metadata tables. class HistoryTable; class MetadataTable; +class MetadataTableScan; +class MetadataTableScanBuilder; class SnapshotsTable; /// \brief Table encryption