From 6c63da23fb7311271ae2a75070080e73992df2c7 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Fri, 20 Nov 2026 12:29:31 +0800 Subject: [PATCH] feat(inspect): prototype parallel manifest metadata tasks Plan one fixed task per data manifest and dispatch independent readers through bounded TaskGroup execution. Add versioned JSON descriptors with worker-injected FileIO, real manifest round-trip/concurrency/error tests, and documentation of the prototype boundaries. Co-authored-by: Codex --- docs/parallel-metadata-poc.md | 95 +++++ src/iceberg/CMakeLists.txt | 1 + src/iceberg/inspect/metadata_scan_task.h | 2 + .../inspect/metadata_scan_task_reader.cc | 6 + src/iceberg/inspect/parallel_metadata_poc.cc | 364 ++++++++++++++++++ .../inspect/parallel_metadata_poc_internal.h | 101 +++++ src/iceberg/test/CMakeLists.txt | 1 + .../inspect/parallel_metadata_poc_test.cc | 348 +++++++++++++++++ 8 files changed, 918 insertions(+) create mode 100644 docs/parallel-metadata-poc.md create mode 100644 src/iceberg/inspect/parallel_metadata_poc.cc create mode 100644 src/iceberg/inspect/parallel_metadata_poc_internal.h create mode 100644 src/iceberg/test/inspect/parallel_metadata_poc_test.cc diff --git a/docs/parallel-metadata-poc.md b/docs/parallel-metadata-poc.md new file mode 100644 index 000000000..81b6c7059 --- /dev/null +++ b/docs/parallel-metadata-poc.md @@ -0,0 +1,95 @@ + + +# Parallel metadata tasks: proof of concept + +This draft builds on Apache Iceberg C++ PR +[#1120](https://github.com/apache/iceberg-cpp/pull/1120). It demonstrates a possible +follow-up to the review discussion about multiple metadata tasks, parallel reads, +and serialization. The implementation is experimental and its new helper header +is internal and is not installed. + +## Work assignment and execution + +`PlanManifestMetadataScan()` resolves the selected snapshot once and reads its +manifest list. It returns a `MetadataScan` that plans one task per data manifest. +Each task has one distinct manifest, the fixed snapshot ID, the source schema, +the applicable partition spec, and the output projection. Deleted manifest entries +are excluded by `ManifestReader::LiveEntries()`. + +The output is a small demonstration schema: `manifest_path`, `file_path`, +`record_count`, and `file_size_in_bytes`. It is not the full FilesTable schema. + +Callers can use the existing `ToArrowArrayStream()` for sequential execution, or +execute the planned tasks through the prototype collector: + +```cpp +// `context` captures table metadata and the PoC schema/projection. `io` and +// `executor` belong to the caller. Results use the original task planning order. +auto scan = PlanManifestMetadataScan(context, io).value(); +auto tasks = scan->PlanTasks().value(); +auto stream = ExecuteManifestMetadataTasks( + context.projected_schema, std::move(tasks), + {.executor = &executor, .max_tasks_in_flight = 4, .batch_size = 1024}).value(); +``` + +`ExecuteManifestMetadataTasks()` uses the existing `TaskGroup` and `Executor`. +Each worker opens its own reader/Arrow stream and consumes it independently. Work +is submitted in bounded windows. On failure, the active window is drained, later +windows are not submitted, and buffered partial output is released. The supplied +FileIO implementation must support concurrent reads. + +The collector is synchronous and eagerly buffers every output batch before +returning its Arrow stream. The in-flight task limit bounds concurrency, not +total result memory. A production implementation needs bounded output queues, +streaming backpressure, and cancellation semantics. + +## Task transport + +`SerializeManifestMetadataTask()` writes versioned JSON containing the task type, +fixed snapshot ID, manifest location/length and inherited snapshot/sequence +metadata, source schema, partition spec, and projection. It carries no function +pointer, object address, FileIO object, or driver-side table handle. + +Workers call `DeserializeManifestMetadataTask(payload, worker_io)` with their own +configured execution FileIO. Reader dispatch uses the task kind. Workers read the +assigned manifest directly and do not resolve the current table snapshot or +reread its manifest list. The format is a prototype, not a stable wire protocol. + +The tests copy only manifest bytes into independent worker storage, destroy the +driver's scan/table/IO, deserialize the task payloads, and compare parallel output +with the original sequential scan. An instrumented FileIO makes two opens +rendezvous to prove concurrent dispatch without timing-based speed assertions. + +## Scope of this draft + +- Unencrypted data manifests, live data-file metadata, fixed snapshot selection, + whole-column projections, and task-local batches are implemented. +- JSON task transport and bounded local parallel execution are executable. A + remote scheduler or RPC transport is outside this draft. +- Delete-file metadata, filters, the full FilesTable schema, and streaming + manifest-entry decoding need further integration. +- Partition summaries need partial aggregation followed by a merge using the + original partition grouping rules. Concatenating per-manifest summaries would + produce duplicate partition rows, so no partition-table parallelism is claimed. +- The manifest-list planning pass remains sequential; manifest grouping and cost + balancing can build on the descriptor without changing the executor boundary. + +Run `metadata_table_test` with the filter `ParallelMetadataPocTest.*` for the +prototype coverage, and run the full target to check existing metadata scans. diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 0768f1aaf..69dd9498b 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -64,6 +64,7 @@ set(ICEBERG_SOURCES inspect/metadata_scan_task.cc inspect/metadata_scan_task_reader.cc inspect/metadata_table_util.cc + inspect/parallel_metadata_poc.cc inspect/refs_table.cc inspect/snapshots_table.cc inheritable_metadata.cc diff --git a/src/iceberg/inspect/metadata_scan_task.h b/src/iceberg/inspect/metadata_scan_task.h index eab2d2693..393dc3ba0 100644 --- a/src/iceberg/inspect/metadata_scan_task.h +++ b/src/iceberg/inspect/metadata_scan_task.h @@ -42,6 +42,8 @@ class ICEBERG_EXPORT MetadataScanTask { kStatic, /// SnapshotMetadataTask reads snapshot manifests and their file entries. kSnapshot, + /// Experimental manifest-sized data-file metadata work (parallel PoC). + kManifestEntries, }; virtual ~MetadataScanTask() = default; diff --git a/src/iceberg/inspect/metadata_scan_task_reader.cc b/src/iceberg/inspect/metadata_scan_task_reader.cc index a43b4f26c..04a45706f 100644 --- a/src/iceberg/inspect/metadata_scan_task_reader.cc +++ b/src/iceberg/inspect/metadata_scan_task_reader.cc @@ -25,6 +25,7 @@ #include "iceberg/inspect/history_table.h" #include "iceberg/inspect/metadata_scan_task.h" #include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/parallel_metadata_poc_internal.h" #include "iceberg/inspect/refs_table.h" #include "iceberg/inspect/snapshots_table.h" #include "iceberg/util/checked_cast.h" @@ -47,6 +48,11 @@ Result> MetadataScanTaskReader::Make( Result MetadataScanTaskReader::Open( const MetadataScanTask& task) const { switch (task.task_kind()) { + case MetadataScanTask::TaskKind::kManifestEntries: { + const auto& manifest_task = + internal::checked_cast(task); + return manifest_task.Open(options_.batch_size); + } case MetadataScanTask::TaskKind::kSnapshot: { const auto& snapshot_task = internal::checked_cast(task); diff --git a/src/iceberg/inspect/parallel_metadata_poc.cc b/src/iceberg/inspect/parallel_metadata_poc.cc new file mode 100644 index 000000000..761b0048c --- /dev/null +++ b/src/iceberg/inspect/parallel_metadata_poc.cc @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "iceberg/arrow_c_data_util_internal.h" +#include "iceberg/arrow_row_builder_internal.h" +#include "iceberg/inspect/metadata_scan_task_reader.h" +#include "iceberg/inspect/metadata_scan_util_internal.h" +#include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/metadata_table_util_internal.h" +#include "iceberg/inspect/parallel_metadata_poc_internal.h" +#include "iceberg/json_serde_internal.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/schema_internal.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_metadata.h" +#include "iceberg/type.h" +#include "iceberg/util/json_util_internal.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/task_group.h" + +namespace iceberg::internal { +namespace { + +struct FileRow { + std::string manifest_path; + std::string file_path; + int64_t record_count; + int64_t file_size_in_bytes; +}; + +constexpr std::array, 4> kColumns = {{ + {.field_id = 1, + .append = [](ArrowArray* array, + const FileRow& row) { return AppendString(array, row.manifest_path); }}, + {.field_id = 2, + .append = [](ArrowArray* array, + const FileRow& row) { return AppendString(array, row.file_path); }}, + {.field_id = 3, + .append = [](ArrowArray* array, + const FileRow& row) { return AppendInt(array, row.record_count); }}, + {.field_id = 4, + .append = + [](ArrowArray* array, const FileRow& row) { + return AppendInt(array, row.file_size_in_bytes); + }}, +}}; + +class ManifestMetadataScan : public MetadataScan { + public: + ManifestMetadataScan(MetadataScanContext context, + std::vector> tasks) + : MetadataScan(std::move(context)), tasks_(std::move(tasks)) {} + + Result PlanTasksStream() const override { + return MetadataScanTaskStream::FromVector(tasks_); + } + + private: + const std::vector> tasks_; +}; + +using Batches = std::vector>; + +class CollectedBatches { + public: + CollectedBatches(std::shared_ptr schema, Batches batches) + : schema_(std::move(schema)), batches_(std::move(batches)) {} + + Status Close() { + batches_.clear(); + return {}; + } + + Result Schema() { + ArrowSchema out{}; + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*schema_, &out)); + return out; + } + + Result> Next() { + while (task_index_ < batches_.size()) { + auto& task_batches = batches_[task_index_]; + if (batch_index_ < task_batches.size()) { + ArrowArray out{}; + task_batches[batch_index_++].move(&out); + return out; + } + ++task_index_; + batch_index_ = 0; + } + return std::nullopt; + } + + private: + std::shared_ptr schema_; + Batches batches_; + size_t task_index_ = 0; + size_t batch_index_ = 0; +}; + +Status CollectTask(const MetadataScanTask& task, int64_t batch_size, + std::vector& batches) { + ICEBERG_ASSIGN_OR_RAISE(auto reader, + MetadataScanTaskReader::Make({.batch_size = batch_size})); + ICEBERG_ASSIGN_OR_RAISE(auto stream, reader->Open(task)); + nanoarrow::UniqueArrayStream owned(&stream); + while (true) { + nanoarrow::UniqueArray array; + if (owned->get_next(owned.get(), array.get()) != 0) { + const char* message = + owned->get_last_error != nullptr ? owned->get_last_error(owned.get()) : nullptr; + return IOError("{}", message != nullptr ? message : "Metadata task read failed"); + } + if (array->release == nullptr) { + return {}; + } + batches.push_back(std::move(array)); + } +} + +} // namespace + +std::shared_ptr ManifestMetadataSchema() { + static const auto schema = std::make_shared(std::vector{ + SchemaField::MakeRequired(1, "manifest_path", string()), + SchemaField::MakeRequired(2, "file_path", string()), + SchemaField::MakeRequired(3, "record_count", int64()), + SchemaField::MakeRequired(4, "file_size_in_bytes", int64())}); + return schema; +} + +Result> ManifestMetadataTask::Make( + ManifestMetadataTaskDescriptor descriptor, std::shared_ptr io) { + ICEBERG_PRECHECK(io != nullptr, "Worker FileIO cannot be null"); + ICEBERG_PRECHECK(descriptor.snapshot_id >= 0, "Snapshot ID must be non-negative"); + ICEBERG_PRECHECK(!descriptor.manifest.manifest_path.empty() && + descriptor.manifest.manifest_length > 0, + "Manifest path and positive length are required"); + ICEBERG_PRECHECK(descriptor.manifest.added_snapshot_id.has_value(), + "Manifest must belong to a committed snapshot"); + ICEBERG_PRECHECK(descriptor.manifest.sequence_number >= 0, + "Manifest sequence number must be non-negative"); + if (descriptor.manifest.content != ManifestContent::kData || + !descriptor.manifest.key_metadata.empty()) { + return NotSupported("PoC supports unencrypted data manifests only"); + } + ICEBERG_PRECHECK(descriptor.source_schema != nullptr && + descriptor.partition_spec != nullptr && + descriptor.projected_schema != nullptr, + "Task schemas and partition spec cannot be null"); + ICEBERG_PRECHECK( + descriptor.manifest.partition_spec_id == descriptor.partition_spec->spec_id(), + "Task partition spec does not match the manifest"); + ICEBERG_RETURN_UNEXPECTED( + descriptor.partition_spec->RawPartitionType(*descriptor.source_schema)); + ICEBERG_RETURN_UNEXPECTED(SelectAppendFns( + *ManifestMetadataSchema(), *descriptor.projected_schema, kColumns)); + return std::shared_ptr( + new ManifestMetadataTask(std::move(descriptor), std::move(io))); +} + +ManifestMetadataTask::ManifestMetadataTask(ManifestMetadataTaskDescriptor descriptor, + std::shared_ptr io) + : descriptor_(std::move(descriptor)), io_(std::move(io)) {} + +MetadataTableKind ManifestMetadataTask::table_kind() const { + return MetadataTableKind::kDataFiles; +} + +Result ManifestMetadataTask::Open(int64_t batch_size) const { + ICEBERG_PRECHECK(batch_size > 0, "Batch size must be positive"); + ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(descriptor_.manifest, io_, + descriptor_.source_schema, + descriptor_.partition_spec)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); + auto rows = std::make_shared>(); + rows->reserve(entries.size()); + for (const auto& entry : entries) { + ICEBERG_CHECK(entry.data_file != nullptr, "Live entry has no data file"); + rows->push_back({.manifest_path = descriptor_.manifest.manifest_path, + .file_path = entry.data_file->file_path, + .record_count = entry.data_file->record_count, + .file_size_in_bytes = entry.data_file->file_size_in_bytes}); + } + return MakeMetadataBatchStream( + ManifestMetadataSchema(), descriptor_.projected_schema, kColumns, rows->size(), + batch_size, [rows](size_t index) -> const FileRow& { return (*rows)[index]; }); +} + +Result> PlanManifestMetadataScan( + MetadataScanContext context, std::shared_ptr io) { + ICEBERG_RETURN_UNEXPECTED(context.Validate()); + ICEBERG_PRECHECK(context.kind == MetadataTableKind::kDataFiles, + "PoC plans data-file metadata only"); + ICEBERG_PRECHECK(io != nullptr, "Planning FileIO cannot be null"); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, + ResolveMetadataTableSnapshot(*context.table_metadata, context)); + std::vector> tasks; + if (snapshot != nullptr) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, + snapshot->schema_id.has_value() + ? context.table_metadata->SchemaById(*snapshot->schema_id) + : context.table_metadata->Schema()); + SnapshotReader reader(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, reader.DataManifests(io)); + tasks.reserve(manifests.size()); + for (const auto& manifest : manifests) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, context.table_metadata->PartitionSpecById( + manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto task, + ManifestMetadataTask::Make({.snapshot_id = snapshot->snapshot_id, + .manifest = manifest, + .source_schema = schema, + .partition_spec = std::move(spec), + .projected_schema = context.projected_schema}, + io)); + tasks.push_back(std::move(task)); + } + } + return std::make_unique(std::move(context), std::move(tasks)); +} + +Result SerializeManifestMetadataTask(const ManifestMetadataTask& task) { + const auto& descriptor = task.descriptor(); + const auto& manifest = descriptor.manifest; + ICEBERG_ASSIGN_OR_RAISE(auto schema, ToJson(*descriptor.source_schema)); + ICEBERG_ASSIGN_OR_RAISE(auto projection, ToJson(*descriptor.projected_schema)); + nlohmann::json manifest_json = { + {"path", manifest.manifest_path}, + {"length", manifest.manifest_length}, + {"spec-id", manifest.partition_spec_id}, + {"sequence-number", manifest.sequence_number}, + {"min-sequence-number", manifest.min_sequence_number}, + {"added-snapshot-id", *manifest.added_snapshot_id}, + }; + if (manifest.first_row_id.has_value()) { + manifest_json["first-row-id"] = *manifest.first_row_id; + } + nlohmann::json json = {{"version", 1}, + {"task-type", "data-file-metadata-manifest"}, + {"snapshot-id", descriptor.snapshot_id}, + {"manifest", std::move(manifest_json)}, + {"source-schema", std::move(schema)}, + {"partition-spec", ToJson(*descriptor.partition_spec)}, + {"projection", std::move(projection)}}; + try { + return json.dump(); + } catch (const nlohmann::json::exception& error) { + return JsonParseError("Cannot encode manifest task: {}", error.what()); + } +} + +Result> DeserializeManifestMetadataTask( + std::string_view text, std::shared_ptr io) { + ICEBERG_ASSIGN_OR_RAISE(auto json, ParseJsonString(text)); + ICEBERG_ASSIGN_OR_RAISE(auto version, GetJsonInteger(json, "version")); + if (version != 1) { + return NotSupported("Unsupported manifest task version: {}", version); + } + ICEBERG_ASSIGN_OR_RAISE(auto type, GetJsonValue(json, "task-type")); + if (type != "data-file-metadata-manifest") { + return NotSupported("Unsupported manifest task type: {}", type); + } + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, GetJsonInteger(json, "snapshot-id")); + ICEBERG_ASSIGN_OR_RAISE(auto manifest_json, + GetJsonValue(json, "manifest")); + ManifestFile manifest; + ICEBERG_ASSIGN_OR_RAISE(manifest.manifest_path, + GetJsonValue(manifest_json, "path")); + ICEBERG_ASSIGN_OR_RAISE(manifest.manifest_length, + GetJsonInteger(manifest_json, "length")); + ICEBERG_ASSIGN_OR_RAISE(manifest.partition_spec_id, + GetJsonInteger(manifest_json, "spec-id")); + ICEBERG_ASSIGN_OR_RAISE(manifest.sequence_number, + GetJsonInteger(manifest_json, "sequence-number")); + ICEBERG_ASSIGN_OR_RAISE(manifest.min_sequence_number, + GetJsonInteger(manifest_json, "min-sequence-number")); + ICEBERG_ASSIGN_OR_RAISE(auto added_snapshot, + GetJsonInteger(manifest_json, "added-snapshot-id")); + manifest.added_snapshot_id = added_snapshot; + if (manifest_json.contains("first-row-id")) { + ICEBERG_ASSIGN_OR_RAISE(auto first_row, + GetJsonInteger(manifest_json, "first-row-id")); + manifest.first_row_id = first_row; + } + ICEBERG_ASSIGN_OR_RAISE(auto schema_json, + GetJsonValue(json, "source-schema")); + ICEBERG_ASSIGN_OR_RAISE(auto schema, SchemaFromJson(schema_json)); + ICEBERG_ASSIGN_OR_RAISE(auto spec_json, + GetJsonValue(json, "partition-spec")); + ICEBERG_ASSIGN_OR_RAISE(auto spec, PartitionSpecFromJson(spec_json)); + ICEBERG_ASSIGN_OR_RAISE(auto projection_json, + GetJsonValue(json, "projection")); + ICEBERG_ASSIGN_OR_RAISE(auto projection, SchemaFromJson(projection_json)); + return ManifestMetadataTask::Make({.snapshot_id = snapshot_id, + .manifest = std::move(manifest), + .source_schema = std::move(schema), + .partition_spec = std::move(spec), + .projected_schema = std::move(projection)}, + std::move(io)); +} + +Result ExecuteManifestMetadataTasks( + std::shared_ptr projected_schema, + std::vector> tasks, + ParallelMetadataReadOptions options) { + ICEBERG_PRECHECK(projected_schema != nullptr, "Output schema cannot be null"); + ICEBERG_PRECHECK(options.batch_size > 0 && options.max_tasks_in_flight > 0, + "Batch size and maximum in-flight task count must be positive"); + for (const auto& task : tasks) { + const auto* manifest_task = dynamic_cast(task.get()); + ICEBERG_PRECHECK(manifest_task != nullptr, "Expected manifest metadata tasks"); + ICEBERG_PRECHECK(*manifest_task->descriptor().projected_schema == *projected_schema, + "All tasks must have the requested output schema"); + } + Batches batches(tasks.size()); + for (size_t begin = 0; begin < tasks.size();) { + const size_t count = std::min(options.max_tasks_in_flight, tasks.size() - begin); + TaskGroup<> group; + if (options.executor != nullptr) { + group.SetExecutor(std::ref(*options.executor)); + } + for (size_t index = begin; index < begin + count; ++index) { + group.Submit([&, index]() { + return CollectTask(*tasks[index], options.batch_size, batches[index]); + }); + } + ICEBERG_RETURN_UNEXPECTED(std::move(group).Run()); + begin += count; + } + return MakeArrowArrayStream(std::make_unique( + std::move(projected_schema), std::move(batches))); +} + +} // namespace iceberg::internal diff --git a/src/iceberg/inspect/parallel_metadata_poc_internal.h b/src/iceberg/inspect/parallel_metadata_poc_internal.h new file mode 100644 index 000000000..978c9055b --- /dev/null +++ b/src/iceberg/inspect/parallel_metadata_poc_internal.h @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "iceberg/arrow_c_data.h" +#include "iceberg/constants.h" +#include "iceberg/iceberg_export.h" +#include "iceberg/inspect/metadata_scan.h" +#include "iceberg/inspect/metadata_scan_task.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg::internal { + +/// Experimental, non-installed API demonstrating manifest-sized metadata tasks. +/// The output columns are manifest_path, file_path, record_count, file_size_in_bytes. +ICEBERG_EXPORT std::shared_ptr ManifestMetadataSchema(); + +/// A fixed work assignment. Schema/spec/projection objects must remain immutable. +/// Runtime FileIO and reader callbacks are deliberately absent from the descriptor. +struct ManifestMetadataTaskDescriptor { + int64_t snapshot_id = kInvalidSnapshotId; + ManifestFile manifest; + std::shared_ptr source_schema; + std::shared_ptr partition_spec; + std::shared_ptr projected_schema; +}; + +class ICEBERG_EXPORT ManifestMetadataTask final : public MetadataScanTask { + public: + static Result> Make( + ManifestMetadataTaskDescriptor descriptor, std::shared_ptr io); + + TaskKind task_kind() const override { return TaskKind::kManifestEntries; } + MetadataTableKind table_kind() const override; + + const ManifestMetadataTaskDescriptor& descriptor() const { return descriptor_; } + Result Open(int64_t batch_size) const; + + private: + ManifestMetadataTask(ManifestMetadataTaskDescriptor descriptor, + std::shared_ptr io); + + const ManifestMetadataTaskDescriptor descriptor_; + const std::shared_ptr io_; +}; + +/// Resolve a snapshot once, then plan one disjoint task per data manifest. +ICEBERG_EXPORT Result> PlanManifestMetadataScan( + MetadataScanContext context, std::shared_ptr io); + +/// Versioned JSON with the work assignment, source schema/spec and projection. +ICEBERG_EXPORT Result SerializeManifestMetadataTask( + const ManifestMetadataTask& task); + +/// Workers reconstruct a task with their own execution FileIO. +ICEBERG_EXPORT Result> +DeserializeManifestMetadataTask(std::string_view json, std::shared_ptr io); + +struct ParallelMetadataReadOptions { + /// Non-owning; the executor must make progress until this synchronous call ends. + Executor* executor = nullptr; + size_t max_tasks_in_flight = 4; + int64_t batch_size = 1024; +}; + +/// Consume independent task streams in bounded parallel windows, preserving task +/// order. This PoC eagerly buffers every output batch before returning. Streaming +/// backpressure, cancellation and partition-stat reduction require further work. +/// An error drains the active window and releases all buffered partial results. +ICEBERG_EXPORT Result ExecuteManifestMetadataTasks( + std::shared_ptr projected_schema, + std::vector> tasks, + ParallelMetadataReadOptions options = {}); + +} // namespace iceberg::internal diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 59506b166..7190eee19 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -207,6 +207,7 @@ if(ICEBERG_BUILD_BUNDLE) USE_BUNDLE SOURCES inspect/metadata_snapshot_scan_test.cc + inspect/parallel_metadata_poc_test.cc inspect/history_table_test.cc inspect/metadata_table_test.cc inspect/refs_table_test.cc diff --git a/src/iceberg/test/inspect/parallel_metadata_poc_test.cc b/src/iceberg/test/inspect/parallel_metadata_poc_test.cc new file mode 100644 index 000000000..ba59428d4 --- /dev/null +++ b/src/iceberg/test/inspect/parallel_metadata_poc_test.cc @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "iceberg/avro/avro_register.h" +#include "iceberg/constants.h" +#include "iceberg/file_io.h" +#include "iceberg/inspect/metadata_scan_task_reader.h" +#include "iceberg/inspect/metadata_table.h" +#include "iceberg/inspect/parallel_metadata_poc_internal.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_writer.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_metadata.h" +#include "iceberg/test/arrow_test_util.h" +#include "iceberg/test/executor.h" +#include "iceberg/test/inspect/metadata_table_test_base.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/type.h" +#include "iceberg/util/macros.h" + +namespace iceberg::internal { +namespace { + +/// The first two manifest opens rendezvous, proving concurrent worker reads rather +/// than relying on elapsed-time assertions. A sequential implementation times out. +class ProbedFileIO : public FileIO { + public: + ProbedFileIO(std::shared_ptr delegate, bool require_overlap) + : delegate_(std::move(delegate)), require_overlap_(require_overlap) {} + + Result> NewInputFile(std::string path) override { + ICEBERG_RETURN_UNEXPECTED(OnOpen()); + return delegate_->NewInputFile(std::move(path)); + } + + Result> NewInputFile(std::string path, + size_t length) override { + ICEBERG_RETURN_UNEXPECTED(OnOpen()); + return delegate_->NewInputFile(std::move(path), length); + } + + int open_count() const { return opens_.load(); } + + int peak() const { + std::lock_guard lock(mutex_); + return peak_; + } + + private: + Status OnOpen() { + opens_.fetch_add(1); + if (!require_overlap_) { + return {}; + } + std::unique_lock lock(mutex_); + ++active_; + peak_ = std::max(peak_, active_); + ready_.notify_all(); + const bool overlapped = + ready_.wait_for(lock, std::chrono::seconds(5), [&] { return peak_ >= 2; }); + --active_; + if (!overlapped) { + return IOError("Manifest tasks did not execute concurrently"); + } + return {}; + } + + const std::shared_ptr delegate_; + const bool require_overlap_; + std::atomic opens_{0}; + mutable std::mutex mutex_; + std::condition_variable ready_; + int active_ = 0; + int peak_ = 0; +}; + +class ParallelMetadataPocTest : public MetadataTableTestBase { + protected: + void SetUp() override { + MetadataTableTestBase::SetUp(); + avro::RegisterAll(); + ICEBERG_UNWRAP_OR_FAIL(spec_, PartitionSpec::Make(0, {})); + metadata_->partition_specs = {spec_}; + metadata_->default_spec_id = 0; + for (int index = 0; index < 4; ++index) { + const auto path = "manifest-" + std::to_string(index) + ".avro"; + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + ManifestWriter::MakeWriter(2, 100, path, io_, spec_, metadata_->schemas[0], + ManifestContent::kData)); + ASSERT_THAT(writer->WriteAddedEntry(File("added-" + std::to_string(index)), 7), + IsOk()); + ASSERT_THAT(writer->WriteExistingEntry(File("existing-" + std::to_string(index)), + 100, 6, 6), + IsOk()); + ASSERT_THAT( + writer->WriteDeletedEntry(File("deleted-" + std::to_string(index)), 5, 5), + IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifest, writer->ToManifestFile()); + manifests_.push_back(std::move(manifest)); + } + ICEBERG_UNWRAP_OR_FAIL(auto list, + ManifestListWriter::MakeWriter(2, 100, std::nullopt, + "manifest-list.avro", io_, 7)); + ASSERT_THAT(list->AddAll(manifests_), IsOk()); + ASSERT_THAT(list->Close(), IsOk()); + metadata_->snapshots = { + std::make_shared(Snapshot{.snapshot_id = 100, + .sequence_number = 7, + .timestamp_ms = TimePointMsFromUnixMs(1000), + .manifest_list = "manifest-list.avro", + .schema_id = metadata_->current_schema_id})}; + metadata_->current_snapshot_id = 100; + } + + static std::shared_ptr File(std::string path) { + return std::make_shared(DataFile{ + .file_path = std::move(path), .record_count = 10, .file_size_in_bytes = 100}); + } + + MetadataScanContext Context(std::shared_ptr projection = nullptr) { + return {.kind = MetadataTableKind::kDataFiles, + .table_metadata = metadata_, + .schema = ManifestMetadataSchema(), + .projected_schema = + projection ? std::move(projection) : ManifestMetadataSchema()}; + } + + std::shared_ptr WorkerStorage() { + auto storage = std::make_shared(); + for (const auto& manifest : manifests_) { + storage->AddFile(manifest.manifest_path, io_->FileData(manifest.manifest_path)); + } + // No manifest list or source-table object is available to the worker. + return storage; + } + + std::shared_ptr spec_; + std::vector manifests_; +}; + +TEST_F(ParallelMetadataPocTest, ParallelRoundTripMatchesSequentialAndOverlapsReads) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + ASSERT_EQ(tasks.size(), 4); + ICEBERG_UNWRAP_OR_FAIL(auto sequential_stream, scan->ToArrowArrayStream()); + ICEBERG_UNWRAP_OR_FAIL(auto expected, ReadArrowTable(std::move(sequential_stream))); + ASSERT_EQ(expected->num_rows(), 8); + + auto worker_io = std::make_shared(WorkerStorage(), true); + std::vector payloads; + for (const auto& task : tasks) { + const auto& manifest_task = dynamic_cast(*task); + ICEBERG_UNWRAP_OR_FAIL(auto json, SerializeManifestMetadataTask(manifest_task)); + payloads.push_back(std::move(json)); + } + std::weak_ptr driver_io = io_; + tasks.clear(); + scan.reset(); + table_.reset(); + metadata_.reset(); + io_.reset(); + EXPECT_TRUE(driver_io.expired()); + + for (const auto& json : payloads) { + ICEBERG_UNWRAP_OR_FAIL(auto task, DeserializeManifestMetadataTask(json, worker_io)); + EXPECT_EQ(task->descriptor().snapshot_id, 100); + tasks.push_back(std::move(task)); + } + test::ThreadExecutor executor; + ICEBERG_UNWRAP_OR_FAIL( + auto parallel_stream, + ExecuteManifestMetadataTasks( + ManifestMetadataSchema(), tasks, + {.executor = &executor, .max_tasks_in_flight = 2, .batch_size = 1})); + ICEBERG_UNWRAP_OR_FAIL(auto actual, ReadArrowTable(std::move(parallel_stream))); + EXPECT_TRUE(TablesEqual(*expected, *actual)); + EXPECT_EQ(worker_io->open_count(), 4); + EXPECT_EQ(worker_io->peak(), 2); + EXPECT_EQ(executor.submit_count(), 4); +} + +TEST_F(ParallelMetadataPocTest, PlanningAssignsEachManifestOnceAndPinsSnapshot) { + auto context = Context(); + context.snapshot_id = 100; + metadata_->snapshots.push_back( + std::make_shared(Snapshot{.snapshot_id = 200})); + metadata_->current_snapshot_id = 200; + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(context, io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + ASSERT_EQ(tasks.size(), 4); + std::set paths; + for (const auto& task : tasks) { + const auto& descriptor = + dynamic_cast(*task).descriptor(); + EXPECT_EQ(descriptor.snapshot_id, 100); + EXPECT_TRUE(paths.insert(descriptor.manifest.manifest_path).second); + } + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->ToArrowArrayStream()); + ICEBERG_UNWRAP_OR_FAIL(auto rows, ReadArrowTable(std::move(stream))); + EXPECT_EQ(rows->num_rows(), 8); +} + +TEST_F(ParallelMetadataPocTest, ProjectionAndBatchSizeSurviveSerialization) { + auto projection = std::make_shared( + std::vector{SchemaField::MakeRequired(4, "bytes", int64()), + SchemaField::MakeRequired(2, "path", string())}); + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(projection), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + const auto& original = dynamic_cast(*tasks[0]); + ICEBERG_UNWRAP_OR_FAIL(auto payload, SerializeManifestMetadataTask(original)); + ICEBERG_UNWRAP_OR_FAIL(auto restored, DeserializeManifestMetadataTask(payload, io_)); + ICEBERG_UNWRAP_OR_FAIL(auto reader, MetadataScanTaskReader::Make({.batch_size = 1})); + ICEBERG_UNWRAP_OR_FAIL(auto stream, reader->Open(*restored)); + ICEBERG_UNWRAP_OR_FAIL(auto batches, ReadAllBatches(std::move(stream))); + ASSERT_EQ(batches.size(), 2); + EXPECT_EQ(batches[0]->num_rows(), 1); + EXPECT_EQ(batches[0]->schema()->field_names(), + (std::vector{"bytes", "path"})); + EXPECT_EQ( + std::static_pointer_cast<::arrow::Int64Array>(batches[0]->column(0))->Value(0), + 100); +} + +TEST_F(ParallelMetadataPocTest, EmptySnapshotProducesAnEmptyStream) { + metadata_->current_snapshot_id = kInvalidSnapshotId; + metadata_->snapshots.clear(); + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + EXPECT_TRUE(tasks.empty()); + ICEBERG_UNWRAP_OR_FAIL(auto stream, + ExecuteManifestMetadataTasks(ManifestMetadataSchema(), tasks)); + ICEBERG_UNWRAP_OR_FAIL(auto actual, ReadArrowTable(std::move(stream))); + EXPECT_EQ(actual->num_rows(), 0); + EXPECT_EQ(actual->num_columns(), 4); +} + +TEST_F(ParallelMetadataPocTest, RejectsMalformedOrUnsupportedTransport) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + const auto& task = dynamic_cast(*tasks[0]); + ICEBERG_UNWRAP_OR_FAIL(auto payload, SerializeManifestMetadataTask(task)); + auto json = nlohmann::json::parse(payload); + json["version"] = 2; + EXPECT_THAT(DeserializeManifestMetadataTask(json.dump(), io_), + IsError(ErrorKind::kNotSupported)); + json = nlohmann::json::parse(payload); + json["snapshot-id"] = 1.5; + EXPECT_FALSE(DeserializeManifestMetadataTask(json.dump(), io_).has_value()); + json = nlohmann::json::parse(payload); + json["manifest"]["spec-id"] = int64_t{1} << 40; + EXPECT_FALSE(DeserializeManifestMetadataTask(json.dump(), io_).has_value()); + json = nlohmann::json::parse(payload); + json.erase("source-schema"); + EXPECT_FALSE(DeserializeManifestMetadataTask(json.dump(), io_).has_value()); + EXPECT_FALSE(DeserializeManifestMetadataTask("{", io_).has_value()); + EXPECT_THAT(DeserializeManifestMetadataTask(payload, nullptr), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(ParallelMetadataPocTest, ReadFailureDrainsTheActiveWindowAndStopsSubmission) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + auto worker_io = std::make_shared(WorkerStorage(), false); + for (size_t index = 0; index < tasks.size(); ++index) { + auto descriptor = + dynamic_cast(*tasks[index]).descriptor(); + if (index == 0) { + descriptor.manifest.manifest_path = "missing.avro"; + } + ICEBERG_UNWRAP_OR_FAIL(auto worker_task, + ManifestMetadataTask::Make(std::move(descriptor), worker_io)); + tasks[index] = std::move(worker_task); + } + test::ThreadExecutor executor; + auto result = ExecuteManifestMetadataTasks( + ManifestMetadataSchema(), tasks, + {.executor = &executor, .max_tasks_in_flight = 2, .batch_size = 1}); + EXPECT_FALSE(result.has_value()); + EXPECT_EQ(executor.submit_count(), 2); + EXPECT_EQ(worker_io->open_count(), 2); +} + +TEST_F(ParallelMetadataPocTest, RejectsInvalidExecutionInputs) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + EXPECT_THAT(ExecuteManifestMetadataTasks(ManifestMetadataSchema(), tasks, + {.max_tasks_in_flight = 0}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT( + ExecuteManifestMetadataTasks(ManifestMetadataSchema(), tasks, {.batch_size = 0}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(ExecuteManifestMetadataTasks(ManifestMetadataSchema(), {nullptr}), + IsError(ErrorKind::kInvalidArgument)); + auto other_schema = std::make_shared(std::vector{}); + EXPECT_THAT(ExecuteManifestMetadataTasks(other_schema, tasks), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(ParallelMetadataPocTest, ExecutorRejectionIsReported) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, PlanManifestMetadataScan(Context(), io_)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanTasks()); + test::ThreadExecutor executor(IOError("Executor rejected task")); + EXPECT_THAT( + ExecuteManifestMetadataTasks(ManifestMetadataSchema(), tasks, + {.executor = &executor, .max_tasks_in_flight = 2}), + IsError(ErrorKind::kIOError)); + EXPECT_EQ(executor.submit_count(), 2); +} + +} // namespace +} // namespace iceberg::internal