Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/iceberg/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ set(ICEBERG_SOURCES
inspect/metadata_table.cc
inspect/metadata_scan_task.cc
inspect/metadata_scan_task_reader.cc
inspect/metadata_table_util.cc
inspect/refs_table.cc
inspect/snapshots_table.cc
inheritable_metadata.cc
Expand Down
9 changes: 3 additions & 6 deletions src/iceberg/inspect/history_table.cc
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ HistoryTable::HistoryTable(std::shared_ptr<const Table> table)

HistoryTable::~HistoryTable() = default;

const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
std::shared_ptr<Schema> HistoryTable::schema() const { return CanonicalSchema(); }

const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
Expand All @@ -95,7 +95,7 @@ Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
}

Result<std::unique_ptr<MetadataScan>> HistoryTable::MakeScan(
internal::MetadataScanContext context) const {
internal::MetadataScanContext context, std::shared_ptr<FileIO> /*io*/) const {
return HistoryScan::Make(std::move(context));
}

Expand Down Expand Up @@ -151,10 +151,7 @@ Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
context.as_of_time_in_millis.has_value()) {
return NotSupported("Metadata table does not support time travel");
}
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
ICEBERG_PRECHECK(context.projected_schema != nullptr,
"Projected schema cannot be null");
ICEBERG_RETURN_UNEXPECTED(context.Validate());
ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<HistoryRow>(
*HistoryTable::CanonicalSchema(), *context.projected_schema, kHistoryColumns));
ICEBERG_ASSIGN_OR_RAISE(auto task,
Expand Down
4 changes: 2 additions & 2 deletions src/iceberg/inspect/history_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ class ICEBERG_EXPORT HistoryTable : public MetadataTable {

MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }

const std::shared_ptr<Schema>& schema() const override;
std::shared_ptr<Schema> schema() const override;

protected:
Result<std::unique_ptr<MetadataScan>> MakeScan(
internal::MetadataScanContext context) const override;
internal::MetadataScanContext context, std::shared_ptr<FileIO> io) const override;

private:
friend class MetadataScanTaskReader;
Expand Down
35 changes: 24 additions & 11 deletions src/iceberg/inspect/metadata_scan.cc
Original file line number Diff line number Diff line change
Expand Up @@ -107,20 +107,34 @@ class TaskArrowStreamSource {

} // namespace

MetadataScanBuilder::MetadataScanBuilder(const MetadataTable& table)
: table_(&table),
context_{.kind = table.kind(),
.table_metadata = table.source_table()->metadata(),
.schema = table.schema(),
.projected_schema = table.schema()} {}
Status internal::MetadataScanContext::Validate() const {
ICEBERG_PRECHECK(table_metadata != nullptr, "Table metadata cannot be null");
ICEBERG_PRECHECK(schema != nullptr, "Metadata table schema cannot be null");
ICEBERG_PRECHECK(projected_schema != nullptr, "Projected schema cannot be null");
return {};
}

MetadataScanBuilder::MetadataScanBuilder(const MetadataTable& table,
internal::MetadataScanContext context,
std::shared_ptr<FileIO> io)
: table_(&table), context_(std::move(context)), io_(std::move(io)) {}

MetadataScanBuilder::~MetadataScanBuilder() = default;

Result<std::unique_ptr<MetadataScanBuilder>> MetadataScanBuilder::Make(
const MetadataTable& table) {
ICEBERG_PRECHECK(table.source_table() != nullptr, "Source table cannot be null");
ICEBERG_PRECHECK(table.schema() != nullptr, "Metadata table schema cannot be null");
return std::unique_ptr<MetadataScanBuilder>(new MetadataScanBuilder(table));
auto metadata = table.source_table()->metadata();
ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null");
auto io = table.source_table()->io();
ICEBERG_ASSIGN_OR_RAISE(auto schema, table.SchemaForScan(*metadata));
internal::MetadataScanContext context{.kind = table.kind(),
.table_metadata = std::move(metadata),
.schema = schema,
.projected_schema = std::move(schema)};
ICEBERG_RETURN_UNEXPECTED(context.Validate());
return std::unique_ptr<MetadataScanBuilder>(
new MetadataScanBuilder(table, std::move(context), std::move(io)));
}

MetadataScanBuilder& MetadataScanBuilder::Select(
Expand Down Expand Up @@ -170,9 +184,8 @@ MetadataScanBuilder& MetadataScanBuilder::AsOfTime(int64_t as_of_time_in_millis)

Result<std::unique_ptr<MetadataScan>> MetadataScanBuilder::Build() {
ICEBERG_RETURN_UNEXPECTED(CheckErrors());
ICEBERG_PRECHECK(context_.projected_schema != nullptr,
"Projected schema cannot be null");
return table_->MakeScan(context_);
ICEBERG_RETURN_UNEXPECTED(context_.Validate());
return table_->MakeScan(context_, io_);
}

MetadataScan::MetadataScan(internal::MetadataScanContext context)
Expand Down
7 changes: 6 additions & 1 deletion src/iceberg/inspect/metadata_scan.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ struct MetadataScanContext {
std::optional<int64_t> snapshot_id;
std::string ref_name;
std::optional<int64_t> as_of_time_in_millis;

/// \brief Validate the metadata and schemas required by all scans.
ICEBERG_EXPORT Status Validate() const;
};

} // namespace internal
Expand Down Expand Up @@ -97,10 +100,12 @@ class ICEBERG_EXPORT MetadataScanBuilder : public ErrorCollector {
Result<std::unique_ptr<MetadataScan>> Build();

private:
explicit MetadataScanBuilder(const MetadataTable& table);
MetadataScanBuilder(const MetadataTable& table, internal::MetadataScanContext context,
std::shared_ptr<FileIO> io);

const MetadataTable* table_;
internal::MetadataScanContext context_;
const std::shared_ptr<FileIO> io_;
};

/// \brief A metadata scan that plans tasks or reads all of them as Arrow batches.
Expand Down
21 changes: 21 additions & 0 deletions src/iceberg/inspect/metadata_scan_task.cc
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,25 @@ StaticMetadataTask::StaticMetadataTask(MetadataTableKind table_kind,
metadata_(std::move(metadata)),
projected_schema_(std::move(projected_schema)) {}

// SnapshotMetadataTask

Result<std::shared_ptr<SnapshotMetadataTask>> SnapshotMetadataTask::Make(
internal::MetadataScanContext context, std::shared_ptr<FileIO> io,
ReadRowsFunction read_rows) {
ICEBERG_PRECHECK(read_rows != nullptr, "Snapshot row reader cannot be null");
ICEBERG_RETURN_UNEXPECTED(context.Validate());
ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null");
return std::shared_ptr<SnapshotMetadataTask>(
new SnapshotMetadataTask(std::move(context), std::move(io), read_rows));
}

SnapshotMetadataTask::SnapshotMetadataTask(internal::MetadataScanContext context,
std::shared_ptr<FileIO> io,
ReadRowsFunction read_rows)
: context_(std::move(context)), io_(std::move(io)), read_rows_(read_rows) {}

Result<ArrowArrayStream> SnapshotMetadataTask::Open(int64_t batch_size) const {
return read_rows_(context_, io_, batch_size);
}

} // namespace iceberg
34 changes: 34 additions & 0 deletions src/iceberg/inspect/metadata_scan_task.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@

#pragma once

#include <cstdint>
#include <memory>

#include "iceberg/arrow_c_data.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/inspect/metadata_scan.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"

Expand All @@ -37,6 +40,8 @@ class ICEBERG_EXPORT MetadataScanTask {
enum class TaskKind {
/// StaticMetadataTask reads in-memory table metadata (snapshots, history, refs).
kStatic,
/// SnapshotMetadataTask reads snapshot manifests and their file entries.
kSnapshot,
};

virtual ~MetadataScanTask() = default;
Expand Down Expand Up @@ -85,4 +90,33 @@ class ICEBERG_EXPORT StaticMetadataTask : public MetadataScanTask {
const std::shared_ptr<Schema> projected_schema_;
};

/// \brief Task retaining a snapshot scan's captured settings and FileIO.
///
/// Rows are read through an in-process callback. Splitting work across multiple
/// tasks and serializable task descriptions are future work for parallel execution.
class ICEBERG_EXPORT SnapshotMetadataTask : public MetadataScanTask {
public:
using ReadRowsFunction = Result<ArrowArrayStream> (*)(
const internal::MetadataScanContext& /*context*/,
const std::shared_ptr<FileIO>& /*io*/, int64_t /*batch_size*/);

/// \brief Create a task owning the scan settings, FileIO, and row reader.
static Result<std::shared_ptr<SnapshotMetadataTask>> Make(
internal::MetadataScanContext context, std::shared_ptr<FileIO> io,
ReadRowsFunction read_rows);

Result<ArrowArrayStream> Open(int64_t batch_size) const;

TaskKind task_kind() const override { return TaskKind::kSnapshot; }
MetadataTableKind table_kind() const override { return context_.kind; }

private:
SnapshotMetadataTask(internal::MetadataScanContext context, std::shared_ptr<FileIO> io,
ReadRowsFunction read_rows);

const internal::MetadataScanContext context_;
const std::shared_ptr<FileIO> io_;
const ReadRowsFunction read_rows_;
};

} // namespace iceberg
5 changes: 5 additions & 0 deletions src/iceberg/inspect/metadata_scan_task_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ Result<std::unique_ptr<MetadataScanTaskReader>> MetadataScanTaskReader::Make(
Result<ArrowArrayStream> MetadataScanTaskReader::Open(
const MetadataScanTask& task) const {
switch (task.task_kind()) {
case MetadataScanTask::TaskKind::kSnapshot: {
const auto& snapshot_task =
internal::checked_cast<const SnapshotMetadataTask&>(task);
return snapshot_task.Open(options_.batch_size);
}
case MetadataScanTask::TaskKind::kStatic: {
const auto& static_task = internal::checked_cast<const StaticMetadataTask&>(task);
switch (static_task.table_kind()) {
Expand Down
5 changes: 5 additions & 0 deletions src/iceberg/inspect/metadata_table.cc
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ const std::shared_ptr<const Table>& MetadataTable::source_table() const {
return source_table_;
}

Result<std::shared_ptr<Schema>> MetadataTable::SchemaForScan(
const TableMetadata& metadata) const {
return schema();
}

Result<std::unique_ptr<MetadataScanBuilder>> MetadataTable::NewScan() const {
return MetadataScanBuilder::Make(*this);
}
Expand Down
11 changes: 8 additions & 3 deletions src/iceberg/inspect/metadata_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ class ICEBERG_EXPORT MetadataTable {
/// \brief Return this metadata table's kind.
virtual Kind kind() const noexcept = 0;

/// \brief Return the complete canonical schema of this metadata table.
virtual const std::shared_ptr<Schema>& schema() const = 0;
/// \brief Return shared ownership of this metadata table's canonical schema.
virtual std::shared_ptr<Schema> schema() const = 0;

/// \brief Return the source table whose metadata is exposed.
const std::shared_ptr<const Table>& source_table() const;
Expand All @@ -93,9 +93,14 @@ class ICEBERG_EXPORT MetadataTable {
protected:
explicit MetadataTable(std::shared_ptr<const Table> source_table);

/// \brief Derive a scan schema from the captured source metadata.
virtual Result<std::shared_ptr<Schema>> SchemaForScan(
const TableMetadata& metadata) const;

/// \brief Plan a scan using the captured metadata, projection, and snapshot selection.
/// \param io FileIO captured when the scan builder was created
virtual Result<std::unique_ptr<MetadataScan>> MakeScan(
internal::MetadataScanContext context) const = 0;
internal::MetadataScanContext context, std::shared_ptr<FileIO> io) const = 0;

private:
friend class MetadataScanBuilder;
Expand Down
109 changes: 109 additions & 0 deletions src/iceberg/inspect/metadata_table_util.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#include <memory>
#include <string>
#include <utility>

#include "iceberg/constants.h"
#include "iceberg/inspect/metadata_table_util_internal.h"
#include "iceberg/schema.h"
#include "iceberg/table_metadata.h"
#include "iceberg/type.h"
#include "iceberg/util/macros.h"
#include "iceberg/util/snapshot_util.h"

namespace iceberg::internal {

namespace {
/// A scan that plans one task for a metadata table's selected snapshot.
///
/// The task retains the captured metadata, schemas, and FileIO so it can outlive the
/// scan. Table-specific row reading is deferred to the supplied callback when
/// MetadataScanTaskReader opens the task.
class SnapshotMetadataScan : public MetadataScan {
public:
SnapshotMetadataScan(MetadataScanContext context,
std::shared_ptr<SnapshotMetadataTask> task)
: MetadataScan(std::move(context)), task_(std::move(task)) {}

Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override {
// Each planning call gets a fresh stream over the same retained task.
return MetadataScanTaskStream::FromVector({task_});
}

private:
const std::shared_ptr<MetadataScanTask> task_;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you plan to support multiple tasks in the future? I'm fine with the current implementation but we need to consider parallel task execution and serialization for better performance.

};
} // namespace

Result<std::unique_ptr<MetadataScan>> MakeSnapshotMetadataScan(
MetadataScanContext context, std::shared_ptr<FileIO> io,
SnapshotMetadataTask::ReadRowsFunction read_rows) {
ICEBERG_ASSIGN_OR_RAISE(auto task,
SnapshotMetadataTask::Make(context, std::move(io), read_rows));
for (const auto& projected : context.projected_schema->fields()) {
ICEBERG_ASSIGN_OR_RAISE(auto field,
context.schema->GetFieldById(projected.field_id()));
ICEBERG_PRECHECK(field.has_value(), "Unknown metadata column: {}", projected.name());
if (*field->get().type() != *projected.type()) {
return NotSupported("Partial projection of metadata column is not supported: {}",
projected.name());
}
ICEBERG_PRECHECK(!field->get().optional() || projected.optional(),
"Nullable metadata column cannot be projected as required: {}",
projected.name());
}
ICEBERG_RETURN_UNEXPECTED(
ResolveMetadataTableSnapshot(*context.table_metadata, context));
return std::make_unique<SnapshotMetadataScan>(std::move(context), std::move(task));
}

Result<std::shared_ptr<Snapshot>> ResolveMetadataTableSnapshot(
const TableMetadata& metadata, const MetadataScanContext& selection) {
const bool is_main_ref = SnapshotUtil::IsMainBranch(selection.ref_name);
if (!is_main_ref) {
ICEBERG_PRECHECK(
!selection.snapshot_id.has_value() && !selection.as_of_time_in_millis.has_value(),
"Cannot combine a snapshot reference with a snapshot ID or timestamp");
}
if (selection.snapshot_id.has_value()) {
return metadata.SnapshotById(*selection.snapshot_id);
}
if (selection.as_of_time_in_millis.has_value()) {
ICEBERG_ASSIGN_OR_RAISE(
auto snapshot_id,
SnapshotUtil::SnapshotIdAsOfTime(
metadata, TimePointMsFromUnixMs(*selection.as_of_time_in_millis)));
return metadata.SnapshotById(snapshot_id);
}
if (is_main_ref && metadata.current_snapshot_id == kInvalidSnapshotId) {
return std::shared_ptr<Snapshot>{nullptr};
}
if (!is_main_ref) {
auto ref = metadata.refs.find(selection.ref_name);
ICEBERG_PRECHECK(ref != metadata.refs.end(), "Cannot find snapshot reference '{}'",
selection.ref_name);
ICEBERG_PRECHECK(ref->second != nullptr, "Snapshot reference '{}' is null",
selection.ref_name);
}
return SnapshotUtil::LatestSnapshot(metadata, selection.ref_name);
}

} // namespace iceberg::internal
Loading
Loading