Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions crates/datafusion/src/metadata/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

use std::sync::Arc;

use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::common::Statistics;
use datafusion::common::tree_node::TreeNodeRecursion;
Expand All @@ -28,9 +29,11 @@ use datafusion::physical_expr::projection::ProjectionExprs;
use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr};
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{DisplayFormatType, Partitioning};
use futures::{StreamExt, TryStreamExt};
use futures::{Stream, StreamExt, TryStreamExt};
use iceberg::inspect::MetadataTableType;

use super::table::IcebergMetadataTableProvider;
use crate::to_datafusion_error;

/// Reads rows from an Iceberg metadata table through DataFusion's `DataSource` API.
#[derive(Debug, Clone)]
Expand Down Expand Up @@ -69,7 +72,7 @@ impl DataSource for IcebergMetadataDataSource {
_partition: usize,
_context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
let fut = self.provider.clone().scan();
let fut = read_all(self.provider.clone());
let projection = self.projection.clone();

// TODO: Push these projections down into the scan layer instead of manually iterating over the result set
Expand Down Expand Up @@ -133,3 +136,17 @@ impl DataSource for IcebergMetadataDataSource {
Ok(TreeNodeRecursion::Continue)
}
}

/// Reads every row of `provider`'s metadata table.
async fn read_all(
provider: IcebergMetadataTableProvider,
) -> Result<impl Stream<Item = Result<RecordBatch>>> {
let metadata_table = provider.table().inspect();
let batches = match provider.metadata_type() {
MetadataTableType::Snapshots => metadata_table.snapshots().scan().await,
MetadataTableType::Manifests => metadata_table.manifests().scan().await,
MetadataTableType::History => metadata_table.history().scan().await,
}
.map_err(to_datafusion_error)?;
Ok(batches.map_err(to_datafusion_error))
}
15 changes: 0 additions & 15 deletions crates/datafusion/src/metadata/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,13 @@
use std::sync::Arc;

use async_trait::async_trait;
use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::catalog::Session;
use datafusion::datasource::source::DataSourceExec;
use datafusion::datasource::{TableProvider, TableType};
use datafusion::error::Result;
use datafusion::logical_expr::Expr;
use datafusion::physical_plan::ExecutionPlan;
use futures::TryStreamExt;
use futures::stream::BoxStream;
use iceberg::arrow::schema_to_arrow_schema;
use iceberg::inspect::MetadataTableType;
use iceberg::table::Table;
Expand Down Expand Up @@ -99,16 +96,4 @@ impl IcebergMetadataTableProvider {
pub fn metadata_type(&self) -> &MetadataTableType {
&self.r#type
}

pub async fn scan(self) -> Result<BoxStream<'static, Result<RecordBatch>>> {
let metadata_table = self.table.inspect();
let stream = match self.r#type {
MetadataTableType::Snapshots => metadata_table.snapshots().scan().await,
MetadataTableType::Manifests => metadata_table.manifests().scan().await,
MetadataTableType::History => metadata_table.history().scan().await,
}
.map_err(to_datafusion_error)?;
let stream = stream.map_err(to_datafusion_error);
Ok(Box::pin(stream))
}
}
28 changes: 28 additions & 0 deletions crates/datafusion/tests/metadata_tables.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,34 @@ mod tests {
Ok(())
}

#[tokio::test]
async fn reads_the_history_table() -> Result<()> {
let table = create_taxi_table().await?;
let catalog = IcebergCatalogProvider::try_new(table.catalog()).await?;

// Leaves out `made_current_at`, the commit time, which changes on every run.
let (plan, results) = IcebergTestHarness::builder()
.with_catalog("iceberg", Arc::new(catalog))
.query(
"SELECT snapshot_id, parent_id, is_current_ancestor \
FROM iceberg.default.taxi$history",
)
.await?;
insta::assert_snapshot!(plan, @"
CooperativeExec
DataSourceExec: format=iceberg_metadata
");
insta::assert_snapshot!(results, @"
+-------------+-----------+---------------------+
| snapshot_id | parent_id | is_current_ancestor |
+-------------+-----------+---------------------+
| 1 | | true |
+-------------+-----------+---------------------+
");

Ok(())
}

#[tokio::test]
async fn test_metadata_table() -> Result<(), Box<dyn Error>> {
let fixtures = FixtureCatalog::try_new().await?;
Expand Down