diff --git a/crates/datafusion/src/metadata/scan.rs b/crates/datafusion/src/metadata/scan.rs index 37eb689..928b112 100644 --- a/crates/datafusion/src/metadata/scan.rs +++ b/crates/datafusion/src/metadata/scan.rs @@ -17,6 +17,7 @@ use std::sync::Arc; +use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::Statistics; use datafusion::common::tree_node::TreeNodeRecursion; @@ -28,9 +29,11 @@ use datafusion::physical_expr::projection::ProjectionExprs; use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayFormatType, Partitioning}; -use futures::{StreamExt, TryStreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; +use iceberg::inspect::MetadataTableType; use super::table::IcebergMetadataTableProvider; +use crate::to_datafusion_error; /// Reads rows from an Iceberg metadata table through DataFusion's `DataSource` API. #[derive(Debug, Clone)] @@ -69,7 +72,7 @@ impl DataSource for IcebergMetadataDataSource { _partition: usize, _context: Arc, ) -> Result { - let fut = self.provider.clone().scan(); + let fut = read_all(self.provider.clone()); let projection = self.projection.clone(); // TODO: Push these projections down into the scan layer instead of manually iterating over the result set @@ -133,3 +136,17 @@ impl DataSource for IcebergMetadataDataSource { Ok(TreeNodeRecursion::Continue) } } + +/// Reads every row of `provider`'s metadata table. +async fn read_all( + provider: IcebergMetadataTableProvider, +) -> Result>> { + let metadata_table = provider.table().inspect(); + let batches = match provider.metadata_type() { + MetadataTableType::Snapshots => metadata_table.snapshots().scan().await, + MetadataTableType::Manifests => metadata_table.manifests().scan().await, + MetadataTableType::History => metadata_table.history().scan().await, + } + .map_err(to_datafusion_error)?; + Ok(batches.map_err(to_datafusion_error)) +} diff --git a/crates/datafusion/src/metadata/table.rs b/crates/datafusion/src/metadata/table.rs index 73e0ee3..2a6a36f 100644 --- a/crates/datafusion/src/metadata/table.rs +++ b/crates/datafusion/src/metadata/table.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use async_trait::async_trait; -use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::catalog::Session; use datafusion::datasource::source::DataSourceExec; @@ -26,8 +25,6 @@ use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::Result; use datafusion::logical_expr::Expr; use datafusion::physical_plan::ExecutionPlan; -use futures::TryStreamExt; -use futures::stream::BoxStream; use iceberg::arrow::schema_to_arrow_schema; use iceberg::inspect::MetadataTableType; use iceberg::table::Table; @@ -99,16 +96,4 @@ impl IcebergMetadataTableProvider { pub fn metadata_type(&self) -> &MetadataTableType { &self.r#type } - - pub async fn scan(self) -> Result>> { - let metadata_table = self.table.inspect(); - let stream = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().scan().await, - MetadataTableType::Manifests => metadata_table.manifests().scan().await, - MetadataTableType::History => metadata_table.history().scan().await, - } - .map_err(to_datafusion_error)?; - let stream = stream.map_err(to_datafusion_error); - Ok(Box::pin(stream)) - } } diff --git a/crates/datafusion/tests/metadata_tables.rs b/crates/datafusion/tests/metadata_tables.rs index efec255..4adca87 100644 --- a/crates/datafusion/tests/metadata_tables.rs +++ b/crates/datafusion/tests/metadata_tables.rs @@ -96,6 +96,34 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reads_the_history_table() -> Result<()> { + let table = create_taxi_table().await?; + let catalog = IcebergCatalogProvider::try_new(table.catalog()).await?; + + // Leaves out `made_current_at`, the commit time, which changes on every run. + let (plan, results) = IcebergTestHarness::builder() + .with_catalog("iceberg", Arc::new(catalog)) + .query( + "SELECT snapshot_id, parent_id, is_current_ancestor \ + FROM iceberg.default.taxi$history", + ) + .await?; + insta::assert_snapshot!(plan, @" + CooperativeExec + DataSourceExec: format=iceberg_metadata + "); + insta::assert_snapshot!(results, @" + +-------------+-----------+---------------------+ + | snapshot_id | parent_id | is_current_ancestor | + +-------------+-----------+---------------------+ + | 1 | | true | + +-------------+-----------+---------------------+ + "); + + Ok(()) + } + #[tokio::test] async fn test_metadata_table() -> Result<(), Box> { let fixtures = FixtureCatalog::try_new().await?;