From 61e91b241fc4a3c1c4cc5e90e9523a67531a10ae Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 17 Nov 2026 09:31:02 -0500 Subject: [PATCH 1/2] refactor: rename IcebergDataSourceBuilder::build to try_build and drop projection - Rename the fallible IcebergDataSourceBuilder::build to try_build. - Remove IcebergDataSource's projection field and accessor. They always held the field names of its schema, which the source now derives when it needs them. --- crates/datafusion/src/read/scan.rs | 30 +++++++++---------- .../table_provider/catalog_table_provider.rs | 10 ++++--- .../table_provider/static_table_provider.rs | 2 +- .../tests/integration_datafusion_test.rs | 7 ++--- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index a3259fd..b669175 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -45,10 +45,9 @@ pub struct IcebergDataSource { table: Table, /// Snapshot of the table to scan. snapshot_id: Option, - /// Schema returned by this data source after projection. + /// Schema returned by this data source after projection. Its columns are + /// read from Iceberg by name. schema: ArrowSchemaRef, - /// Projection column names in the output schema. - projection: Vec, /// Filter to apply to the table scan predicate: Option, /// Optional limit on the number of rows to return @@ -82,10 +81,6 @@ impl IcebergDataSource { self.schema.clone() } - pub fn projection(&self) -> &[String] { - &self.projection - } - pub fn predicate(&self) -> Option<&Predicate> { self.predicate.as_ref() } @@ -93,6 +88,15 @@ impl IcebergDataSource { pub fn fetch(&self) -> Option { self.fetch } + + /// The names of the columns this source reads, in output order. + fn column_names(&self) -> Vec { + self.schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect() + } } /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`]. @@ -150,21 +154,15 @@ impl IcebergDataSourceBuilder { /// # Errors /// /// Returns an error if the projection is out of bounds for the schema. - pub fn build(self) -> Result { + pub fn try_build(self) -> Result { let schema = match &self.projection { None => self.schema, Some(projection) => Arc::new(self.schema.project(projection)?), }; - let projection = schema - .fields() - .iter() - .map(|field| field.name().clone()) - .collect(); Ok(IcebergDataSource { table: self.table, snapshot_id: self.snapshot_id, schema, - projection, predicate: self.predicate, fetch: self.fetch, }) @@ -187,7 +185,7 @@ impl DataSource for IcebergDataSource { let fut = get_batch_stream( self.table.clone(), self.snapshot_id, - Some(self.projection.clone()), + Some(self.column_names()), self.predicate.clone(), context .session_config() @@ -231,7 +229,7 @@ impl DataSource for IcebergDataSource { f: &mut std::fmt::Formatter, ) -> std::fmt::Result { write!(f, "format=iceberg",)?; - write!(f, ", projection=[{}]", self.projection.join(", "))?; + write!(f, ", projection=[{}]", self.column_names().join(", "))?; if let Some(predicate) = &self.predicate { write!(f, ", predicate={predicate}")?; } diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index b804f56..ca45152 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -143,7 +143,7 @@ impl TableProvider for IcebergCatalogTableProvider { .with_projection(projection) .with_filters(filters) .with_fetch(limit) - .build()?; + .try_build()?; Ok(DataSourceExec::from_data_source(source)) } @@ -457,9 +457,11 @@ mod tests { // Columns come back in the projection's order, not the table's. let source = IcebergDataSource::builder(table.clone(), schema.clone()) .with_projection(Some(&vec![1, 0])) - .build() + .try_build() .unwrap(); - insta::assert_debug_snapshot!(source.projection(), @r#" + let projected = source.schema(); + let names: Vec<_> = projected.fields().iter().map(|f| f.name()).collect(); + insta::assert_debug_snapshot!(names, @r#" [ "name", "id", @@ -468,7 +470,7 @@ mod tests { let err = IcebergDataSource::builder(table, schema) .with_projection(Some(&vec![0, out_of_range])) - .build() + .try_build() .unwrap_err(); assert_eq!( err.to_string(), diff --git a/crates/datafusion/src/table_provider/static_table_provider.rs b/crates/datafusion/src/table_provider/static_table_provider.rs index 5ebabc8..9dbd98d 100644 --- a/crates/datafusion/src/table_provider/static_table_provider.rs +++ b/crates/datafusion/src/table_provider/static_table_provider.rs @@ -131,7 +131,7 @@ impl TableProvider for IcebergStaticTableProvider { .with_projection(projection) .with_filters(filters) .with_fetch(limit) - .build()?; + .try_build()?; Ok(DataSourceExec::from_data_source(source)) } diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs index 42d350a..209f768 100644 --- a/crates/datafusion/tests/integration_datafusion_test.rs +++ b/crates/datafusion/tests/integration_datafusion_test.rs @@ -920,7 +920,7 @@ fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource { .with_snapshot_id(scan.snapshot_id()) .with_predicate(scan.predicate().cloned()) .with_fetch(scan.fetch()) - .build() + .try_build() .unwrap() } @@ -1041,7 +1041,6 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { ); let rebuilt = rebuild_scan(scan); assert_eq!(rebuilt.schema(), scan.schema()); - assert_eq!(rebuilt.projection(), scan.projection()); let expected = run_source(scan, &ctx).await?; insta::assert_snapshot!(&expected, @r#" +------+------+ @@ -1061,7 +1060,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { .data_source() .downcast_ref::() .expect("an Iceberg data source"); - assert_eq!(scan.projection(), ["foo1".to_string(), "foo2".to_string()]); + assert_eq!(scan.schema(), pinned.schema()); let rebuilt = rebuild_scan(scan); assert_eq!(rebuilt.fetch(), Some(1)); let expected = run_source(scan, &ctx).await?; @@ -1078,7 +1077,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { let foo2_only = Arc::new(pinned.schema().project(&[1])?); let partial = IcebergDataSource::builder(pinned.table().clone(), foo2_only) .with_snapshot_id(Some(snapshot_id)) - .build()?; + .try_build()?; insta::assert_snapshot!(&run_source(&partial, &ctx).await?, @r#" +--------+ | foo2 | From 240d239bedd2dee75a285b1c9888e1ed95d02d12 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 17 Nov 2026 22:33:00 -0500 Subject: [PATCH 2/2] refactor: keep IcebergDataSourceBuilder::build A builder whose only terminal method can fail calls it build() and returns Result, as DataFusion's and iceberg's builders do. --- crates/datafusion/src/read/scan.rs | 2 +- .../datafusion/src/table_provider/catalog_table_provider.rs | 6 +++--- .../datafusion/src/table_provider/static_table_provider.rs | 2 +- crates/datafusion/tests/integration_datafusion_test.rs | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index b669175..dbf7915 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -154,7 +154,7 @@ impl IcebergDataSourceBuilder { /// # Errors /// /// Returns an error if the projection is out of bounds for the schema. - pub fn try_build(self) -> Result { + pub fn build(self) -> Result { let schema = match &self.projection { None => self.schema, Some(projection) => Arc::new(self.schema.project(projection)?), diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index ca45152..22e51b4 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -143,7 +143,7 @@ impl TableProvider for IcebergCatalogTableProvider { .with_projection(projection) .with_filters(filters) .with_fetch(limit) - .try_build()?; + .build()?; Ok(DataSourceExec::from_data_source(source)) } @@ -457,7 +457,7 @@ mod tests { // Columns come back in the projection's order, not the table's. let source = IcebergDataSource::builder(table.clone(), schema.clone()) .with_projection(Some(&vec![1, 0])) - .try_build() + .build() .unwrap(); let projected = source.schema(); let names: Vec<_> = projected.fields().iter().map(|f| f.name()).collect(); @@ -470,7 +470,7 @@ mod tests { let err = IcebergDataSource::builder(table, schema) .with_projection(Some(&vec![0, out_of_range])) - .try_build() + .build() .unwrap_err(); assert_eq!( err.to_string(), diff --git a/crates/datafusion/src/table_provider/static_table_provider.rs b/crates/datafusion/src/table_provider/static_table_provider.rs index 9dbd98d..5ebabc8 100644 --- a/crates/datafusion/src/table_provider/static_table_provider.rs +++ b/crates/datafusion/src/table_provider/static_table_provider.rs @@ -131,7 +131,7 @@ impl TableProvider for IcebergStaticTableProvider { .with_projection(projection) .with_filters(filters) .with_fetch(limit) - .try_build()?; + .build()?; Ok(DataSourceExec::from_data_source(source)) } diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs index 209f768..08b8d40 100644 --- a/crates/datafusion/tests/integration_datafusion_test.rs +++ b/crates/datafusion/tests/integration_datafusion_test.rs @@ -920,7 +920,7 @@ fn rebuild_scan(scan: &IcebergDataSource) -> IcebergDataSource { .with_snapshot_id(scan.snapshot_id()) .with_predicate(scan.predicate().cloned()) .with_fetch(scan.fetch()) - .try_build() + .build() .unwrap() } @@ -1077,7 +1077,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { let foo2_only = Arc::new(pinned.schema().project(&[1])?); let partial = IcebergDataSource::builder(pinned.table().clone(), foo2_only) .with_snapshot_id(Some(snapshot_id)) - .try_build()?; + .build()?; insta::assert_snapshot!(&run_source(&partial, &ctx).await?, @r#" +--------+ | foo2 |