diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index a3259fd..dbf7915 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -45,10 +45,9 @@ pub struct IcebergDataSource { table: Table, /// Snapshot of the table to scan. snapshot_id: Option, - /// Schema returned by this data source after projection. + /// Schema returned by this data source after projection. Its columns are + /// read from Iceberg by name. schema: ArrowSchemaRef, - /// Projection column names in the output schema. - projection: Vec, /// Filter to apply to the table scan predicate: Option, /// Optional limit on the number of rows to return @@ -82,10 +81,6 @@ impl IcebergDataSource { self.schema.clone() } - pub fn projection(&self) -> &[String] { - &self.projection - } - pub fn predicate(&self) -> Option<&Predicate> { self.predicate.as_ref() } @@ -93,6 +88,15 @@ impl IcebergDataSource { pub fn fetch(&self) -> Option { self.fetch } + + /// The names of the columns this source reads, in output order. + fn column_names(&self) -> Vec { + self.schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect() + } } /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`]. @@ -155,16 +159,10 @@ impl IcebergDataSourceBuilder { None => self.schema, Some(projection) => Arc::new(self.schema.project(projection)?), }; - let projection = schema - .fields() - .iter() - .map(|field| field.name().clone()) - .collect(); Ok(IcebergDataSource { table: self.table, snapshot_id: self.snapshot_id, schema, - projection, predicate: self.predicate, fetch: self.fetch, }) @@ -187,7 +185,7 @@ impl DataSource for IcebergDataSource { let fut = get_batch_stream( self.table.clone(), self.snapshot_id, - Some(self.projection.clone()), + Some(self.column_names()), self.predicate.clone(), context .session_config() @@ -231,7 +229,7 @@ impl DataSource for IcebergDataSource { f: &mut std::fmt::Formatter, ) -> std::fmt::Result { write!(f, "format=iceberg",)?; - write!(f, ", projection=[{}]", self.projection.join(", "))?; + write!(f, ", projection=[{}]", self.column_names().join(", "))?; if let Some(predicate) = &self.predicate { write!(f, ", predicate={predicate}")?; } diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index b804f56..22e51b4 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -459,7 +459,9 @@ mod tests { .with_projection(Some(&vec![1, 0])) .build() .unwrap(); - insta::assert_debug_snapshot!(source.projection(), @r#" + let projected = source.schema(); + let names: Vec<_> = projected.fields().iter().map(|f| f.name()).collect(); + insta::assert_debug_snapshot!(names, @r#" [ "name", "id", diff --git a/crates/datafusion/tests/integration_datafusion_test.rs b/crates/datafusion/tests/integration_datafusion_test.rs index 42d350a..08b8d40 100644 --- a/crates/datafusion/tests/integration_datafusion_test.rs +++ b/crates/datafusion/tests/integration_datafusion_test.rs @@ -1041,7 +1041,6 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { ); let rebuilt = rebuild_scan(scan); assert_eq!(rebuilt.schema(), scan.schema()); - assert_eq!(rebuilt.projection(), scan.projection()); let expected = run_source(scan, &ctx).await?; insta::assert_snapshot!(&expected, @r#" +------+------+ @@ -1061,7 +1060,7 @@ async fn test_plan_nodes_are_inspectable() -> Result<(), Box> { .data_source() .downcast_ref::() .expect("an Iceberg data source"); - assert_eq!(scan.projection(), ["foo1".to_string(), "foo2".to_string()]); + assert_eq!(scan.schema(), pinned.schema()); let rebuilt = rebuild_scan(scan); assert_eq!(rebuilt.fetch(), Some(1)); let expected = run_source(scan, &ctx).await?;