From 85f6b1bbeab9002e682782e4fa4f594f43b4abb9 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Nov 2026 16:35:49 -0500 Subject: [PATCH 1/3] feat: report snapshot row count statistics to DataFusion --- crates/datafusion/src/read/scan.rs | 251 +++++++++++++++++- crates/datafusion/tests/statistics.rs | 44 +++ .../df_test/binary_predicate_pushdown.slt | 2 +- 3 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 crates/datafusion/tests/statistics.rs diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index a3259fd..feadc99 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; use datafusion::common::Statistics; +use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::datasource::source::DataSource; use datafusion::error::Result; @@ -37,6 +38,10 @@ use iceberg::table::Table; use super::predicate::convert_filters_to_predicate; use crate::to_datafusion_error; +/// Snapshot summary fields from the spec's "Optional Snapshot Summary Fields". +const TOTAL_RECORDS: &str = "total-records"; +const TOTAL_POSITION_DELETES: &str = "total-position-deletes"; + /// Manages the scanning process of an Iceberg [`Table`], encapsulating the /// necessary details and computed properties required for execution planning. #[derive(Debug, Clone)] @@ -93,6 +98,38 @@ impl IcebergDataSource { pub fn fetch(&self) -> Option { self.fetch } + + /// Estimates the rows the scan returns before its predicate and fetch, + /// from the snapshot summary. + /// + /// `total-records` counts rows before deletes. A position delete removes + /// at most one row, so position deletes are subtracted. An equality delete + /// can remove any number of rows, so equality deletes are ignored. Summary + /// fields are optional, so a missing or invalid one leaves the estimate + /// unknown rather than failing the query. + fn estimated_num_rows(&self) -> Precision { + let metadata = self.table.metadata(); + let snapshot = match self.snapshot_id { + Some(snapshot_id) => metadata.snapshot_by_id(snapshot_id), + None => match metadata.current_snapshot() { + Some(snapshot) => Some(snapshot), + // The scan of a table with no current snapshot reads nothing. + None => return Precision::Exact(0), + }, + }; + // The scan fails on a snapshot id that doesn't exist. + let Some(snapshot) = snapshot else { + return Precision::Absent; + }; + let summary = &snapshot.summary().additional_properties; + let count = |key| summary.get(key)?.parse::().ok(); + match count(TOTAL_RECORDS) { + Some(records) => Precision::Inexact( + records.saturating_sub(count(TOTAL_POSITION_DELETES).unwrap_or(0)), + ), + None => Precision::Absent, + } + } } /// Builds an [`IcebergDataSource`]. Created by [`IcebergDataSource::builder`]. @@ -250,7 +287,13 @@ impl DataSource for IcebergDataSource { } fn partition_statistics(&self, _partition: Option) -> Result> { - Ok(Arc::new(Statistics::new_unknown(&self.schema))) + let mut statistics = Statistics::new_unknown(&self.schema); + statistics.num_rows = self.estimated_num_rows(); + // Iceberg's `total-files-size` is compressed and covers every column + // and delete file, so estimate the in-memory size of the projected + // columns instead, as DataFusion's Parquet source does. + statistics.calculate_total_byte_size(&self.schema); + Ok(Arc::new(statistics.with_fetch(self.fetch, 0, 1)?)) } fn with_fetch(&self, fetch: Option) -> Option> { @@ -309,3 +352,209 @@ async fn get_batch_stream( .map_err(to_datafusion_error); Ok(Box::pin(stream)) } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use std::time::{SystemTime, UNIX_EPOCH}; + + use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use datafusion::common::Statistics; + use datafusion::common::stats::Precision; + use datafusion::datasource::source::DataSource; + use iceberg::io::FileIO; + use iceberg::spec::{ + FormatVersion, MAIN_BRANCH, NestedField, Operation, PartitionSpec, PrimitiveType, + Schema, Snapshot, SortOrder, Summary, TableMetadataBuilder, Type, + }; + use iceberg::table::Table; + use iceberg::{Runtime, TableIdent}; + + use super::IcebergDataSource; + + /// Returns a table with one snapshot per summary, oldest first, with ids + /// counting up from 1. The last snapshot is the current one. + fn table_with_snapshot_summaries(summaries: &[&[(&str, &str)]]) -> Table { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)) + .into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(); + let partition_spec = PartitionSpec::builder(schema.clone()).build().unwrap(); + let sort_order = SortOrder::builder().build(&schema).unwrap(); + let mut builder = TableMetadataBuilder::new( + schema, + partition_spec, + sort_order, + "/test/table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap(); + let now_ms = i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + for (snapshot_id, summary) in (1..).zip(summaries) { + let snapshot = Snapshot::builder() + .with_snapshot_id(snapshot_id) + .with_parent_snapshot_id((snapshot_id > 1).then_some(snapshot_id - 1)) + .with_sequence_number(snapshot_id) + .with_timestamp_ms(now_ms + snapshot_id) + .with_manifest_list(format!( + "/test/table/metadata/snap-{snapshot_id}.avro" + )) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: summary + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(), + }) + .with_schema_id(0) + .build(); + builder = builder.set_branch_snapshot(snapshot, MAIN_BRANCH).unwrap(); + } + + Table::builder() + .metadata(builder.build().unwrap().metadata) + .identifier(TableIdent::from_strs(["test", "table"]).unwrap()) + .file_io(FileIO::new_with_fs()) + .metadata_location("/test/metadata.json") + .runtime(Runtime::current()) + .build() + .unwrap() + } + + /// Statistics of a scan of `table`'s fixed-width `id` column. + fn id_scan_statistics( + table: Table, + snapshot_id: Option, + fetch: Option, + ) -> Statistics { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ])); + let source = IcebergDataSource::builder(table, schema) + .with_projection(Some(&vec![0])) + .with_snapshot_id(snapshot_id) + .with_fetch(fetch) + .build() + .unwrap(); + Arc::unwrap_or_clone(source.partition_statistics(None).unwrap()) + } + + #[tokio::test] + async fn reports_no_rows_for_a_table_without_snapshots() { + let statistics = + id_scan_statistics(table_with_snapshot_summaries(&[]), None, None); + assert_eq!(statistics.num_rows, Precision::Exact(0)); + assert_eq!(statistics.total_byte_size, Precision::Exact(0)); + } + + #[tokio::test] + async fn estimates_rows_from_the_snapshot_summary() { + let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!(statistics.num_rows, Precision::Inexact(100)); + assert_eq!(statistics.total_byte_size, Precision::Inexact(800)); + } + + #[tokio::test] + async fn subtracts_position_deletes_but_not_equality_deletes() { + let table = table_with_snapshot_summaries(&[&[ + ("total-records", "100"), + ("total-position-deletes", "30"), + ("total-equality-deletes", "50"), + ]]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!(statistics.num_rows, Precision::Inexact(70)); + } + + #[tokio::test] + async fn estimates_no_rows_when_position_deletes_exceed_records() { + let table = table_with_snapshot_summaries(&[&[ + ("total-records", "10"), + ("total-position-deletes", "20"), + ]]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!(statistics.num_rows, Precision::Inexact(0)); + } + + #[tokio::test] + async fn ignores_an_invalid_position_delete_count() { + let table = table_with_snapshot_summaries(&[&[ + ("total-records", "100"), + ("total-position-deletes", "-1"), + ]]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!(statistics.num_rows, Precision::Inexact(100)); + } + + #[tokio::test] + async fn reports_unknown_rows_without_a_valid_record_count() { + for summary in [ + &[][..], + &[("total-records", "-1")], + &[("total-records", "many")], + ] { + let table = table_with_snapshot_summaries(&[summary]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!(statistics.num_rows, Precision::Absent, "{summary:?}"); + assert_eq!(statistics.total_byte_size, Precision::Absent, "{summary:?}"); + } + } + + #[tokio::test] + async fn reports_unknown_rows_for_a_missing_snapshot() { + let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); + let statistics = id_scan_statistics(table, Some(2), None); + assert_eq!(statistics.num_rows, Precision::Absent); + } + + #[tokio::test] + async fn uses_the_summary_of_the_scanned_snapshot() { + let table = table_with_snapshot_summaries(&[ + &[("total-records", "100")], + &[("total-records", "200")], + ]); + assert_eq!( + id_scan_statistics(table.clone(), Some(1), None).num_rows, + Precision::Inexact(100) + ); + assert_eq!( + id_scan_statistics(table, None, None).num_rows, + Precision::Inexact(200) + ); + } + + #[tokio::test] + async fn caps_rows_at_the_fetch() { + let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); + let statistics = id_scan_statistics(table, None, Some(10)); + assert_eq!(statistics.num_rows, Precision::Inexact(10)); + assert_eq!(statistics.total_byte_size, Precision::Inexact(80)); + } + + #[tokio::test] + async fn omits_the_byte_size_of_variable_width_columns() { + let table = table_with_snapshot_summaries(&[&[("total-records", "100")]]); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ])); + let source = IcebergDataSource::builder(table, schema).build().unwrap(); + let statistics = source.partition_statistics(None).unwrap(); + assert_eq!(statistics.num_rows, Precision::Inexact(100)); + assert_eq!(statistics.total_byte_size, Precision::Absent); + } +} diff --git a/crates/datafusion/tests/statistics.rs b/crates/datafusion/tests/statistics.rs new file mode 100644 index 0000000..ff531b7 --- /dev/null +++ b/crates/datafusion/tests/statistics.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[cfg(test)] +mod tests { + use datafusion::error::Result; + use iceberg_datafusion::test_utils::IcebergTestHarness; + + #[tokio::test] + async fn reports_the_snapshot_row_count_to_datafusion() -> Result<()> { + let harness = IcebergTestHarness::new().await?; + harness + .query("SET datafusion.explain.show_statistics = true") + .await?; + let (_, explain) = harness.query("EXPLAIN SELECT vendor_id FROM taxi").await?; + + insta::assert_snapshot!(explain, @r" + +---------------+---------------------------------------------------------------------------------------------------------------------------------+ + | plan_type | plan | + +---------------+---------------------------------------------------------------------------------------------------------------------------------+ + | logical_plan | TableScan: taxi projection=[vendor_id] | + | physical_plan | CooperativeExec, statistics=[Rows=Inexact(175000), Bytes=Inexact(700000), [(Col[0]:)]] | + | | DataSourceExec: format=iceberg, projection=[vendor_id], statistics=[Rows=Inexact(175000), Bytes=Inexact(700000), [(Col[0]:)]] | + | | | + +---------------+---------------------------------------------------------------------------------------------------------------------------------+ + "); + + Ok(()) + } +} diff --git a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt index 7c725ee..ebe6f50 100644 --- a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt @@ -27,7 +27,7 @@ logical_plan 02)--TableScan: default.default.test_binary_table projection=[id, data], partial_filters=[default.default.test_binary_table.data = LargeBinary("1,2")] physical_plan 01)FilterExec: data@1 = 0102 -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +02)--CooperativeExec 03)----DataSourceExec: format=iceberg, projection=[id, data], predicate=data = 0102 # Verify empty result from empty table From 3ef58a1b7085426003134f9f903bd0a017882960 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Tue, 17 Nov 2026 16:39:42 -0500 Subject: [PATCH 2/3] address review: clarify statistics docs, deterministic test timestamps --- crates/datafusion/src/read/scan.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index feadc99..a071f96 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -105,8 +105,9 @@ impl IcebergDataSource { /// `total-records` counts rows before deletes. A position delete removes /// at most one row, so position deletes are subtracted. An equality delete /// can remove any number of rows, so equality deletes are ignored. Summary - /// fields are optional, so a missing or invalid one leaves the estimate - /// unknown rather than failing the query. + /// fields are optional. A missing or invalid `total-records` leaves the + /// estimate unknown, and a missing or invalid `total-position-deletes` + /// counts as zero, rather than failing the query. fn estimated_num_rows(&self) -> Precision { let metadata = self.table.metadata(); let snapshot = match self.snapshot_id { @@ -286,12 +287,15 @@ impl DataSource for IcebergDataSource { EquivalenceProperties::new(self.schema.clone()) } + // The source has one output partition (see `output_partitioning`), so a + // partition's statistics are the scan's. fn partition_statistics(&self, _partition: Option) -> Result> { let mut statistics = Statistics::new_unknown(&self.schema); statistics.num_rows = self.estimated_num_rows(); // Iceberg's `total-files-size` is compressed and covers every column // and delete file, so estimate the in-memory size of the projected - // columns instead, as DataFusion's Parquet source does. + // columns from their widths, as DataFusion's Parquet source does for + // fixed-width columns. statistics.calculate_total_byte_size(&self.schema); Ok(Arc::new(statistics.with_fetch(self.fetch, 0, 1)?)) } @@ -357,7 +361,6 @@ async fn get_batch_stream( mod tests { use std::collections::HashMap; use std::sync::Arc; - use std::time::{SystemTime, UNIX_EPOCH}; use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; use datafusion::common::Statistics; @@ -396,19 +399,12 @@ mod tests { HashMap::new(), ) .unwrap(); - let now_ms = i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis(), - ) - .unwrap(); for (snapshot_id, summary) in (1..).zip(summaries) { let snapshot = Snapshot::builder() .with_snapshot_id(snapshot_id) .with_parent_snapshot_id((snapshot_id > 1).then_some(snapshot_id - 1)) .with_sequence_number(snapshot_id) - .with_timestamp_ms(now_ms + snapshot_id) + .with_timestamp_ms(1_700_000_000_000 + snapshot_id) .with_manifest_list(format!( "/test/table/metadata/snap-{snapshot_id}.avro" )) @@ -506,6 +502,8 @@ mod tests { &[][..], &[("total-records", "-1")], &[("total-records", "many")], + // u64::MAX + 1 + &[("total-records", "18446744073709551616")], ] { let table = table_with_snapshot_summaries(&[summary]); let statistics = id_scan_statistics(table, None, None); From 13b6b3e4a0d5765c778ec008f0105fdd602d5088 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 18 Nov 2026 11:48:57 -0500 Subject: [PATCH 3/3] address PR feedback --- crates/datafusion/src/read/scan.rs | 73 +++++++++++++----------------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/crates/datafusion/src/read/scan.rs b/crates/datafusion/src/read/scan.rs index a071f96..934d744 100644 --- a/crates/datafusion/src/read/scan.rs +++ b/crates/datafusion/src/read/scan.rs @@ -38,9 +38,8 @@ use iceberg::table::Table; use super::predicate::convert_filters_to_predicate; use crate::to_datafusion_error; -/// Snapshot summary fields from the spec's "Optional Snapshot Summary Fields". +/// Snapshot summary field from the spec's "Optional Snapshot Summary Fields". const TOTAL_RECORDS: &str = "total-records"; -const TOTAL_POSITION_DELETES: &str = "total-position-deletes"; /// Manages the scanning process of an Iceberg [`Table`], encapsulating the /// necessary details and computed properties required for execution planning. @@ -102,12 +101,15 @@ impl IcebergDataSource { /// Estimates the rows the scan returns before its predicate and fetch, /// from the snapshot summary. /// - /// `total-records` counts rows before deletes. A position delete removes - /// at most one row, so position deletes are subtracted. An equality delete - /// can remove any number of rows, so equality deletes are ignored. Summary - /// fields are optional. A missing or invalid `total-records` leaves the - /// estimate unknown, and a missing or invalid `total-position-deletes` - /// counts as zero, rather than failing the query. + /// `total-records` counts rows before deletes, so the estimate is an upper + /// bound. Deletes aren't subtracted, because the spec lets + /// `total-position-deletes` count more rows than the deletes remove. A + /// position delete file can stay live after the data files it references + /// are removed. A deletion vector repeats the deletes of position delete + /// files that can stay live beside it. An underestimate can make join + /// selection collect a large table as the build side. Summary fields are + /// optional, so a missing or invalid `total-records` leaves the estimate + /// unknown rather than failing the query. fn estimated_num_rows(&self) -> Precision { let metadata = self.table.metadata(); let snapshot = match self.snapshot_id { @@ -122,12 +124,13 @@ impl IcebergDataSource { let Some(snapshot) = snapshot else { return Precision::Absent; }; - let summary = &snapshot.summary().additional_properties; - let count = |key| summary.get(key)?.parse::().ok(); - match count(TOTAL_RECORDS) { - Some(records) => Precision::Inexact( - records.saturating_sub(count(TOTAL_POSITION_DELETES).unwrap_or(0)), - ), + match snapshot + .summary() + .additional_properties + .get(TOTAL_RECORDS) + .and_then(|records| records.parse::().ok()) + { + Some(records) => Precision::Inexact(records), None => Precision::Absent, } } @@ -466,34 +469,20 @@ mod tests { } #[tokio::test] - async fn subtracts_position_deletes_but_not_equality_deletes() { - let table = table_with_snapshot_summaries(&[&[ - ("total-records", "100"), - ("total-position-deletes", "30"), - ("total-equality-deletes", "50"), - ]]); - let statistics = id_scan_statistics(table, None, None); - assert_eq!(statistics.num_rows, Precision::Inexact(70)); - } - - #[tokio::test] - async fn estimates_no_rows_when_position_deletes_exceed_records() { - let table = table_with_snapshot_summaries(&[&[ - ("total-records", "10"), - ("total-position-deletes", "20"), - ]]); - let statistics = id_scan_statistics(table, None, None); - assert_eq!(statistics.num_rows, Precision::Inexact(0)); - } - - #[tokio::test] - async fn ignores_an_invalid_position_delete_count() { - let table = table_with_snapshot_summaries(&[&[ - ("total-records", "100"), - ("total-position-deletes", "-1"), - ]]); - let statistics = id_scan_statistics(table, None, None); - assert_eq!(statistics.num_rows, Precision::Inexact(100)); + async fn ignores_deletes() { + for (position_deletes, equality_deletes) in [("30", "50"), ("200", "0")] { + let table = table_with_snapshot_summaries(&[&[ + ("total-records", "100"), + ("total-position-deletes", position_deletes), + ("total-equality-deletes", equality_deletes), + ]]); + let statistics = id_scan_statistics(table, None, None); + assert_eq!( + statistics.num_rows, + Precision::Inexact(100), + "{position_deletes} position deletes" + ); + } } #[tokio::test]