diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..5c3ddfdbd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -96,7 +96,12 @@ return `PushedDown::No` for these filters, so DataFusion still checks every row. or aggregate only computes a dynamic filter when a scan below it is a consumer of that filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consumers of the dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as -`dynamic_filters=`. Scans don't skip data with these filters yet. +`dynamic_filters=`. Just before iceberg-rust's Parquet reader opens each file, the scan +checks the latest version of the filters: + +- It skips the file if the file's partition value can't match them. +- Otherwise, it gives the filters to the Parquet reader, which uses them to skip row groups, + and to drop the rows that don't match. ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs new file mode 100644 index 000000000..38a122662 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod partition_filter; +pub(crate) mod physical_predicate; +pub(crate) mod pruner; +pub(crate) mod row_group_filter; +pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs new file mode 100644 index 000000000..baaf5f1d6 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs @@ -0,0 +1,299 @@ +//! A copy of the code iceberg-rust uses to skip data files by their partition values. That +//! code isn't part of iceberg-rust's public API yet. When it becomes public, this file should +//! be deleted and its callers should use the public API. +//! Ref: https://github.com/apache/iceberg-rust/issues/3675 +//! +//! Each part of this file copies one part of iceberg-rust: +//! +//! | This file | iceberg-rust | +//! |------------------------------|--------------------------------------------------------------| +//! | `PartitionFilter::try_new` | `TableScanBuilder::with_filter`, `PartitionFilterCache::get` | +//! | `PartitionFilter::can_match` | `ExpressionEvaluator::eval` | +//! | `project`, `project_column` | `InclusiveProjection` | +//! | `evaluate` | `ExpressionEvaluatorVisitor` | +//! | `partition_value`, `datum` | `StructAccessor::get` | +//! +//! The copy works the same way as iceberg-rust, except for one case in `datum` that's +//! explained there. + +use std::sync::Arc; + +use iceberg::expr::{Bind, BoundPredicate, BoundReference, Predicate, PredicateOperator}; +use iceberg::spec::{ + Datum, Literal, PartitionSpec, PartitionSpecRef, PrimitiveLiteral, PrimitiveType, Schema, + SchemaRef, Struct, Type, +}; +use iceberg::{Error, ErrorKind, Result}; + +/// Checks whether a data file's partition can hold rows that match a filter, as a table scan +/// does when it plans files. +pub(crate) struct PartitionFilter { + /// The filter projected onto the spec's partition columns, bound to its partition schema. + predicate: BoundPredicate, + spec: PartitionSpecRef, +} + +impl PartitionFilter { + /// Creates a partition filter for `filter` on data files written with `partition_spec`. + /// + /// If `partition_spec` uses a column that `schema` no longer has, every partition matches. + /// Returns an error if `filter` doesn't bind to `schema`, or if it can't be projected onto + /// `partition_spec`. + pub(crate) fn try_new( + filter: Predicate, + schema: SchemaRef, + partition_spec: &PartitionSpecRef, + case_sensitive: bool, + ) -> Result { + let filter = filter + .rewrite_not() + .bind(Arc::clone(&schema), case_sensitive)?; + let has_dropped_source_column = partition_spec + .fields() + .iter() + .any(|field| schema.field_by_id(field.source_id).is_none()); + let predicate = if has_dropped_source_column { + BoundPredicate::AlwaysTrue + } else { + let partition_type = partition_spec.partition_type(&schema)?; + let partition_schema = Arc::new( + Schema::builder() + .with_schema_id(partition_spec.spec_id()) + .with_fields(partition_type.fields().to_owned()) + .build()?, + ); + project(&filter, partition_spec)? + .rewrite_not() + .bind(partition_schema, case_sensitive)? + }; + Ok(Self { + predicate, + spec: Arc::clone(partition_spec), + }) + } + + /// Returns whether a data file with `partition` can hold rows that match the filter. + /// Returns an error if `partition` doesn't match the partition spec's fields. + pub(crate) fn can_match(&self, partition: &Struct) -> Result { + evaluate(&self.predicate, &self.spec, partition) + } +} + +/// Rewrites a predicate on table columns into one on the spec's partition columns. The result +/// matches every partition that holds a matching row, and can match more. +fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> Result { + Ok(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + project(left, spec)?.and(project(right, spec)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + project(left, spec)?.or(project(right, spec)?) + } + // `PartitionFilter::try_new` removes every NOT first, so none should reach here. If one + // does, this returns an error, as iceberg-rust does. + BoundPredicate::Not(_) => { + return Err(Error::new( + ErrorKind::Unexpected, + "found a NOT that `PartitionFilter::try_new` should have removed", + )); + } + BoundPredicate::Unary(expr) => project_column(expr.term(), predicate, spec)?, + BoundPredicate::Binary(expr) => project_column(expr.term(), predicate, spec)?, + BoundPredicate::Set(expr) => project_column(expr.term(), predicate, spec)?, + }) +} + +/// Projects a predicate on one column through every partition field built from that column. +/// A column with no partition field gives `AlwaysTrue`. +fn project_column( + column: &BoundReference, + predicate: &BoundPredicate, + spec: &PartitionSpec, +) -> Result { + spec.fields() + .iter() + .filter(|field| field.source_id == column.field().id) + .try_fold(Predicate::AlwaysTrue, |projected, field| { + // `None` means the transform can't express the predicate, so it keeps every file. + Ok(match field.transform.project(&field.name, predicate)? { + Some(part) => projected.and(part), + None => projected, + }) + }) +} + +/// Checks whether one data file's partition values pass a filter on the partition columns. +fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct) -> Result { + match predicate { + BoundPredicate::AlwaysTrue => Ok(true), + BoundPredicate::AlwaysFalse => Ok(false), + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + let (left, right) = ( + evaluate(left, spec, partition)?, + evaluate(right, spec, partition)?, + ); + Ok(left && right) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + let (left, right) = ( + evaluate(left, spec, partition)?, + evaluate(right, spec, partition)?, + ); + Ok(left || right) + } + BoundPredicate::Not(_) => Err(Error::new( + ErrorKind::Unexpected, + "found a NOT that `PartitionFilter::try_new` should have removed", + )), + BoundPredicate::Unary(expr) => { + let value = partition_value(expr.term(), spec, partition)?; + let is_nan = value.as_ref().is_some_and(Datum::is_nan); + Ok(match expr.op() { + PredicateOperator::IsNull => value.is_none(), + PredicateOperator::NotNull => value.is_some(), + PredicateOperator::IsNan => is_nan, + PredicateOperator::NotNan => !is_nan, + _ => true, + }) + } + BoundPredicate::Binary(expr) => { + let Some(value) = partition_value(expr.term(), spec, partition)? else { + return Ok(matches!( + expr.op(), + PredicateOperator::NotEq | PredicateOperator::NotStartsWith + )); + }; + let literal = expr.literal(); + Ok(match expr.op() { + PredicateOperator::StartsWith => starts_with(&value, literal), + PredicateOperator::NotStartsWith => !starts_with(&value, literal), + PredicateOperator::LessThan => &value < literal, + PredicateOperator::LessThanOrEq => &value <= literal, + PredicateOperator::GreaterThan => &value > literal, + PredicateOperator::GreaterThanOrEq => &value >= literal, + PredicateOperator::Eq => &value == literal, + PredicateOperator::NotEq => &value != literal, + _ => true, + }) + } + BoundPredicate::Set(expr) => { + let Some(value) = partition_value(expr.term(), spec, partition)? else { + return Ok(matches!(expr.op(), PredicateOperator::NotIn)); + }; + let contains = expr.literals().contains(&value); + Ok(match expr.op() { + PredicateOperator::In => contains, + PredicateOperator::NotIn => !contains, + _ => true, + }) + } + } +} + +/// Returns the partition value that `column` points to, with the partition column's type. +/// Returns `None` when the value is null. +/// +/// iceberg-rust reads this value with `StructAccessor::get`, which isn't public. So this finds +/// the value's position by looking up its partition field ID in the spec. +fn partition_value( + column: &BoundReference, + spec: &PartitionSpec, + partition: &Struct, +) -> Result> { + let field = column.field(); + let position = spec + .fields() + .iter() + .position(|partition_field| partition_field.field_id == field.id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("partition field {} is not in the spec", field.id), + ) + })?; + let Some(value) = partition.fields().get(position) else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("the partition has no value at {position}"), + )); + }; + let Some(value) = value else { + return Ok(None); + }; + let (Literal::Primitive(literal), Type::Primitive(r#type)) = (value, field.field_type.as_ref()) + else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("partition field {} is not a primitive", field.id), + )); + }; + datum(r#type, literal).map(Some) +} + +/// Builds a `Datum` of type `r#type` from a stored partition value. iceberg-rust uses +/// `Datum::new` for this, which isn't public, so this uses the public constructors. +fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { + Ok(match (r#type, literal) { + (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => Datum::bool(*value), + (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => Datum::int(*value), + (PrimitiveType::Long, PrimitiveLiteral::Long(value)) => Datum::long(*value), + (PrimitiveType::Float, PrimitiveLiteral::Float(value)) => Datum::float(value.0), + (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => Datum::double(value.0), + (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => Datum::date(*value), + (PrimitiveType::Time, PrimitiveLiteral::Long(value)) => Datum::time_micros(*value)?, + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(value)) => { + Datum::timestamp_micros(*value) + } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(value)) => { + Datum::timestamptz_micros(*value) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(value)) => { + Datum::timestamp_nanos(*value) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(value)) => { + Datum::timestamptz_nanos(*value) + } + (PrimitiveType::String, PrimitiveLiteral::String(value)) => Datum::string(value), + (PrimitiveType::Binary, PrimitiveLiteral::Binary(value)) => { + Datum::binary(value.iter().copied()) + } + (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(value)) => { + Datum::fixed(value.iter().copied()) + } + // Decimal and uuid have no public constructor that takes the raw value, so these go + // through their byte encoding. + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(value)) => { + Datum::try_from_bytes(&value.to_be_bytes(), r#type.clone())? + } + (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => { + Datum::try_from_bytes(&value.to_be_bytes(), PrimitiveType::Uuid)? + } + // The stored value doesn't fit its column's type. This happens in files written before + // a column was widened, for example from int to long. iceberg-rust builds the value + // anyway. None of its comparisons then match, so it skips the file. The public + // constructors can't build a value like that, so we return an error and don't skip + // the file. + _ => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("can't read a {type} partition value"), + )); + } + }) +} + +/// Returns whether `value` starts with `prefix` +fn starts_with(value: &Datum, prefix: &Datum) -> bool { + match (value.literal(), prefix.literal()) { + (PrimitiveLiteral::String(value), PrimitiveLiteral::String(prefix)) => { + value.starts_with(prefix.as_str()) + } + _ => false, + } +} diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs new file mode 100644 index 000000000..a38adc9f0 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs @@ -0,0 +1,581 @@ +use std::sync::Arc; + +use datafusion::arrow::array::StructArray; +use datafusion::functions::core::r#struct::StructFunc; +use datafusion::logical_expr::Operator; +use datafusion::physical_expr::expressions::{ + BinaryExpr, CaseExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, Literal, +}; +use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; +use datafusion::scalar::ScalarValue; +use iceberg::expr::{Predicate, Reference}; +use iceberg::spec::{Datum, PrimitiveType}; + +/// Turns a DataFusion filter into an iceberg predicate. The predicate matches every row that the +/// filter would match, and *can* match more if the DF expression cannot be translated into an +/// Iceberg predicate completely. +/// +/// Returns `None` when no part of the filter can be converted. +pub(crate) fn convert_physical_filter_to_predicate( + filter: &Arc, +) -> Option { + convert(filter) +} + +/// Converts a given `PhysicalExpr` into an `iceberg::Predicate` +/// +/// We only convert the expressions DataFusion's dynamic filters use: comparisons, IN lists, +/// IS NULL, IS NOT NULL, AND, OR and CASE. +/// +/// Returns None if the expression could not be converted +fn convert(expr: &Arc) -> Option { + if let Some(binary) = expr.downcast_ref::() { + return convert_binary(binary); + } + if let Some(case) = expr.downcast_ref::() { + return convert_case(case); + } + if let Some(in_list) = expr.downcast_ref::() { + return convert_in_list(in_list); + } + if let Some(is_null) = expr.downcast_ref::() { + let column = column_name(is_null.arg())?; + return Some(Reference::new(column).is_null()); + } + if let Some(is_not_null) = expr.downcast_ref::() { + let column = column_name(is_not_null.arg())?; + return Some(Reference::new(column).is_not_null()); + } + // A bare `true`, `false` or NULL. NULL matches nothing. + match literal_value(expr)? { + ScalarValue::Boolean(Some(true)) => Some(Predicate::AlwaysTrue), + ScalarValue::Boolean(_) | ScalarValue::Null => Some(Predicate::AlwaysFalse), + _ => None, + } +} + +/// Converts a binary expression into an `iceberg::Predicate` +fn convert_binary(binary: &BinaryExpr) -> Option { + let (left, right) = (binary.left(), binary.right()); + match binary.op() { + // An AND expression can be converted if *either* side can be converted + // If one of the sides are dropped, iceberg would return extra rows + // which would then be filtered out by DataFusion + Operator::And => both(convert(left), convert(right)), + // An OR expression can only be converted if *both* sides can be converted + // We cannot drop one of the sides, as that would lead to data being skipped incorrectly + Operator::Or => Some(convert(left)?.or(convert(right)?)), + op => compare(left, *op, right), + } +} + +/// ANDs two converted sides. An AND can drop a side it can't convert: +/// `a = 1 AND f(b)` becomes `a = 1`. The result matches extra rows, which is fine. +fn both(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.and(right)), + (Some(side), None) | (None, Some(side)) => Some(side), + (None, None) => None, + } +} + +/// Converts a CaseExpr into an `iceberg::Predicate` +/// +/// In a CASE statement, each WHEN/THEN pair applies a filter over the rows. +/// +/// `CASE WHEN c1 THEN x WHEN c2 THEN y ELSE z END` becomes `(c1 AND x) OR (c2 AND y) OR z`. +/// It ignores that the second branch only applies when `c1` is false, so it can match +/// extra rows. +/// +/// `CASE a WHEN 1 THEN x ...` works the same way, with `a = 1` as the WHEN. +/// +/// A missing ELSE means NULL, which matches nothing. +fn convert_case(case: &CaseExpr) -> Option { + // This handles the two variants for CASE + // 1. CASE WHEN a = 1 THEN ... + // In this variant, `case.expr()` is `None` because the expression is imbued in the WHEN clause. + // We pass the WHEN clause to our general predicate converter for this. + // + // 2. CASE a WHEN 0 THEN ... + // In this variant, `case.expr()` is `Some(a)`, and each WHEN clause compares its own + // expression with this expression. + // To handle this, we convert this into a comparison predicate. + // Iceberg predicates can't always represent a given expression; if that is the case, we drop + // the predicate for the WHEN clause. + let convert_when = |when| match case.expr() { + Some(value) => compare(value, Operator::Eq, when), + None => convert(when), + }; + + case.when_then_expr() + .iter() + // Converts each individual WHEN/THEN pair into a conjunctive pair + // WHEN x > 5 THEN y < 3 becomes (x > 5 AND y < 3) + // If either side is dropped because it can't be converted, the filter is still valid + // and will just match more rows + .map(|(when, then)| both(convert_when(when), convert(then))) + // If there is an ELSE at the end, convert it into a predicate directly + .chain(case.else_expr().map(convert)) + .collect::>>()? + .into_iter() + // Combines all predicates in a disjunction, i.e, OR + // Multiple WHEN/THEN statements are joined together with ORs + // E.g: + // WHEN x > 5 THEN y ELSE + // WHEN x > 10 THEN z ELSE ... + // becomes + // (x > 5 AND y) OR (x > 10 AND z) + .reduce(Predicate::or) +} + +/// Converts a column <-> value comparison into an `iceberg::Predicate` +fn compare( + left: &Arc, + op: Operator, + right: &Arc, +) -> Option { + // Try to match ` ` first + // If unsuccessful, swap the order of the two inputs and check again + // If unsuccessful again, we can't convert this expression and we bail early + let (column, op, datum) = match (column_name(left), literal_datum(right)) { + (Some(column), Some(datum)) => (column, op, datum), + _ => (column_name(right)?, op.swap()?, literal_datum(left)?), + }; + + let column = Reference::new(column); + match op { + Operator::Eq => Some(column.equal_to(datum)), + Operator::NotEq => Some(column.not_equal_to(datum)), + Operator::Lt => Some(column.less_than(datum)), + Operator::LtEq => Some(column.less_than_or_equal_to(datum)), + Operator::Gt => Some(column.greater_than(datum)), + Operator::GtEq => Some(column.greater_than_or_equal_to(datum)), + _ => None, + } +} + +/// Converts an InList into an `iceberg::Predicate` +fn convert_in_list(in_list: &InListExpr) -> Option { + // Dynamic filters only build IN lists, so we don't convert NOT IN. + if in_list.negated() { + return None; + } + + // Something like `struct(a, b) IN (...)`, which checks several columns at once. + let Some(column) = column_name(in_list.expr()) else { + return convert_struct_in_list(in_list); + }; + + let literals = in_list + .list() + .iter() + // We expect every value in the list to be a literal, i.e, x IN (1, 2, 3) + // If it isn't, we can't convert this list, and bail early + .map(literal_value) + .collect::>>()? + .into_iter() + // NULL never equals anything, so `x IN (1, NULL)` matches the same rows as `x IN (1)` + .filter(|value| !value.is_null()) + .map(scalar_value_to_datum) + .collect::>>()?; + + Some(Reference::new(column).is_in(literals)) +} + +/// Converts a multi-column InList into an `iceberg::Predicate` +/// +/// Hash joins on two or more columns build filters like `struct(a, b) IN ({1, 3}, {2, 4})`. +/// +/// Iceberg can't check pairs, so each column is checked on its own: +/// `a IN (1, 2) AND b IN (3, 4)`. This can result in extra rows from pairs like `{1, 4}`, +/// which is okay. +/// +/// A column with a NULL value is left out, because the join decides if NULL matches NULL. +fn convert_struct_in_list(in_list: &InListExpr) -> Option { + let columns = + ScalarFunctionExpr::try_downcast_func::(in_list.expr().as_ref())?.args(); + + let items = in_list + .list() + .iter() + // If any of the items in the list aren't struct literals, we can't convert this + // and we bail early + .map(|item| match literal_value(item)? { + ScalarValue::Struct(item) => Some(item), + _ => None, + }) + .collect::>>()?; + + // One IN list per plain column. A column left out only makes the filter match more rows. + columns + .iter() + .enumerate() + .filter_map(|(index, column)| { + let column = column_name(column)?; + let values = field_values(&items, index)?; + Some(Reference::new(column).is_in(values)) + }) + .reduce(Predicate::and) +} + +/// Field `index` of each struct as an Iceberg value, or `None` if any of them can't be converted. +/// Each struct is a one-row array, so its value is in row 0. +fn field_values(items: &[&Arc], index: usize) -> Option> { + items + .iter() + .map(|item| { + let field = item.columns().get(index)?; + scalar_value_to_datum(&ScalarValue::try_from_array(field, 0).ok()?) + }) + .collect() +} + +fn column_name(expr: &Arc) -> Option<&str> { + expr.downcast_ref::().map(Column::name) +} + +fn literal_value(expr: &Arc) -> Option<&ScalarValue> { + expr.downcast_ref::().map(Literal::value) +} + +fn literal_datum(expr: &Arc) -> Option { + scalar_value_to_datum(literal_value(expr)?) +} + +/// Converts a DataFusion value to an iceberg value, for the types iceberg columns use. +/// +/// Floats aren't converted. DataFusion sorts NaN above every number and -0.0 below 0.0. +/// Iceberg doesn't, so a float filter like `a > 1.0` could skip rows DataFusion keeps. +/// +/// Times aren't converted either, because iceberg-rust's file reader can't compare them yet. +fn scalar_value_to_datum(value: &ScalarValue) -> Option { + match value { + ScalarValue::Boolean(Some(v)) => Some(Datum::bool(*v)), + ScalarValue::Int32(Some(v)) => Some(Datum::int(*v)), + ScalarValue::Int64(Some(v)) => Some(Datum::long(*v)), + ScalarValue::Decimal128(Some(v), precision, scale) => { + let data_type = PrimitiveType::Decimal { + precision: u32::from(*precision), + scale: u32::try_from(*scale).ok()?, + }; + Datum::try_from_bytes(&v.to_be_bytes(), data_type).ok() + } + ScalarValue::Date32(Some(v)) => Some(Datum::date(*v)), + ScalarValue::TimestampMicrosecond(Some(v), None) => Some(Datum::timestamp_micros(*v)), + ScalarValue::TimestampMicrosecond(Some(v), Some(_)) => Some(Datum::timestamptz_micros(*v)), + ScalarValue::TimestampNanosecond(Some(v), None) => Some(Datum::timestamp_nanos(*v)), + ScalarValue::TimestampNanosecond(Some(v), Some(_)) => Some(Datum::timestamptz_nanos(*v)), + ScalarValue::Utf8(Some(v)) + | ScalarValue::LargeUtf8(Some(v)) + | ScalarValue::Utf8View(Some(v)) => Some(Datum::string(v)), + ScalarValue::Binary(Some(v)) + | ScalarValue::LargeBinary(Some(v)) + | ScalarValue::BinaryView(Some(v)) => Some(Datum::binary(v.clone())), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::array::{ArrayRef, Int64Array, StructArray}; + use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema}; + use datafusion::common::{DFSchema, Result}; + use datafusion::config::ConfigOptions; + use datafusion::functions::core::r#struct; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{ + BinaryExpr, Column, InListExpr, col, in_list, lit, + }; + use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; + use datafusion::prelude::SessionContext; + use datafusion::scalar::ScalarValue; + use insta::assert_snapshot; + + use super::convert_physical_filter_to_predicate; + + // The filters below use the columns from `schema()`: `a` and `b` are integers, `s` is a + // string, `f` is a float and `flag` is a boolean. + + #[test] + fn converts_null_checks() { + assert_snapshot!(convert_sql("a IS NULL"), @"a IS NULL"); + assert_snapshot!(convert_sql("a IS NOT NULL"), @"a IS NOT NULL"); + } + + #[test] + fn converts_comparisons_with_a_literal() { + assert_snapshot!(convert_sql("a = 5"), @"a = 5"); + assert_snapshot!(convert_sql("a != 5"), @"a != 5"); + assert_snapshot!(convert_sql("a < 5"), @"a < 5"); + assert_snapshot!(convert_sql("a <= 5"), @"a <= 5"); + assert_snapshot!(convert_sql("a > 5"), @"a > 5"); + assert_snapshot!(convert_sql("a >= 5"), @"a >= 5"); + } + + #[test] + fn flips_comparisons_with_the_literal_on_the_left() { + assert_snapshot!(convert_sql("5 < a"), @"a > 5"); + assert_snapshot!(convert_sql("5 >= a"), @"a <= 5"); + } + + #[test] + fn leaves_comparisons_between_columns_unconverted() { + assert_snapshot!(convert_sql("a < b"), @"None"); + } + + #[test] + fn leaves_float_comparisons_unconverted() { + assert_snapshot!(convert_sql("f > 1.5"), @"None"); + assert_snapshot!(convert_sql("a = 1 AND f > 1.5"), @"a = 1"); + } + + #[test] + fn converts_in_lists() { + assert_snapshot!(convert_sql("a IN (1, 2)"), @"a IN (1, 2)"); + } + + #[test] + fn leaves_not_in_lists_unconverted() { + assert_snapshot!(convert_sql("a NOT IN (1, 2)"), @"None"); + } + + #[test] + fn drops_nulls_from_an_in_list() -> Result<()> { + // Built by hand, the way a hash join builds it. SQL would wrap the NULL in a cast. + let schema = schema(); + let null = lit(ScalarValue::Int64(None)); + let values = vec![lit(1i64), Arc::clone(&null)]; + let one_or_null = in_list(col("a", &schema)?, values, &false, &schema)?; + let only_null = in_list(col("a", &schema)?, vec![null], &false, &schema)?; + assert_snapshot!(convert_filter(&one_or_null), @"a IN (1)"); + // `a IN (NULL)` matches nothing. + assert_snapshot!(convert_filter(&only_null), @"a IN ()"); + Ok(()) + } + + #[test] + fn leaves_in_lists_with_a_column_item_unconverted() { + assert_snapshot!(convert_sql("a IN (b, 1)"), @"None"); + } + + #[test] + fn leaves_an_in_list_over_an_expression_unconverted() { + assert_snapshot!(convert_sql("a + 1 IN (1, 2)"), @"None"); + } + + #[test] + fn leaves_filters_that_dynamic_filters_dont_use_unconverted() { + assert_snapshot!(convert_sql("NOT (a < 5)"), @"None"); + assert_snapshot!(convert_sql("s LIKE 'ab%'"), @"None"); + assert_snapshot!(convert_sql("flag"), @"None"); + } + + #[test] + fn converts_true_false_and_null() { + assert_snapshot!(convert_sql("true"), @"TRUE"); + assert_snapshot!(convert_sql("false"), @"FALSE"); + // A NULL filter keeps no rows. + assert_snapshot!(convert_sql("NULL"), @"FALSE"); + } + + #[test] + fn converts_an_and() { + assert_snapshot!(convert_sql("a = 1 AND b = 2"), @"(a = 1) AND (b = 2)"); + } + + #[test] + fn drops_the_unconverted_side_of_an_and() { + assert_snapshot!(convert_sql("a = 1 AND a < b"), @"a = 1"); + } + + #[test] + fn converts_an_or_only_when_both_sides_convert() { + // Hash joins build this when rows with a NULL key can still match. + assert_snapshot!( + convert_sql("a IS NULL OR a IN (1, 2)"), + @"(a IS NULL) OR (a IN (1, 2))" + ); + assert_snapshot!(convert_sql("a = 1 OR a < b"), @"None"); + } + + #[test] + fn converts_a_case_to_each_when_and_its_then_ored_together() { + assert_snapshot!( + convert_sql("CASE WHEN a = 1 THEN b = 2 WHEN a = 2 THEN b = 3 ELSE b = 4 END"), + @"(((a = 1) AND (b = 2)) OR ((a = 2) AND (b = 3))) OR (b = 4)" + ); + } + + #[test] + fn converts_case_value_whens_to_equalities() { + assert_snapshot!( + convert_sql("CASE a WHEN 1 THEN b = 2 WHEN 2 THEN b = 3 END"), + @"((a = 1) AND (b = 2)) OR ((a = 2) AND (b = 3))" + ); + } + + #[test] + fn keeps_only_the_thens_when_the_whens_cant_be_converted() { + // `a % 2` stands in for the `hash(a) % N` a partitioned hash join uses. + assert_snapshot!( + convert_sql("CASE a % 2 WHEN 0 THEN b IN (1, 2) WHEN 1 THEN b = 3 ELSE false END"), + @"(b IN (1, 2)) OR (b = 3)" + ); + } + + #[test] + fn keeps_the_when_if_its_then_cant_be_converted() { + assert_snapshot!(convert_sql("CASE WHEN a = 1 THEN a < b ELSE false END"), @"a = 1"); + } + + #[test] + fn converts_a_case_with_an_always_true_branch_to_true() { + // Hash joins use `true` for a partition whose build side was canceled. + assert_snapshot!(convert_sql("CASE a % 2 WHEN 0 THEN b = 1 ELSE true END"), @"TRUE"); + } + + #[test] + fn leaves_a_case_with_an_unconverted_branch_unconverted() { + assert_snapshot!( + convert_sql("CASE a % 2 WHEN 0 THEN b IN (1, 2) WHEN 1 THEN a < b ELSE false END"), + @"None" + ); + assert_snapshot!(convert_sql("CASE WHEN a = 1 THEN b = 2 ELSE a < b END"), @"None"); + } + + #[test] + fn converts_a_multi_column_join_filter_to_an_in_list_per_column() -> Result<()> { + let filter = struct_in(vec![Some(1), Some(2)], vec![Some(5), Some(6)])?; + assert_snapshot!(convert_filter(&filter), @"(a IN (1, 2)) AND (b IN (5, 6))"); + Ok(()) + } + + #[test] + fn drops_join_filter_columns_with_a_null() -> Result<()> { + let filter = struct_in(vec![Some(1), Some(2)], vec![Some(3), None])?; + assert_snapshot!(convert_filter(&filter), @"a IN (1, 2)"); + Ok(()) + } + + #[test] + fn converts_literals_to_iceberg_values() { + // Each value is compared with `a`. The translator doesn't check that the types match. + let values = [ + ScalarValue::Boolean(Some(true)), + ScalarValue::Int32(Some(1)), + ScalarValue::Int64(Some(1)), + ScalarValue::Decimal128(Some(12345), 10, 2), + ScalarValue::Decimal128(Some(-12345), 10, 2), + ScalarValue::Date32(Some(19723)), + ScalarValue::TimestampMicrosecond(Some(1_704_067_200_000_000), None), + ScalarValue::TimestampMicrosecond(Some(1_704_067_200_000_000), Some("+00:00".into())), + ScalarValue::TimestampNanosecond(Some(1_704_067_200_000_000_000), None), + ScalarValue::TimestampNanosecond( + Some(1_704_067_200_000_000_000), + Some("+00:00".into()), + ), + ScalarValue::Utf8(Some("ab".into())), + ScalarValue::LargeUtf8(Some("ab".into())), + ScalarValue::Utf8View(Some("ab".into())), + ScalarValue::Binary(Some(vec![1, 2])), + ScalarValue::LargeBinary(Some(vec![1, 2])), + ScalarValue::BinaryView(Some(vec![1, 2])), + ScalarValue::Float32(Some(1.5)), + ScalarValue::Float64(Some(1.5)), + ScalarValue::Time64Microsecond(Some(3_600_000_000)), + ScalarValue::Int64(None), + ]; + let converted = values + .into_iter() + .map(|value| { + let column = Arc::new(Column::new("a", 0)); + let filter: Arc = + Arc::new(BinaryExpr::new(column, Operator::Eq, lit(value.clone()))); + format!("{value:?} => {}", convert_filter(&filter)) + }) + .collect::>() + .join("\n"); + assert_snapshot!(converted, @r#" + Boolean(true) => a = true + Int32(1) => a = 1 + Int64(1) => a = 1 + Decimal128(123.45,10,2) => a = 123.45 + Decimal128(-123.45,10,2) => a = -123.45 + Date32("2024-01-01") => a = 2024-01-01 + TimestampMicrosecond(1704067200000000, None) => a = 2024-01-01 00:00:00 + TimestampMicrosecond(1704067200000000, Some("+00:00")) => a = 2024-01-01 00:00:00 UTC + TimestampNanosecond(1704067200000000000, None) => a = 2024-01-01 00:00:00 + TimestampNanosecond(1704067200000000000, Some("+00:00")) => a = 2024-01-01 00:00:00 UTC + Utf8("ab") => a = "ab" + LargeUtf8("ab") => a = "ab" + Utf8View("ab") => a = "ab" + Binary("1,2") => a = 0102 + LargeBinary("1,2") => a = 0102 + BinaryView("1,2") => a = 0102 + Float32(1.5) => None + Float64(1.5) => None + Time64Microsecond("3600000000") => None + Int64(NULL) => None + "#); + } + + fn schema() -> Schema { + Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + Field::new("f", DataType::Float64, true), + Field::new("flag", DataType::Boolean, true), + ]) + } + + /// Turns SQL like `a < 5` into a DataFusion filter over [`schema`], then converts it. + fn convert_sql(filter: &str) -> String { + let schema = DFSchema::try_from(schema()).unwrap(); + let ctx = SessionContext::new(); + let expr = ctx.parse_sql_expr(filter, &schema).unwrap(); + let expr = ctx.state().create_physical_expr(expr, &schema).unwrap(); + convert_filter(&expr) + } + + /// Converts the filter and prints the predicate, or `None`. + fn convert_filter(filter: &Arc) -> String { + match convert_physical_filter_to_predicate(filter) { + Some(predicate) => predicate.to_string(), + None => "None".to_string(), + } + } + + /// Builds the filter a hash join on `a` and `b` makes: `struct(a, b) IN (...)`. + /// Each pair takes the values at the same position in `a` and `b`. + fn struct_in(a: Vec>, b: Vec>) -> Result> { + let schema = schema(); + let fields = Fields::from(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Int64, true), + ]); + let columns = ScalarFunctionExpr::new( + "struct", + r#struct(), + vec![col("a", &schema)?, col("b", &schema)?], + Arc::new(Field::new("struct", DataType::Struct(fields.clone()), true)), + Arc::new(ConfigOptions::default()), + ); + let values = StructArray::new( + fields, + vec![ + Arc::new(Int64Array::from(a)) as ArrayRef, + Arc::new(Int64Array::from(b)) as ArrayRef, + ], + None, + ); + Ok(Arc::new(InListExpr::try_new_from_array( + Arc::new(columns), + Arc::new(values), + false, + &schema, + )?)) + } +} diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs new file mode 100644 index 000000000..e26bf5bc0 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -0,0 +1,154 @@ +use std::sync::Arc; + +use datafusion::common::HashMap; +use datafusion::physical_expr::{PhysicalExpr, conjunction}; +use datafusion::physical_expr_common::physical_expr::{ + snapshot_generation, snapshot_physical_expr, +}; +use datafusion::physical_plan::metrics::{ + ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, +}; +use iceberg::expr::Predicate; +use iceberg::scan::FileScanTask; + +use crate::common::dynamic_filters::partition_filter::PartitionFilter; +use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; +use crate::common::dynamic_filters::row_group_filter::reader_filter; +use crate::common::dynamic_filters::task_filter::and_predicate; + +/// Applies the scan's dynamic filters to each file just before iceberg-rust's `ArrowReader` opens +/// it. It skips files whose partition value can't match the filters. For the other files, it can +/// add the filters to the file's own filter, so `ArrowReader` can skip row groups. +pub(crate) struct DynamicFilterPruner { + /// The scan's dynamic filters, combined with AND. + dynamic_filters: Arc, + + /// The generation of `dynamic_filters` that `predicate` was built from. + cached_snapshot_generation: u64, + + /// The dynamic filters are stored as iceberg Predicates after being converted + /// `None` if the filters can't be converted + predicate: Option, + + /// A map of partition spec IDs to their corresponding partition filter executors + partition_filters: HashMap>, + + /// Metrics to track pruning + files: PruningMetrics, +} + +impl DynamicFilterPruner { + pub(crate) fn new( + dynamic_filters: Vec>, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + ) -> Self { + Self { + dynamic_filters: conjunction(dynamic_filters), + cached_snapshot_generation: 0, + predicate: None, + partition_filters: HashMap::new(), + files: MetricBuilder::new(metrics) + .with_type(MetricType::Summary) + .pruning_metrics("files_pruned_by_partition", partition), + } + } + + /// Whether the filters rule out every row in the task's partition. Counts the task as pruned + /// or matched + pub(crate) fn should_prune(&mut self, task: &FileScanTask) -> bool { + let prune = self.can_skip(task); + if prune { + self.files.add_pruned(1); + } else { + self.files.add_matched(1); + } + prune + } + + /// Adds the dynamic filters to the task's own filter, so iceberg-rust's `ArrowReader` can use + /// them to skip row groups. Returns the task unchanged if adding them fails. + pub(crate) fn with_reader_filter(&mut self, task: FileScanTask) -> FileScanTask { + self.refresh(); + let Some(predicate) = &self.predicate else { + return task; + }; + + // Some data files may not contain identity partition columns in their rows. + // `ArrowReader` reads a column the file doesn't store as NULL, so a filter on it drops + // every row in the file. The same goes for columns added later with a default value. + // We accept this until iceberg-rust fixes it. + // + // We AND onto the task's existing filter, so `ArrowReader` still applies the query's + // WHERE clause. + and_predicate(&task, reader_filter(predicate)).unwrap_or(task) + } + + /// Whether no row in the task's partition can match the filters. + fn can_skip(&mut self, task: &FileScanTask) -> bool { + let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { + return false; + }; + if spec.is_unpartitioned() { + return false; + } + self.refresh(); + let Some(predicate) = &self.predicate else { + return false; + }; + + // Every `FileScanTask` within a scan has the same Iceberg schema, taken from the selected + // Iceberg snapshot. + // The only difference in partition filters can come from the partition spec being different + // between data files. + // The PartitionFilter struct - + // 1. Binds the `Predicate`, which just contains column names and values, to the actual + // schema, resolving column names to Iceberg field IDs etc. + // 2. Projects the column filter into a partition filter. A partition spec is not limited + // to just direct column filters; it can have expressions like `bucket(2, x)` or + // `day(ts)`. A column filter like `ts >= '2024-03-15 10:00'` becomes + // `ts_day >= 2024-03-15` which can be checked against every data file's partition value. + // These steps can be expensive, so we maintain a cache of a `PartitionFilter` per spec ID. + let partition_filter = self + .partition_filters + .entry(spec.spec_id()) + .or_insert_with(|| { + PartitionFilter::try_new( + predicate.clone(), + task.schema_ref(), + spec, + task.case_sensitive(), + ) + .ok() + }); + let Some(filter) = partition_filter.as_ref() else { + return false; + }; + + // Check whether the file can be dropped, based on the partition values. + // If there's an error while evaluating the filter, + // we return `false` so that the file is not skipped + filter.can_match(partition).is_ok_and(|matches| !matches) + } + + /// Rebuilds the predicate when any of the filters has changed. + fn refresh(&mut self) { + // Read the current generation before snapshotting the filters. + // The two reads aren't atomic, so a filter can change between them. + // + // In this order, the snapshot can be newer than the generation we have stored + // which is okay - because we're still doing the most 'efficient' pruning + // + // In the opposite order, we could store a newer generation but an older filter, + // which would prune less until the filter updated. + let generation = snapshot_generation(&self.dynamic_filters); + if self.cached_snapshot_generation == generation { + return; + } + self.cached_snapshot_generation = generation; + self.partition_filters.clear(); + self.predicate = snapshot_physical_expr(Arc::clone(&self.dynamic_filters)) + .ok() + .and_then(|snapshot| convert_physical_filter_to_predicate(&snapshot)); + } +} diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs new file mode 100644 index 000000000..870852512 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs @@ -0,0 +1,101 @@ +use iceberg::expr::Predicate; + +/// The maximum number of values we give iceberg-rust's `ArrowReader` across all the IN lists in the filter. +/// +/// `ArrowReader` checks an IN list by comparing each row with every value in it, one value at a +/// time. So the more values there are, the slower every row gets. A join's filter can have +/// thousands of values. DataFusion keeps each IN list to at most 150 values, but a partitioned +/// join has one list per partition, and the filter combines them with OR. +/// +/// When the filter has more values than this, we leave out all of its IN lists. `ArrowReader` +/// still gets the join's min and max values, so it can still skip row groups outside that range. +/// We use 200 because Iceberg's own row group checks ignore any IN list longer than that. This +/// limit can be removed once `ArrowReader` checks IN lists more optimally (eg. with a hash set) +const MAX_IN_LIST_VALUES: usize = 200; + +/// Returns the dynamic filters to give iceberg-rust's `ArrowReader`. `ArrowReader` uses them to +/// skip row groups, and to drop the rows that don't match. +/// +/// `predicate` is the dynamic filters, converted to Iceberg's filter type. +/// +/// If the filters have too many IN list values, we leave out every IN list. See +/// [`MAX_IN_LIST_VALUES`]. +pub(crate) fn reader_filter(predicate: &Predicate) -> Predicate { + if count_in_list_values(predicate) <= MAX_IN_LIST_VALUES { + return predicate.clone(); + } + keep_filters(predicate, &|filter| !matches!(filter, Predicate::Set(_))) +} + +/// Walks the whole filter and adds up the values in every IN and NOT IN list it finds. +fn count_in_list_values(predicate: &Predicate) -> usize { + match predicate { + Predicate::And(expr) | Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + count_in_list_values(left) + count_in_list_values(right) + } + Predicate::Not(expr) => { + let [inner] = expr.inputs(); + count_in_list_values(inner) + } + Predicate::Set(expr) => expr.literals().len(), + _ => 0, + } +} + +/// Returns a copy of `predicate` with only the filters that `keep` accepts. We replace every +/// other filter with "always true". +/// +/// This is safe because it can only make `predicate` match more rows. So iceberg-rust's +/// `ArrowReader` may read some rows it didn't need to, but it never drops a row that matches. +/// +/// A NOT is the exception. Inside a NOT, "always true" turns into "always false", which would +/// drop rows that match. So we replace the whole NOT with "always true". Our filter converter +/// never produces a NOT, so this doesn't come up in practice. +fn keep_filters(predicate: &Predicate, keep: &dyn Fn(&Predicate) -> bool) -> Predicate { + match predicate { + Predicate::And(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).and(keep_filters(right, keep)) + } + Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).or(keep_filters(right, keep)) + } + Predicate::Not(_) => Predicate::AlwaysTrue, + filter if keep(filter) => filter.clone(), + _ => Predicate::AlwaysTrue, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::expr::Reference; + use iceberg::spec::Datum; + + #[test] + fn leaves_out_long_lists_and_keeps_their_bounds() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(id().less_than_or_equal_to(Datum::long(1000))) + .and(id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"(id >= 0) AND (id <= 1000)"); + } + + #[test] + fn leaves_out_a_not_around_a_long_list() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(!id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"id >= 0"); + } + + fn id() -> Reference { + Reference::new("id") + } +} diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs new file mode 100644 index 000000000..2794d59ed --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs @@ -0,0 +1,232 @@ +//! Adds a filter to a `FileScanTask`. +//! +//! iceberg-rust has no API for this. A task's filter can't be changed, and iceberg-rust has no +//! public way to AND two bound filters. So we turn the task's filter back into an unbound one, +//! AND ours onto it, bind the result, and build a new task with it. +//! +//! If iceberg-rust adds an API for this, delete this file and use the API. + +use iceberg::Result; +use iceberg::expr::{ + BinaryExpression, Bind, BoundPredicate, BoundReference, Predicate, Reference, SetExpression, + UnaryExpression, +}; +use iceberg::scan::FileScanTask; +use iceberg::spec::Schema; + +/// Returns a copy of `task` whose filter is its existing filter AND `predicate`. Returns `None` +/// if that fails. +pub(crate) fn and_predicate(task: &FileScanTask, predicate: Predicate) -> Option { + let combined = match task.predicate() { + Some(existing) => unbind(existing, task.schema())?.and(predicate), + None => predicate, + }; + let combined = combined + .bind(task.schema_ref(), task.case_sensitive()) + .ok()?; + with_predicate(task, combined).ok() +} + +/// Turns a bound predicate back into an unbound one, so we can AND it with another predicate and +/// bind it again. Each column gets its full name from `schema`, like `location.lat`. Returns +/// `None` if a column isn't in `schema`. +fn unbind(predicate: &BoundPredicate, schema: &Schema) -> Option { + let reference = |column: &BoundReference| { + schema + .name_by_field_id(column.field().id) + .map(Reference::new) + }; + Some(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.and(unbind(right, schema)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.or(unbind(right, schema)?) + } + BoundPredicate::Not(expr) => { + let [inner] = expr.inputs(); + !unbind(inner, schema)? + } + BoundPredicate::Unary(expr) => { + Predicate::Unary(UnaryExpression::new(expr.op(), reference(expr.term())?)) + } + BoundPredicate::Binary(expr) => Predicate::Binary(BinaryExpression::new( + expr.op(), + reference(expr.term())?, + expr.literal().clone(), + )), + BoundPredicate::Set(expr) => Predicate::Set(SetExpression::new( + expr.op(), + reference(expr.term())?, + expr.literals().clone(), + )), + }) +} + +/// Returns a copy of `task` with `predicate` as its filter. Every other field is copied as is. +/// This is only required because iceberg-rust doesn't provide a way to add predicates to an existing FileScanTask +fn with_predicate(task: &FileScanTask, predicate: BoundPredicate) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(task.file_size_in_bytes()) + .with_start(task.start()) + .with_length(task.length()) + .with_record_count(task.record_count()) + .with_first_row_id(task.first_row_id()) + .with_data_sequence_number(task.data_sequence_number()) + .with_data_file_path(task.data_file_path().to_string()) + .with_data_file_format(task.data_file_format()) + .with_schema(task.schema_ref()) + .with_project_field_ids(task.project_field_ids().to_vec()) + .with_predicate(Some(predicate)) + .with_deletes(task.deletes().to_vec()) + .with_partition(task.partition().cloned()) + .with_partition_spec(task.partition_spec().cloned()) + .with_name_mapping(task.name_mapping().cloned()) + .with_unified_partition_type(task.unified_partition_type().cloned()) + .with_sort_order_id(task.sort_order_id()) + .with_sort_order(task.sort_order().cloned()) + .with_case_sensitive(task.case_sensitive()) + .with_key_metadata(task.key_metadata().map(Box::from)) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::scan::FileScanTaskDeleteFile; + use iceberg::spec::{ + DataContentType, DataFileFormat, Datum, Literal, MappedField, NameMapping, NestedField, + PartitionSpec, PrimitiveType, SortOrder, Struct, StructType, Transform, Type, + }; + use std::error::Error; + use std::sync::Arc; + + #[test] + fn unbinding_and_binding_again_gives_the_same_predicate() + -> std::result::Result<(), Box> { + let location = StructType::new(vec![ + NestedField::optional(3, "lat", Type::Primitive(PrimitiveType::Int)).into(), + ]); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "location", Type::Struct(location)).into(), + ]) + .build()?, + ); + let id = || Reference::new("id"); + let predicate = id() + .less_than(Datum::long(5)) + .and(!id().is_in([Datum::long(1), Datum::long(2)])) + .or(Reference::new("name").starts_with(Datum::string("a"))) + .or(Reference::new("location.lat").is_null()) + .bind(Arc::clone(&schema), true)?; + + let unbound = unbind(&predicate, &schema).ok_or("a column isn't in the schema")?; + + assert_eq!(unbound.bind(schema, true)?, predicate); + Ok(()) + } + + /// `with_predicate` copies a task one field at a time. Every field of the task below differs + /// from its default, so the `assert_eq!` fails if the copy leaves a field out. If iceberg-rust + /// adds a field, the copy would silently leave it out. The snapshot lists the task's fields, + /// so it fails when a field is added. + #[test] + fn the_copy_of_a_task_keeps_every_field() -> std::result::Result<(), Box> { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(Arc::clone(&schema)) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let deletes = FileScanTaskDeleteFile::builder() + .with_file_path("deletes.parquet".to_string()) + .with_file_size_in_bytes(1) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build()?; + let name_mapping = NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )]); + let task = FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(2) + .with_length(3) + .with_record_count(Some(4)) + .with_first_row_id(Some(5)) + .with_data_sequence_number(Some(6)) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&schema)) + .with_project_field_ids(vec![1]) + .with_predicate(Some( + Reference::new("id") + .is_null() + .bind(Arc::clone(&schema), true)?, + )) + .with_deletes(vec![deletes]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(7))]))) + .with_unified_partition_type(Some(Arc::new(spec.partition_type(&schema)?))) + .with_partition_spec(Some(Arc::new(spec))) + .with_name_mapping(Some(Arc::new(name_mapping))) + .with_sort_order_id(Some(8)) + .with_sort_order(Some(Arc::new(SortOrder::unsorted_order()))) + .with_case_sensitive(true) + .with_key_metadata(Some(Box::from(b"key".as_slice()))) + .build()?; + let predicate = task.predicate().ok_or("the task has no filter")?; + + let copy = with_predicate(&task, predicate.clone())?; + + assert_eq!(copy, task); + insta::assert_snapshot!(field_names(&task).join("\n"), @" + file_size_in_bytes + start + length + record_count + first_row_id + data_sequence_number + data_file_path + data_file_format + schema + project_field_ids + predicate + deletes + partition + partition_spec + name_mapping + unified_partition_type + sort_order_id + sort_order + case_sensitive + key_metadata + "); + Ok(()) + } + + /// Reads a struct's field names from its pretty-printed `Debug` output. The top-level fields + /// are the lines indented by exactly four spaces. + fn field_names(value: &impl std::fmt::Debug) -> Vec { + format!("{value:#?}") + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter(|line| !line.starts_with(' ')) + .filter_map(|line| line.split_once(':')) + .map(|(name, _)| name.to_string()) + .collect() + } +} diff --git a/datafusion/distributed-iceberg/src/common/mod.rs b/datafusion/distributed-iceberg/src/common/mod.rs index 77c185916..984b811f8 100644 --- a/datafusion/distributed-iceberg/src/common/mod.rs +++ b/datafusion/distributed-iceberg/src/common/mod.rs @@ -2,6 +2,7 @@ mod error; pub(crate) use error::{df_err, iceberg_err}; +mod dynamic_filters; mod file_partition; pub(crate) mod scan_config; pub(crate) use file_partition::file_partition; diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 71eebfe19..a212ed60b 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -1,3 +1,4 @@ +use crate::common::dynamic_filters::pruner::DynamicFilterPruner; use crate::{IcebergConfig, common::df_err}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::stats::Precision; @@ -5,10 +6,8 @@ use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::common::{ColumnStatistics, Statistics}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::expressions::{InListExpr, lit}; -use datafusion::physical_expr::{ - DynamicFilterTracking, Partitioning, PhysicalExpr, conjunction, split_conjunction, -}; +use datafusion::physical_expr::expressions::{DynamicFilterPhysicalExpr, InListExpr, lit}; +use datafusion::physical_expr::{Partitioning, PhysicalExpr, conjunction, split_conjunction}; use datafusion::physical_expr_common::physical_expr::snapshot_physical_expr; use datafusion::physical_plan::apply_expression_roots; use datafusion::physical_plan::limit::LimitStream; @@ -23,6 +22,7 @@ use iceberg::io::FileIO; use iceberg::scan::FileScanTaskStream; use iceberg::spec::SnapshotRef; use std::fmt; +use std::future::ready; use std::sync::Arc; /// Execution settings shared by assigned-file and feed-backed scans. @@ -96,11 +96,7 @@ impl ScanConfig { let Some(filters) = &self.filters else { return Ok(()); }; - let dynamic_filters: Vec<_> = split_conjunction(filters) - .into_iter() - .filter(|filter| DynamicFilterTracking::classify(filter).contains_dynamic_filter()) - .cloned() - .collect(); + let dynamic_filters = dynamic_filters(filters); if dynamic_filters.is_empty() { return Ok(()); } @@ -149,9 +145,22 @@ impl ScanConfig { .with_category(MetricCategory::Bytes) .gauge("bytes_scanned", partition); + let mut pruner = self.dynamic_filter_pruner(partition); // The reader pulls a task when it starts reading that task's file. It reads ahead, - // so with a LIMIT this can count files that never produced output. - let tasks = tasks.inspect_ok(move |_| files_opened.add(1)).boxed(); + // so with a LIMIT this can count files that never produced output. Skipped files + // are dropped before this, and the pruner counts them. + let tasks = tasks + .try_filter_map(move |mut task| { + if let Some(pruner) = pruner.as_mut() { + if pruner.should_prune(&task) { + return ready(Ok(None)); + } + task = pruner.with_reader_filter(task); + } + ready(Ok(Some(task))) + }) + .inspect_ok(move |_| files_opened.add(1)) + .boxed(); let result = self.reader.clone().read(tasks).map_err(df_err)?; let reader_metrics = result.metrics().clone(); @@ -173,6 +182,32 @@ impl ScanConfig { Ok(Box::pin(LimitStream::new(stream, 0, self.fetch, metrics))) } + + /// A pruner over the scan's dynamic filters. `None` when dynamic filtering is off or the + /// scan has no dynamic filter. + fn dynamic_filter_pruner(&self, partition: usize) -> Option { + if !self.dynamic_filtering_enabled { + return None; + } + let dynamic_filters = dynamic_filters(self.filters.as_ref()?); + (!dynamic_filters.is_empty()) + .then(|| DynamicFilterPruner::new(dynamic_filters, &self.metrics, partition)) + } +} + +/// The parts of `filters` that contain a dynamic filter. It only looks at the expressions, so +/// it doesn't subscribe to the filters' updates. +fn dynamic_filters(filters: &Arc) -> Vec> { + split_conjunction(filters) + .into_iter() + .filter(|filter| { + matches!( + filter.exists(|expr| Ok(expr.is::())), + Ok(true) + ) + }) + .cloned() + .collect() } impl fmt::Debug for ScanConfig { diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..3c7f6f94e 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -18,9 +18,11 @@ extensions_options! { pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false - /// (Experimental) Whether Iceberg scans are consumers of the dynamic filters pushed - /// into them, so joins and aggregates compute those filters. Scans don't skip data - /// with them yet. + /// (Experimental) Whether Iceberg scans use the dynamic filters pushed into them. When + /// on, joins, sorts with a limit and aggregates build these filters while the query + /// runs. Scans use them to skip files whose partition value can't match. Scans also give + /// them to iceberg-rust's Parquet reader, which uses them to skip row groups and to drop + /// the rows that don't match. pub dynamic_filtering_enabled: bool, default = false } } diff --git a/datafusion/distributed-iceberg/tests/partition_skipping.rs b/datafusion/distributed-iceberg/tests/partition_skipping.rs new file mode 100644 index 000000000..bd6fc1f68 --- /dev/null +++ b/datafusion/distributed-iceberg/tests/partition_skipping.rs @@ -0,0 +1,514 @@ +mod tests { + use datafusion::arrow::array::{ + ArrayRef, Date32Array, Int32Array, Int64Array, RecordBatch, StringArray, + TimestampMicrosecondArray, + }; + use datafusion::arrow::datatypes::SchemaRef; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::common::Result; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::metrics::MetricValue; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::create_taxi_table; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Transform, Type}; + use std::error::Error; + use std::sync::Arc; + + // The taxi table has one file per day, for the 7 days from 2024-01-08. The join keeps two of + // them, so its filter lets the scan skip the other 5 files. + const TWO_DAYS_OF_TRIPS: &str = "SELECT COUNT(*) AS trips FROM taxi \ + JOIN (VALUES (DATE '2024-01-10'), (DATE '2024-01-12')) AS days(day) \ + ON taxi.pickup_date = days.day"; + + #[tokio::test] + async fn lazy_planning_workers_skip_files_the_join_filter_rules_out() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let plan = harness.physical_plan(TWO_DAYS_OF_TRIPS).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------+ + | trips | + +-------+ + | 50000 | + +-------+ + "); + // The first `DataSourceExec` is the join's `VALUES` table. The second is the taxi scan. + assert_eq!(node_metrics::(&plan, "files_opened", 1), 2); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_skip_files_the_join_filter_rules_out() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let plan = harness.physical_plan(TWO_DAYS_OF_TRIPS).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------+ + | trips | + +-------+ + | 50000 | + +-------+ + "); + assert_eq!( + node_metrics::(&plan, "files_opened", 0), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn top_n_skips_files_once_it_fills_up() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let plan = harness + .physical_plan("SELECT pickup_date FROM taxi ORDER BY pickup_date LIMIT 3") + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------------+ + | pickup_date | + +-------------+ + | 2024-01-08 | + | 2024-01-08 | + | 2024-01-08 | + +-------------+ + "); + // The first file, 2024-01-08, fills the top 3. The filter then becomes + // `pickup_date < 2024-01-08`, which rules out the other 6 days. + assert_eq!(node_metrics::(&plan, "files_opened", 0), 1); + Ok(()) + } + + #[tokio::test] + async fn month_partitions_skip_files_from_other_months() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_month().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let plan = harness + .physical_plan( + "SELECT events.day FROM events \ + JOIN (VALUES (DATE '2024-02-14')) AS days(day) ON events.day = days.day", + ) + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +------------+ + | day | + +------------+ + | 2024-02-14 | + +------------+ + "); + // The first `DataSourceExec` is the join's `VALUES` table. The second is the events scan, + // which opens only February's file. + assert_eq!(node_metrics::(&plan, "files_opened", 1), 1); + Ok(()) + } + + #[tokio::test] + async fn null_partitions_are_read_when_the_filter_can_match_null() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(groups_by_key().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let plan = harness + .physical_plan("SELECT k FROM groups ORDER BY k NULLS FIRST LIMIT 2") + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +---+ + | k | + +---+ + | | + | | + +---+ + "); + // The scan reads `k = 1` first, and top-N's filter becomes `k IS NULL OR k < 1`. That + // skips the `k = 2` file and keeps the null file. + assert_eq!(node_metrics::(&plan, "files_opened", 0), 2); + Ok(()) + } + + #[tokio::test] + async fn validate_predicate_translation() -> Result<(), Box> { + let skipping = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(readings().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + // A join with more than 2 keys filters only on their bounds, so the cases cover + // both kinds of join filter. + .with_config_option( + "datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values", + 2, + ) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + let reading_every_file = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(readings().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .with_config_option( + "datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values", + 2, + ) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + // Keys on partition edges, keys in no partition, and nulls, for each transform. + let schema = skipping.table().schema()?; + let cases = [ + join(&schema, "id", &["3"]), + join(&schema, "id", &["0"]), + join(&schema, "id", &["2", "6"]), + join(&schema, "id", &["2", "3", "4"]), + join(&schema, "id", &["NULL"]), + join(&schema, "id", &["1", "NULL"]), + join(&schema, "name", &["'avocado'"]), + join(&schema, "name", &["'b'"]), + join(&schema, "name", &["'éclair'"]), + join(&schema, "name", &["'é'"]), + join(&schema, "name", &["'zebra'"]), + join(&schema, "day", &["'2024-02-01'"]), + join(&schema, "day", &["'2024-01-15'"]), + join(&schema, "day", &["'2024-04-01'"]), + join(&schema, "day", &["'2023-12-31'", "'2024-03-01'"]), + join(&schema, "n", &["12"]), + join(&schema, "n", &["99"]), + join(&schema, "n", &["-1", "13"]), + join(&schema, "at", &["'2024-01-02T00:00:00'"]), + join(&schema, "at", &["'2024-01-01T23:59:59.999999'"]), + join(&schema, "at", &["'2024-01-04T00:00:00'"]), + top_n("id", 1), + top_n("id DESC", 1), + top_n("id NULLS FIRST", 2), + top_n("id DESC NULLS LAST", 2), + top_n("name", 1), + top_n("name DESC", 1), + top_n("day", 2), + top_n("day DESC", 1), + top_n("n", 1), + top_n("at", 1), + top_n("at DESC", 2), + ]; + + // Each case must return the same rows whether the scan skips files or reads them all. + // The snapshot shows how many files each case kept. + let mut skipped = Vec::new(); + for (case, sql) in cases { + let plan = skipping.physical_plan(&sql).await?; + let rows = collect(Arc::clone(&plan), skipping.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + let expected = reading_every_file.query(&sql).await?.1; + assert_eq!( + pretty_format_batches(&rows)?.to_string(), + expected, + "{case}" + ); + let files = scan_metric(&plan, "files_pruned_by_partition") + .unwrap_or_else(|| panic!("{case}: the scan has no dynamic filter")); + let rows: usize = rows.iter().map(RecordBatch::num_rows).sum(); + skipped.push(format!("{case}: rows: {rows}, files: {files}")); + } + insta::assert_snapshot!(skipped.join("\n"), @" + join on id with 3: rows: 1, files: 6 total → 1 matched + join on id with 0: rows: 0, files: 6 total → 0 matched + join on id with 2, 6: rows: 2, files: 6 total → 2 matched + join on id with 2, 3, 4: rows: 3, files: 6 total → 3 matched + join on id with NULL: rows: 0, files: 0 total → 0 matched + join on id with 1, NULL: rows: 1, files: 6 total → 1 matched + join on name with 'avocado': rows: 1, files: 6 total → 2 matched + join on name with 'b': rows: 0, files: 6 total → 1 matched + join on name with 'éclair': rows: 1, files: 6 total → 1 matched + join on name with 'é': rows: 0, files: 6 total → 1 matched + join on name with 'zebra': rows: 0, files: 6 total → 0 matched + join on day with '2024-02-01': rows: 1, files: 6 total → 2 matched + join on day with '2024-01-15': rows: 0, files: 6 total → 1 matched + join on day with '2024-04-01': rows: 0, files: 6 total → 0 matched + join on day with '2023-12-31', '2024-03-01': rows: 2, files: 6 total → 2 matched + join on n with 12: rows: 1, files: 6 total → 3 matched + join on n with 99: rows: 0, files: 6 total → 1 matched + join on n with -1, 13: rows: 2, files: 6 total → 4 matched + join on at with '2024-01-02T00:00:00': rows: 1, files: 6 total → 2 matched + join on at with '2024-01-01T23:59:59.999999': rows: 1, files: 6 total → 1 matched + join on at with '2024-01-04T00:00:00': rows: 0, files: 6 total → 0 matched + ORDER BY id LIMIT 1: rows: 1, files: 6 total → 1 matched + ORDER BY id DESC LIMIT 1: rows: 1, files: 6 total → 6 matched + ORDER BY id NULLS FIRST LIMIT 2: rows: 2, files: 6 total → 3 matched + ORDER BY id DESC NULLS LAST LIMIT 2: rows: 2, files: 6 total → 5 matched + ORDER BY name LIMIT 1: rows: 1, files: 6 total → 2 matched + ORDER BY name DESC LIMIT 1: rows: 1, files: 6 total → 5 matched + ORDER BY day LIMIT 2: rows: 2, files: 6 total → 3 matched + ORDER BY day DESC LIMIT 1: rows: 1, files: 6 total → 5 matched + ORDER BY n LIMIT 1: rows: 1, files: 6 total → 6 matched + ORDER BY at LIMIT 1: rows: 1, files: 6 total → 2 matched + ORDER BY at DESC LIMIT 2: rows: 2, files: 6 total → 5 matched + "); + Ok(()) + } + + // Table `events`, partitioned by the month of `day`, with one file for each of January, + // February and March 2024. + async fn events_by_month() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "day", Type::Primitive(PrimitiveType::Date)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .partition_by("day", Transform::Month) + .create() + .await?; + // Days since 1970-01-01. + let january = [19727, 19742]; // 2024-01-05, 2024-01-20 + let february = [19756, 19767]; // 2024-02-03, 2024-02-14 + let march = [19783]; // 2024-03-01 + let mut append = table.append(); + for days in [&january[..], &february[..], &march[..]] { + let days: ArrayRef = Arc::new(Date32Array::from(days.to_vec())); + append = append.file(RecordBatch::try_from_iter([("day", days)])?); + } + append.commit().await?; + Ok(table) + } + + // Table `groups`, partitioned by `k`, with one file each for `k = 1`, `k = 2` and a null `k`, + // in that order. Each file has two rows. + async fn groups_by_key() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "k", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "groups"])?, schema) + .partition_by("k", Transform::Identity) + .create() + .await?; + let mut append = table.append(); + for k in [Some(1), Some(2), None] { + let keys: ArrayRef = Arc::new(Int32Array::from(vec![k, k])); + append = append.file(RecordBatch::try_from_iter([("k", keys)])?); + } + append.commit().await?; + Ok(table) + } + + // Table `readings`, partitioned by `id`, the first letter of `name`, the month of `day`, 4 + // buckets of `n` and the day of `at`. Each row is its own file, and the last row is all + // nulls. + async fn readings() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "day", Type::Primitive(PrimitiveType::Date)).into(), + NestedField::optional(4, "n", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(5, "at", Type::Primitive(PrimitiveType::Timestamp)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "readings"])?, schema) + .partition_by("id", Transform::Identity) + .partition_by("name", Transform::Truncate(1)) + .partition_by("day", Transform::Month) + .partition_by("n", Transform::Bucket(4)) + .partition_by("at", Transform::Day) + .create() + .await?; + let id: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + Some(3), + Some(4), + Some(6), + None, + ])); + let name: ArrayRef = Arc::new(StringArray::from(vec![ + Some("apple"), + Some("avocado"), + Some("banana"), + Some("éclair"), + Some("cherry"), + None, + ])); + // Days since 1970-01-01: 2024-01-31, 2024-02-01, 2024-02-29, 2024-03-01, 2023-12-31. + let day: ArrayRef = Arc::new(Date32Array::from(vec![ + Some(19753), + Some(19754), + Some(19782), + Some(19783), + Some(19722), + None, + ])); + let n: ArrayRef = Arc::new(Int64Array::from(vec![ + Some(10), + Some(11), + Some(12), + Some(13), + Some(-1), + None, + ])); + // Microseconds since 1970-01-01: 2024-01-01 23:59:59.999999, 2024-01-02 00:00:00, + // 2024-01-02 12:00:00, 2024-01-03 00:00:00, 2023-12-31 23:59:59. + let at: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![ + Some(1704153599999999), + Some(1704153600000000), + Some(1704196800000000), + Some(1704240000000000), + Some(1704067199000000), + None, + ])); + let rows = RecordBatch::try_from_iter([ + ("id", id), + ("name", name), + ("day", day), + ("n", n), + ("at", at), + ])?; + let mut append = table.append(); + for row in 0..rows.num_rows() { + append = append.file(rows.slice(row, 1)); + } + append.commit().await?; + Ok(table) + } + + /// A case that joins `readings` with `keys` on `column`. Each key is cast to the column's + /// type, so the join filters the column itself, without a cast around it. + fn join(schema: &SchemaRef, column: &str, keys: &[&str]) -> (String, String) { + let data_type = schema.field_with_name(column).unwrap().data_type(); + let values = keys + .iter() + .map(|key| format!("(arrow_cast({key}, '{data_type}'))")) + .collect::>() + .join(", "); + ( + format!("join on {column} with {}", keys.join(", ")), + format!( + "SELECT t.* FROM readings t JOIN (VALUES {values}) AS k(x) \ + ON t.{column} = k.x ORDER BY t.id" + ), + ) + } + + /// A case that reads the first `limit` rows of `readings` in `order`. + fn top_n(order: &str, limit: usize) -> (String, String) { + ( + format!("ORDER BY {order} LIMIT {limit}"), + format!("SELECT * FROM readings ORDER BY {order} LIMIT {limit}"), + ) + } + + /// The metric `name` of the Iceberg scan in `plan`, summed over its partitions. + fn scan_metric(plan: &Arc, name: &str) -> Option { + plan.metrics() + .and_then(|metrics| metrics.sum_by_name(name)) + .or_else(|| { + plan.children() + .into_iter() + .find_map(|child| scan_metric(child, name)) + }) + } +} diff --git a/datafusion/distributed-iceberg/tests/row_group_skipping.rs b/datafusion/distributed-iceberg/tests/row_group_skipping.rs new file mode 100644 index 000000000..933691b8d --- /dev/null +++ b/datafusion/distributed-iceberg/tests/row_group_skipping.rs @@ -0,0 +1,234 @@ +mod tests { + use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; + use parquet::file::properties::WriterProperties; + use std::error::Error; + use std::sync::Arc; + + // The join keeps one id from each of the two `events` files. + const TWO_IDS: &str = "SELECT events.id FROM events \ + JOIN (VALUES (5), (150005)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + + #[tokio::test] + async fn the_reader_skips_row_groups_the_join_filter_rules_out() -> Result<(), Box> { + let query = "SELECT events.id, events.payload FROM events \ + JOIN (VALUES (150005), (150010)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + let with_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + let without_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .build() + .await?; + + let (with_filter_results, with_filter_plan) = run(&with_filter, query).await?; + let (without_filter_results, without_filter_plan) = run(&without_filter, query).await?; + + insta::assert_snapshot!(with_filter_results, @" + +--------+----------------------------------------+ + | id | payload | + +--------+----------------------------------------+ + | 150005 | event 150005 | + | 150010 | event 150010 | + +--------+----------------------------------------+ + "); + assert_eq!(with_filter_results, without_filter_results); + // `DataSourceExec` 0 is the join's `VALUES` table, and 1 is the `events` scan. + // Both ids are in one row group of the second file, so with the filter iceberg-rust's + // `ArrowReader` reads 1 of the 40 row groups. Both runs also read the last 512 KiB of each + // file to get its footer, so we only check that the filter cuts the bytes read by more + // than 5x. + let with_filter_bytes = + node_metrics::(&with_filter_plan, "bytes_scanned", 1); + let without_filter_bytes = + node_metrics::(&without_filter_plan, "bytes_scanned", 1); + assert!( + with_filter_bytes * 5 < without_filter_bytes, + "read {with_filter_bytes} bytes with the join filter and {without_filter_bytes} without" + ); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_give_the_reader_the_join_filter() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let (results, plan) = run(&harness, TWO_IDS).await?; + + insta::assert_snapshot!(results, @" + +--------+ + | id | + +--------+ + | 5 | + | 150005 | + +--------+ + "); + // The coordinator plans the files and sends them to the workers. Each worker still gives + // `ArrowReader` the join filter, so `ArrowReader` drops the rows it rules out and the scan + // returns only the two ids. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn the_top_n_filter_reaches_files_that_start_later() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so the top-N filter can tighten before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let (results, plan) = run(&harness, "SELECT id FROM events ORDER BY id LIMIT 3").await?; + + insta::assert_snapshot!(results, @" + +----+ + | id | + +----+ + | 0 | + | 1 | + | 2 | + +----+ + "); + // When the first file starts, the top 3 aren't known yet, so the scan returns all 100,000 + // of its rows. By the time the second file starts, they are, and `ArrowReader` drops + // every row in it. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 100_000 + ); + Ok(()) + } + + #[tokio::test] + async fn the_reader_keeps_the_where_filter_too() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let (results, plan) = run( + &harness, + "SELECT events.id, events.kind FROM events \ + JOIN (VALUES (5), (60000), (150004)) AS ids(id) ON events.id = ids.id \ + WHERE events.kind = 'click' ORDER BY events.id", + ) + .await?; + + insta::assert_snapshot!(results, @" + +--------+-------+ + | id | kind | + +--------+-------+ + | 60000 | click | + | 150004 | click | + +--------+-------+ + "); + // `ArrowReader` applies both filters. The join filter keeps ids 5, 60000 and 150004. The + // `WHERE` filter keeps only clicks, which drops the odd id 5. If adding the join filter + // lost the `WHERE` filter, the scan would return 3 rows. + assert_eq!(node_metrics::(&plan, "output_rows", 1), 2); + Ok(()) + } + + /// Runs `sql`, and returns its results and its plan with the metrics from the run. + async fn run( + harness: &IcebergTestHarness, + sql: &str, + ) -> Result<(String, Arc), Box> { + let plan = harness.physical_plan(sql).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + Ok((pretty_format_batches(&batches)?.to_string(), plan)) + } + + // The `events` table has two files of 100,000 rows each, sorted by `id`: + // - The first file has ids 0 to 99,999, and the second has 100,000 to 199,999. + // - `kind` is "click" for even ids and "view" for odd ones. + // - Each file has 20 row groups of 5,000 rows. + // - Each file is larger than the 512 KiB `ArrowReader` reads first to find the footer, so + // skipping row groups reads fewer bytes. + async fn events_by_id() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "kind", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "payload", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .create() + .await?; + let writer_properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(5_000)) + .build(); + let mut append = table.append().writer_properties(writer_properties); + for ids in [0..100_000, 100_000..200_000] { + let kinds: Vec<&str> = ids + .clone() + .map(|id| if id % 2 == 0 { "click" } else { "view" }) + .collect(); + let payloads: Vec = ids.clone().map(|id| format!("event {id:>32}")).collect(); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(ids)); + let kinds: ArrayRef = Arc::new(StringArray::from(kinds)); + let payloads: ArrayRef = Arc::new(StringArray::from(payloads)); + append = append.file(RecordBatch::try_from_iter([ + ("id", ids), + ("kind", kinds), + ("payload", payloads), + ])?); + } + append.commit().await?; + Ok(table) + } +}