From 13220b9ac7413d2c9679497bed1264e613658933 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:37:38 +0000 Subject: [PATCH 01/17] feat: turn DataFusion filters into Iceberg predicates Adds a translator from a DataFusion physical filter to an iceberg predicate. It handles comparisons with a value, IN lists, IS NULL, LIKE prefixes, isnan, starts_with, AND, OR, NOT and CASE. It also handles the filters hash joins build while the query runs: IN lists, struct(a, b) IN (...) for joins on several columns, and the CASE a partitioned join builds with one branch per partition. The predicate matches every row the filter keeps, and can match more, so the filter still runs on every row. An AND can drop a side it can't convert. A NOT is converted only when its inside was converted fully. Floats and times aren't converted, and neither are LIKE prefixes with non-ASCII text. iceberg-rust handles these in ways that could skip matching rows or fail the read. Nothing calls it yet. Partition skipping will use it next. --- .../distributed-iceberg/src/common/mod.rs | 1 + .../src/common/physical_predicate.rs | 733 ++++++++++++++++++ 2 files changed, 734 insertions(+) create mode 100644 datafusion/distributed-iceberg/src/common/physical_predicate.rs diff --git a/datafusion/distributed-iceberg/src/common/mod.rs b/datafusion/distributed-iceberg/src/common/mod.rs index 77c185916..cce214161 100644 --- a/datafusion/distributed-iceberg/src/common/mod.rs +++ b/datafusion/distributed-iceberg/src/common/mod.rs @@ -3,5 +3,6 @@ mod error; pub(crate) use error::{df_err, iceberg_err}; mod file_partition; +mod physical_predicate; pub(crate) mod scan_config; pub(crate) use file_partition::file_partition; diff --git a/datafusion/distributed-iceberg/src/common/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/physical_predicate.rs new file mode 100644 index 000000000..b735d7692 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/physical_predicate.rs @@ -0,0 +1,733 @@ +use std::sync::Arc; + +use datafusion::logical_expr::Operator; +use datafusion::physical_expr::expressions::{ + BinaryExpr, CaseExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NotExpr, +}; +use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; +use datafusion::scalar::ScalarValue; +use iceberg::expr::{Predicate, Reference}; +use iceberg::spec::{Datum, PrimitiveType}; + +/// Turns a DataFusion filter into an iceberg predicate. The predicate matches every row that the +/// filter would match, and *can* match more if the DF expression cannot be translated into an Iceberg predicate completely. +/// Returns `None` when no part of the filter can be converted. +pub(crate) fn convert_physical_filter_to_predicate( + filter: &Arc, +) -> Option { + convert(filter).map(|converted| converted.predicate) +} + +/// An iceberg predicate that matches every row the DataFusion filter matches. +/// It can also match extra rows, so the caller must still apply the original filter. +struct Converted { + predicate: Predicate, + /// True if the predicate is false wherever the filter is false. + /// Only an exact predicate can go under a NOT. + exact: bool, +} + +impl Converted { + fn exact(predicate: Predicate) -> Self { + Self { + predicate, + exact: true, + } + } + + fn wider(predicate: Predicate) -> Self { + Self { + predicate, + exact: false, + } + } + + fn and(self, other: Self) -> Self { + Self { + predicate: self.predicate.and(other.predicate), + exact: self.exact && other.exact, + } + } + + fn or(self, other: Self) -> Self { + Self { + predicate: self.predicate.or(other.predicate), + exact: self.exact && other.exact, + } + } +} + +fn convert(expr: &Arc) -> Option { + if let Some(binary) = expr.downcast_ref::() { + return convert_binary(binary); + } + if let Some(not) = expr.downcast_ref::() { + return convert_not(not); + } + if let Some(case) = expr.downcast_ref::() { + return convert_case(case); + } + if let Some(in_list) = expr.downcast_ref::() { + return convert_in_list(in_list); + } + if let Some(is_null) = expr.downcast_ref::() { + let column = column_name(is_null.arg())?; + return Some(Converted::exact(Reference::new(column).is_null())); + } + if let Some(is_not_null) = expr.downcast_ref::() { + let column = column_name(is_not_null.arg())?; + return Some(Converted::exact(Reference::new(column).is_not_null())); + } + if let Some(like) = expr.downcast_ref::() { + return convert_like(like).map(Converted::exact); + } + if let Some(function) = expr.downcast_ref::() { + return convert_function(function).map(Converted::exact); + } + // A bare boolean column like `WHERE flag` becomes `flag = true`. + if let Some(column) = expr.downcast_ref::() { + let predicate = Reference::new(column.name()).equal_to(Datum::bool(true)); + return Some(Converted::exact(predicate)); + } + // A bare `true`, `false` or NULL. NULL matches nothing. + match literal_value(expr)? { + ScalarValue::Boolean(Some(true)) => Some(Converted::exact(Predicate::AlwaysTrue)), + ScalarValue::Boolean(_) | ScalarValue::Null => { + Some(Converted::exact(Predicate::AlwaysFalse)) + } + _ => None, + } +} + +fn convert_binary(binary: &BinaryExpr) -> Option { + let (left, right) = (binary.left(), binary.right()); + match binary.op() { + Operator::And => both(convert(left), convert(right)), + // An OR needs both sides. Turning `a = 1 OR f(b)` into `a = 1` would skip rows where + // only `f(b)` is true. + Operator::Or => Some(convert(left)?.or(convert(right)?)), + op => compare(left, *op, right).map(Converted::exact), + } +} + +/// ANDs two converted sides. An AND can drop a side it can't convert: +/// `a = 1 AND f(b)` becomes `a = 1`. The result matches extra rows, which is fine. +fn both(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.and(right)), + (Some(side), None) | (None, Some(side)) => Some(Converted::wider(side.predicate)), + (None, None) => None, + } +} + +/// A NOT needs an exact predicate. In `NOT (a = 1 AND f(b))`, the inside becomes `a = 1`, +/// and `NOT (a = 1)` would skip rows where `a = 1` but `f(b)` is false. +fn convert_not(not: &NotExpr) -> Option { + convert(not.arg()) + .filter(|inner| inner.exact) + .map(|inner| Converted::exact(inner.predicate.negate())) +} + +/// In a CASE, each WHEN picks which THEN decides if a row is kept. So every row the CASE +/// keeps passed some WHEN and its THEN, and this matches it: +/// `CASE WHEN c1 THEN x WHEN c2 THEN y ELSE z END` becomes `(c1 AND x) OR (c2 AND y) OR z`. +/// It ignores that the second branch only applies when `c1` is false, so it can match +/// extra rows. That's fine here, but a NOT over it could skip rows the filter keeps. +/// +/// `CASE a WHEN 1 THEN x ...` works the same way, with `a = 1` as the WHEN. +/// +/// Partitioned hash joins build filters like this, with one branch per partition: +/// `CASE hash(a) % 2 WHEN 0 THEN a IN (1, 2) WHEN 1 THEN a IN (3) ELSE false END`. +/// Iceberg can't compute the hash, so the WHENs are dropped, leaving +/// `a IN (1, 2) OR a IN (3)`. +/// +/// A missing ELSE means NULL, which matches nothing. +fn convert_case(case: &CaseExpr) -> Option { + let convert_when = |when| match case.expr() { + Some(value) => compare(value, Operator::Eq, when).map(Converted::exact), + None => convert(when), + }; + let predicate = case + .when_then_expr() + .iter() + .map(|(when, then)| both(convert_when(when), convert(then))) + .chain(case.else_expr().map(convert)) + .collect::>>()? + .into_iter() + .map(|branch| branch.predicate) + .reduce(Predicate::or)?; + Some(Converted::wider(predicate)) +} + +/// Converts a column compared with a value, like `a < 5`. +/// The value can come first too: `5 > a` becomes `a < 5`. +fn compare( + left: &Arc, + op: Operator, + right: &Arc, +) -> Option { + let (column, op, datum) = match (column_name(left), literal_datum(right)) { + (Some(column), Some(datum)) => (column, op, datum), + _ => (column_name(right)?, op.swap()?, literal_datum(left)?), + }; + let column = Reference::new(column); + match op { + Operator::Eq => Some(column.equal_to(datum)), + Operator::NotEq => Some(column.not_equal_to(datum)), + Operator::Lt => Some(column.less_than(datum)), + Operator::LtEq => Some(column.less_than_or_equal_to(datum)), + Operator::Gt => Some(column.greater_than(datum)), + Operator::GtEq => Some(column.greater_than_or_equal_to(datum)), + _ => None, + } +} + +fn convert_in_list(in_list: &InListExpr) -> Option { + // Something like `struct(a, b) IN (...)`, which checks several columns at once. + let Some(column) = column_name(in_list.expr()) else { + return convert_struct_in_list(in_list).map(Converted::wider); + }; + let column = Reference::new(column); + let values = in_list + .list() + .iter() + .map(literal_value) + .collect::>>()?; + let (nulls, values): (Vec<_>, Vec<_>) = values.into_iter().partition(|value| value.is_null()); + let datums = values + .into_iter() + .map(scalar_value_to_datum) + .collect::>>()?; + let predicate = match (in_list.negated(), nulls.is_empty()) { + // `x IN (1, NULL)` matches the same rows as `x IN (1)`. + (false, _) => column.is_in(datums), + (true, true) => column.is_not_in(datums), + // `x NOT IN (1, NULL)` never matches anything. It's rare, so we don't convert it. + (true, false) => return None, + }; + Some(Converted::exact(predicate)) +} + +/// Hash joins on two or more columns build filters like `struct(a, b) IN ({1, 3}, {2, 4})`. +/// Iceberg can't check pairs, so each column is checked on its own: +/// `a IN (1, 2) AND b IN (3, 4)`. This also matches pairs like `{1, 4}`, which is fine. +/// A column with a NULL value is left out, because the join decides if NULL matches NULL. +fn convert_struct_in_list(in_list: &InListExpr) -> Option { + if in_list.negated() { + return None; + } + let columns = in_list + .expr() + .downcast_ref::() + .filter(|function| function.name() == "struct")? + .args(); + let items = in_list + .list() + .iter() + .map(|item| match literal_value(item)? { + ScalarValue::Struct(item) => Some(item), + _ => None, + }) + .collect::>>()?; + columns + .iter() + .enumerate() + .filter_map(|(index, column)| { + let datums = items + .iter() + .map(|item| { + let value = ScalarValue::try_from_array(item.columns().get(index)?, 0).ok()?; + scalar_value_to_datum(&value) + }) + .collect::>>()?; + Some(Reference::new(column_name(column)?).is_in(datums)) + }) + .reduce(Predicate::and) +} + +/// Only simple prefix patterns: `a LIKE 'abc%'` becomes `a STARTS WITH 'abc'`. +/// ILIKE can't be converted, because iceberg's STARTS WITH is case-sensitive. +fn convert_like(like: &LikeExpr) -> Option { + if like.case_insensitive() { + return None; + } + let column = Reference::new(column_name(like.expr())?); + let prefix = string_literal(like.pattern())?.strip_suffix('%')?; + // Any other `%` or `_`, or a `\`, means it's more than a simple prefix. + if prefix.contains(['%', '_', '\\']) { + return None; + } + let prefix = prefix_datum(prefix)?; + Some(if like.negated() { + column.not_starts_with(prefix) + } else { + column.starts_with(prefix) + }) +} + +/// Functions are matched by name: `isnan(a)` and `starts_with(a, 'abc')`. +fn convert_function(function: &ScalarFunctionExpr) -> Option { + match (function.name(), function.args()) { + ("isnan", [arg]) => Some(Reference::new(column_name(arg)?).is_nan()), + ("starts_with", [arg, prefix]) => { + let prefix = prefix_datum(string_literal(prefix)?)?; + Some(Reference::new(column_name(arg)?).starts_with(prefix)) + } + _ => None, + } +} + +/// Iceberg counts bytes where it should count characters when it checks STARTS WITH against +/// a `truncate` partition. With `truncate(2)`, the prefix `é` is 2 bytes, so iceberg only +/// looks in the partition `é` and skips the partition `éa`, which also has matching rows. +/// ASCII text has one byte per character, so only ASCII prefixes are converted. +fn prefix_datum(prefix: &str) -> Option { + prefix.is_ascii().then(|| Datum::string(prefix)) +} + +fn column_name(expr: &Arc) -> Option<&str> { + expr.downcast_ref::().map(Column::name) +} + +fn literal_value(expr: &Arc) -> Option<&ScalarValue> { + expr.downcast_ref::().map(Literal::value) +} + +fn literal_datum(expr: &Arc) -> Option { + scalar_value_to_datum(literal_value(expr)?) +} + +fn string_literal(expr: &Arc) -> Option<&str> { + literal_value(expr)?.try_as_str().flatten() +} + +/// Converts a DataFusion value to an iceberg value, for the types iceberg columns use. +/// +/// Floats aren't converted. DataFusion sorts NaN above every number and -0.0 below 0.0. +/// Iceberg doesn't, so a float filter like `a > 1.0` could skip rows DataFusion keeps. +/// +/// Times aren't converted either, because iceberg-rust's file reader can't compare them yet. +fn scalar_value_to_datum(value: &ScalarValue) -> Option { + match value { + ScalarValue::Boolean(Some(v)) => Some(Datum::bool(*v)), + ScalarValue::Int32(Some(v)) => Some(Datum::int(*v)), + ScalarValue::Int64(Some(v)) => Some(Datum::long(*v)), + ScalarValue::Decimal128(Some(v), precision, scale) => { + let data_type = PrimitiveType::Decimal { + precision: u32::from(*precision), + scale: u32::try_from(*scale).ok()?, + }; + Datum::try_from_bytes(&v.to_be_bytes(), data_type).ok() + } + ScalarValue::Date32(Some(v)) => Some(Datum::date(*v)), + ScalarValue::TimestampMicrosecond(Some(v), None) => Some(Datum::timestamp_micros(*v)), + ScalarValue::TimestampMicrosecond(Some(v), Some(_)) => Some(Datum::timestamptz_micros(*v)), + ScalarValue::TimestampNanosecond(Some(v), None) => Some(Datum::timestamp_nanos(*v)), + ScalarValue::TimestampNanosecond(Some(v), Some(_)) => Some(Datum::timestamptz_nanos(*v)), + ScalarValue::Utf8(Some(v)) + | ScalarValue::LargeUtf8(Some(v)) + | ScalarValue::Utf8View(Some(v)) => Some(Datum::string(v)), + ScalarValue::Binary(Some(v)) + | ScalarValue::LargeBinary(Some(v)) + | ScalarValue::BinaryView(Some(v)) => Some(Datum::binary(v.clone())), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::arrow::array::{ArrayRef, Int64Array, StructArray}; + use datafusion::arrow::datatypes::{DataType, Field, Fields, Schema}; + use datafusion::common::{DFSchema, Result}; + use datafusion::config::ConfigOptions; + use datafusion::functions::core::r#struct; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{ + BinaryExpr, Column, InListExpr, NotExpr, col, in_list, lit, + }; + use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; + use datafusion::prelude::SessionContext; + use datafusion::scalar::ScalarValue; + use insta::assert_snapshot; + + use super::convert_physical_filter_to_predicate; + + // The filters below use the columns from `schema()`: `a` and `b` are integers, `s` is a + // string, `f` is a float and `flag` is a boolean. + + #[test] + fn converts_null_checks() { + assert_snapshot!(convert_sql("a IS NULL"), @"a IS NULL"); + assert_snapshot!(convert_sql("a IS NOT NULL"), @"a IS NOT NULL"); + } + + #[test] + fn converts_comparisons_with_a_literal() { + assert_snapshot!(convert_sql("a = 5"), @"a = 5"); + assert_snapshot!(convert_sql("a != 5"), @"a != 5"); + assert_snapshot!(convert_sql("a < 5"), @"a < 5"); + assert_snapshot!(convert_sql("a <= 5"), @"a <= 5"); + assert_snapshot!(convert_sql("a > 5"), @"a > 5"); + assert_snapshot!(convert_sql("a >= 5"), @"a >= 5"); + } + + #[test] + fn flips_comparisons_with_the_literal_on_the_left() { + assert_snapshot!(convert_sql("5 < a"), @"a > 5"); + assert_snapshot!(convert_sql("5 >= a"), @"a <= 5"); + } + + #[test] + fn leaves_comparisons_between_columns_unconverted() { + assert_snapshot!(convert_sql("a < b"), @"None"); + } + + #[test] + fn leaves_float_comparisons_unconverted() { + assert_snapshot!(convert_sql("f > 1.5"), @"None"); + assert_snapshot!(convert_sql("a = 1 AND f > 1.5"), @"a = 1"); + } + + #[test] + fn converts_in_lists() { + assert_snapshot!(convert_sql("a IN (1, 2)"), @"a IN (1, 2)"); + assert_snapshot!(convert_sql("a NOT IN (1, 2)"), @"a NOT IN (1, 2)"); + } + + #[test] + fn drops_nulls_from_an_in_list() -> Result<()> { + // Built by hand, the way a hash join builds it. SQL would wrap the NULL in a cast. + let schema = schema(); + let null = lit(ScalarValue::Int64(None)); + let values = vec![lit(1i64), Arc::clone(&null)]; + let one_or_null = in_list(col("a", &schema)?, values, &false, &schema)?; + let only_null = in_list(col("a", &schema)?, vec![null], &false, &schema)?; + assert_snapshot!(convert_filter(&one_or_null), @"a IN (1)"); + // `a IN (NULL)` matches nothing. + assert_snapshot!(convert_filter(&only_null), @"a IN ()"); + Ok(()) + } + + #[test] + fn leaves_not_in_with_a_null_unconverted() -> Result<()> { + // Built by hand, the way a hash join builds it. SQL would wrap the NULL in a cast. + let schema = schema(); + let values = vec![lit(1i64), lit(ScalarValue::Int64(None))]; + let filter = in_list(col("a", &schema)?, values, &true, &schema)?; + assert_snapshot!(convert_filter(&filter), @"None"); + Ok(()) + } + + #[test] + fn leaves_in_lists_with_a_column_item_unconverted() { + assert_snapshot!(convert_sql("a IN (b, 1)"), @"None"); + } + + #[test] + fn leaves_an_in_list_over_an_expression_unconverted() { + assert_snapshot!(convert_sql("a + 1 IN (1, 2)"), @"None"); + } + + #[test] + fn converts_like_prefixes() { + assert_snapshot!(convert_sql("s LIKE 'ab%'"), @r#"s STARTS WITH "ab""#); + assert_snapshot!(convert_sql("s NOT LIKE 'ab%'"), @r#"s NOT STARTS WITH "ab""#); + } + + #[test] + fn leaves_other_like_patterns_unconverted() { + assert_snapshot!(convert_sql("s LIKE 'a_b%'"), @"None"); + assert_snapshot!(convert_sql("s LIKE 'a%b%'"), @"None"); + assert_snapshot!(convert_sql(r"s LIKE 'a\b%'"), @"None"); + assert_snapshot!(convert_sql("s LIKE 'ab'"), @"None"); + assert_snapshot!(convert_sql("s ILIKE 'ab%'"), @"None"); + } + + #[test] + fn leaves_non_ascii_prefixes_unconverted() { + assert_snapshot!(convert_sql("s LIKE 'éa%'"), @"None"); + assert_snapshot!(convert_sql("starts_with(s, 'éa')"), @"None"); + } + + #[test] + fn converts_isnan_and_starts_with() { + assert_snapshot!(convert_sql("isnan(f)"), @"f IS NAN"); + assert_snapshot!(convert_sql("starts_with(s, 'ab')"), @r#"s STARTS WITH "ab""#); + } + + #[test] + fn converts_a_boolean_column() { + assert_snapshot!(convert_sql("flag"), @"flag = true"); + } + + #[test] + fn converts_true_false_and_null() { + assert_snapshot!(convert_sql("true"), @"TRUE"); + assert_snapshot!(convert_sql("false"), @"FALSE"); + // A NULL filter keeps no rows. + assert_snapshot!(convert_sql("NULL"), @"FALSE"); + } + + #[test] + fn converts_an_and() { + assert_snapshot!(convert_sql("a = 1 AND b = 2"), @"(a = 1) AND (b = 2)"); + } + + #[test] + fn drops_the_unconverted_side_of_an_and() { + assert_snapshot!(convert_sql("a = 1 AND a < b"), @"a = 1"); + } + + #[test] + fn converts_an_or_only_when_both_sides_convert() { + // Hash joins build this when rows with a NULL key can still match. + assert_snapshot!( + convert_sql("a IS NULL OR a IN (1, 2)"), + @"(a IS NULL) OR (a IN (1, 2))" + ); + assert_snapshot!(convert_sql("a = 1 OR a < b"), @"None"); + } + + #[test] + fn negates_exact_predicates() { + assert_snapshot!(convert_sql("NOT (a < 5)"), @"a >= 5"); + assert_snapshot!(convert_sql("NOT (a IS NULL)"), @"a IS NOT NULL"); + // DataFusion turns `NOT (a IN (1, 2))` into `a NOT IN (1, 2)` itself, and does the + // same for LIKE. Putting them inside an OR keeps the NOT. + assert_snapshot!( + convert_sql("NOT (a IN (1, 2) OR b = 2)"), + @"(a NOT IN (1, 2)) AND (b != 2)" + ); + assert_snapshot!( + convert_sql("NOT (s LIKE 'ab%' OR b = 2)"), + @r#"(s NOT STARTS WITH "ab") AND (b != 2)"# + ); + assert_snapshot!(convert_sql("NOT starts_with(s, 'ab')"), @r#"s NOT STARTS WITH "ab""#); + assert_snapshot!(convert_sql("NOT isnan(f)"), @"f IS NOT NAN"); + assert_snapshot!(convert_sql("NOT flag"), @"flag != true"); + assert_snapshot!(convert_sql("NOT (a = 1 AND b = 2)"), @"(a != 1) OR (b != 2)"); + assert_snapshot!(convert_sql("NOT (a = 1 OR b = 2)"), @"(a != 1) AND (b != 2)"); + } + + #[test] + fn leaves_not_over_a_partly_converted_filter_unconverted() { + assert_snapshot!(convert_sql("NOT (a = 1 AND a < b)"), @"None"); + assert_snapshot!(convert_sql("NOT (a = 1 AND (b = 2 AND a < b))"), @"None"); + assert_snapshot!(convert_sql("NOT (a = 1 OR (b = 2 AND a < b))"), @"None"); + } + + #[test] + fn converts_a_case_to_each_when_and_its_then_ored_together() { + assert_snapshot!( + convert_sql("CASE WHEN a = 1 THEN b = 2 WHEN a = 2 THEN b = 3 ELSE b = 4 END"), + @"(((a = 1) AND (b = 2)) OR ((a = 2) AND (b = 3))) OR (b = 4)" + ); + } + + #[test] + fn converts_case_value_whens_to_equalities() { + assert_snapshot!( + convert_sql("CASE a WHEN 1 THEN b = 2 WHEN 2 THEN b = 3 END"), + @"((a = 1) AND (b = 2)) OR ((a = 2) AND (b = 3))" + ); + } + + #[test] + fn keeps_only_the_thens_when_the_whens_cant_be_converted() { + // `a % 2` stands in for the `hash(a) % N` a partitioned hash join uses. + assert_snapshot!( + convert_sql("CASE a % 2 WHEN 0 THEN b IN (1, 2) WHEN 1 THEN b = 3 ELSE false END"), + @"(b IN (1, 2)) OR (b = 3)" + ); + } + + #[test] + fn keeps_the_when_if_its_then_cant_be_converted() { + assert_snapshot!(convert_sql("CASE WHEN a = 1 THEN a < b ELSE false END"), @"a = 1"); + } + + #[test] + fn converts_a_case_with_an_always_true_branch_to_true() { + // Hash joins use `true` for a partition whose build side was canceled. + assert_snapshot!(convert_sql("CASE a % 2 WHEN 0 THEN b = 1 ELSE true END"), @"TRUE"); + } + + #[test] + fn leaves_a_case_with_an_unconverted_branch_unconverted() { + assert_snapshot!( + convert_sql("CASE a % 2 WHEN 0 THEN b IN (1, 2) WHEN 1 THEN a < b ELSE false END"), + @"None" + ); + assert_snapshot!(convert_sql("CASE WHEN a = 1 THEN b = 2 ELSE a < b END"), @"None"); + } + + #[test] + fn leaves_not_over_a_case_unconverted() { + assert_snapshot!(convert_sql("NOT (CASE WHEN a = 1 THEN b = 2 ELSE false END)"), @"None"); + } + + #[test] + fn converts_a_multi_column_join_filter_to_an_in_list_per_column() -> Result<()> { + let filter = struct_in(vec![Some(1), Some(2)], vec![Some(5), Some(6)])?; + assert_snapshot!(convert_filter(&filter), @"(a IN (1, 2)) AND (b IN (5, 6))"); + Ok(()) + } + + #[test] + fn drops_join_filter_columns_with_a_null() -> Result<()> { + let filter = struct_in(vec![Some(1), Some(2)], vec![Some(3), None])?; + assert_snapshot!(convert_filter(&filter), @"a IN (1, 2)"); + Ok(()) + } + + #[test] + fn leaves_a_multi_column_not_in_unconverted() -> Result<()> { + let filter = struct_not_in(vec![Some(1), Some(2)], vec![Some(3), Some(4)])?; + assert_snapshot!(convert_filter(&filter), @"None"); + Ok(()) + } + + #[test] + fn leaves_not_over_a_multi_column_join_filter_unconverted() -> Result<()> { + let filter = struct_in(vec![Some(1), Some(2)], vec![Some(3), Some(4)])?; + let filter: Arc = Arc::new(NotExpr::new(filter)); + assert_snapshot!(convert_filter(&filter), @"None"); + Ok(()) + } + + #[test] + fn converts_literals_to_iceberg_values() { + // Each value is compared with `a`. The translator doesn't check that the types match. + let values = [ + ScalarValue::Boolean(Some(true)), + ScalarValue::Int32(Some(1)), + ScalarValue::Int64(Some(1)), + ScalarValue::Decimal128(Some(12345), 10, 2), + ScalarValue::Decimal128(Some(-12345), 10, 2), + ScalarValue::Date32(Some(19723)), + ScalarValue::TimestampMicrosecond(Some(1_704_067_200_000_000), None), + ScalarValue::TimestampMicrosecond(Some(1_704_067_200_000_000), Some("+00:00".into())), + ScalarValue::TimestampNanosecond(Some(1_704_067_200_000_000_000), None), + ScalarValue::TimestampNanosecond( + Some(1_704_067_200_000_000_000), + Some("+00:00".into()), + ), + ScalarValue::Utf8(Some("ab".into())), + ScalarValue::LargeUtf8(Some("ab".into())), + ScalarValue::Utf8View(Some("ab".into())), + ScalarValue::Binary(Some(vec![1, 2])), + ScalarValue::LargeBinary(Some(vec![1, 2])), + ScalarValue::BinaryView(Some(vec![1, 2])), + ScalarValue::Float32(Some(1.5)), + ScalarValue::Float64(Some(1.5)), + ScalarValue::Time64Microsecond(Some(3_600_000_000)), + ScalarValue::Int64(None), + ]; + let converted = values + .into_iter() + .map(|value| { + let column = Arc::new(Column::new("a", 0)); + let filter: Arc = + Arc::new(BinaryExpr::new(column, Operator::Eq, lit(value.clone()))); + format!("{value:?} => {}", convert_filter(&filter)) + }) + .collect::>() + .join("\n"); + assert_snapshot!(converted, @r#" + Boolean(true) => a = true + Int32(1) => a = 1 + Int64(1) => a = 1 + Decimal128(123.45,10,2) => a = 123.45 + Decimal128(-123.45,10,2) => a = -123.45 + Date32("2024-01-01") => a = 2024-01-01 + TimestampMicrosecond(1704067200000000, None) => a = 2024-01-01 00:00:00 + TimestampMicrosecond(1704067200000000, Some("+00:00")) => a = 2024-01-01 00:00:00 UTC + TimestampNanosecond(1704067200000000000, None) => a = 2024-01-01 00:00:00 + TimestampNanosecond(1704067200000000000, Some("+00:00")) => a = 2024-01-01 00:00:00 UTC + Utf8("ab") => a = "ab" + LargeUtf8("ab") => a = "ab" + Utf8View("ab") => a = "ab" + Binary("1,2") => a = 0102 + LargeBinary("1,2") => a = 0102 + BinaryView("1,2") => a = 0102 + Float32(1.5) => None + Float64(1.5) => None + Time64Microsecond("3600000000") => None + Int64(NULL) => None + "#); + } + + fn schema() -> Schema { + Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Int64, true), + Field::new("s", DataType::Utf8, true), + Field::new("f", DataType::Float64, true), + Field::new("flag", DataType::Boolean, true), + ]) + } + + /// Turns SQL like `a < 5` into a DataFusion filter over [`schema`], then converts it. + /// DataFusion's simplify step is skipped, so `NOT (a < 5)` stays a NOT. + fn convert_sql(filter: &str) -> String { + let schema = DFSchema::try_from(schema()).unwrap(); + let ctx = SessionContext::new(); + let expr = ctx.parse_sql_expr(filter, &schema).unwrap(); + let expr = ctx.state().create_physical_expr(expr, &schema).unwrap(); + convert_filter(&expr) + } + + /// Converts the filter and prints the predicate, or `None`. + fn convert_filter(filter: &Arc) -> String { + match convert_physical_filter_to_predicate(filter) { + Some(predicate) => predicate.to_string(), + None => "None".to_string(), + } + } + + /// Builds the filter a hash join on `a` and `b` makes: `struct(a, b) IN (...)`. + /// Each pair takes the values at the same position in `a` and `b`. + fn struct_in(a: Vec>, b: Vec>) -> Result> { + struct_in_list(a, b, false) + } + + /// Like [`struct_in`], but `NOT IN`. Hash joins never build this. + fn struct_not_in(a: Vec>, b: Vec>) -> Result> { + struct_in_list(a, b, true) + } + + fn struct_in_list( + a: Vec>, + b: Vec>, + negated: bool, + ) -> Result> { + let schema = schema(); + let fields = Fields::from(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Int64, true), + ]); + let columns = ScalarFunctionExpr::new( + "struct", + r#struct(), + vec![col("a", &schema)?, col("b", &schema)?], + Arc::new(Field::new("struct", DataType::Struct(fields.clone()), true)), + Arc::new(ConfigOptions::default()), + ); + let values = StructArray::new( + fields, + vec![ + Arc::new(Int64Array::from(a)) as ArrayRef, + Arc::new(Int64Array::from(b)) as ArrayRef, + ], + None, + ); + Ok(Arc::new(InListExpr::try_new_from_array( + Arc::new(columns), + Arc::new(values), + negated, + &schema, + )?)) + } +} From cb5531b42e63fc3c2b46a1db51cc2bd6e0880b2c Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:12:53 +0000 Subject: [PATCH 02/17] feat: add support for skipping partitions using dynamic filters --- .../src/common/dynamic_filters/mod.rs | 1 + .../dynamic_filters/partition_skipper.rs | 322 ++++++++++++++++++ .../distributed-iceberg/src/common/mod.rs | 2 +- .../src/common/scan_config.rs | 36 +- .../src/table_provider/mod.rs | 1 + .../physical_predicate.rs | 0 6 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs rename datafusion/distributed-iceberg/src/{common => table_provider}/physical_predicate.rs (100%) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs new file mode 100644 index 000000000..3000e3b2f --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -0,0 +1 @@ +pub(crate) mod partition_skipper; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs new file mode 100644 index 000000000..49f80bec1 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs @@ -0,0 +1,322 @@ +use std::cmp::Ordering; +use std::sync::Arc; + +use datafusion::common::HashMap; +use datafusion::physical_expr::{PhysicalExpr, conjunction}; +use datafusion::physical_expr_common::physical_expr::{ + snapshot_generation, snapshot_physical_expr, +}; +use iceberg::expr::{Bind, BoundPredicate, BoundReference, Predicate, PredicateOperator}; +use iceberg::scan::FileScanTask; +use iceberg::spec::{ + Datum, Literal, PartitionSpec, PrimitiveLiteral, PrimitiveType, Schema, Struct, Type, +}; +use iceberg::{Error, ErrorKind}; + +use crate::table_provider::physical_predicate::convert_physical_filter_to_predicate; + +/// Skips files whose partition value can't match the scan's dynamic filters. +pub(crate) struct PartitionSkipper { + /// The dynamic filters that will be used to evaluate whether a task can be skipped based on it's partition keys + dynamic_filters: Arc, + + /// The generation of `dynamic_filters` that `predicate` was built from. + cached_snapshot_generation: u64, + + /// The filters as an Iceberg predicate bound to the table schema. + predicate: Option, + + /// `predicate` projected onto each partition spec and bound to its partition schema, by spec + /// id. `None` keeps every file written with that spec. Cleared when `predicate` changes. + partition_predicates: HashMap>, +} + +impl PartitionSkipper { + pub(crate) fn new(dynamic_filters: Vec>) -> Self { + Self { + dynamic_filters: conjunction(dynamic_filters), + cached_snapshot_generation: 0, + predicate: None, + partition_predicates: HashMap::new(), + } + } + + /// Whether no row in the task's partition can match the filters. Any error keeps the file. + pub(crate) fn should_skip_task(&mut self, task: &FileScanTask) -> bool { + self.refresh(task); + let Some(predicate) = &self.predicate else { + return false; + }; + let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { + return false; + }; + if spec.is_unpartitioned() { + return false; + } + // Every task in a scan has the same table schema, so the spec id is enough as a key. + let partition_predicate = self + .partition_predicates + .entry(spec.spec_id()) + .or_insert_with(|| project_onto_spec(predicate, spec, task.schema()).ok()); + let Some(partition_predicate) = partition_predicate else { + return false; + }; + !evaluate(partition_predicate, spec, partition).unwrap_or(true) + } + + /// Rebuilds the predicate when any of the filters has changed. + fn refresh(&mut self, task: &FileScanTask) { + // Read the generation before taking the snapshot. If a filter changes in between, the + // next file sees a newer generation and rebuilds. + let generation = snapshot_generation(&self.dynamic_filters); + if self.cached_snapshot_generation == generation { + return; + } + self.cached_snapshot_generation = generation; + self.partition_predicates.clear(); + self.predicate = snapshot_physical_expr(Arc::clone(&self.dynamic_filters)) + .ok() + .and_then(|snapshot| convert_physical_filter_to_predicate(&snapshot)) + .and_then(|predicate| { + predicate + .bind(task.schema_ref(), task.case_sensitive()) + .ok() + }); + } +} + +/// Projects a predicate on table columns onto a partition spec, then binds it to the spec's +/// partition schema. Binding gives its values the partition columns' types, so they compare +/// with the partition values. +fn project_onto_spec( + predicate: &BoundPredicate, + spec: &PartitionSpec, + table_schema: &Schema, +) -> iceberg::Result { + let partition_type = spec.partition_type(table_schema)?; + let partition_schema = Schema::builder() + .with_fields(partition_type.fields().iter().cloned()) + .build()?; + project(predicate, spec)?.bind(Arc::new(partition_schema), true) +} + +/// Rewrites a predicate on table columns into one on the spec's partition columns. The result +/// matches every partition that holds a matching row, and can match more. +/// +/// Mirrors iceberg-rust's `InclusiveProjection`, which is private. +fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> iceberg::Result { + Ok(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + project(left, spec)?.and(project(right, spec)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + project(left, spec)?.or(project(right, spec)?) + } + // `refresh` removes every NOT before binding. Keep the file if one shows up anyway. + BoundPredicate::Not(_) => Predicate::AlwaysTrue, + BoundPredicate::Unary(expr) => project_column(expr.term(), predicate, spec)?, + BoundPredicate::Binary(expr) => project_column(expr.term(), predicate, spec)?, + BoundPredicate::Set(expr) => project_column(expr.term(), predicate, spec)?, + }) +} + +/// Projects a predicate on one column through every partition field built from that column. +/// A column with no partition field gives `AlwaysTrue`. +fn project_column( + column: &BoundReference, + predicate: &BoundPredicate, + spec: &PartitionSpec, +) -> iceberg::Result { + spec.fields() + .iter() + .filter(|field| field.source_id == column.field().id) + .try_fold(Predicate::AlwaysTrue, |projected, field| { + // `None` means the transform can't express the predicate, so it keeps every file. + Ok(match field.transform.project(&field.name, predicate)? { + Some(part) => projected.and(part), + None => projected, + }) + }) +} + +/// Evaluates a predicate bound to the partition schema against one partition value. Null values +/// follow iceberg-rust's `ExpressionEvaluator`. +fn evaluate( + predicate: &BoundPredicate, + spec: &PartitionSpec, + partition: &Struct, +) -> iceberg::Result { + match predicate { + BoundPredicate::AlwaysTrue => Ok(true), + BoundPredicate::AlwaysFalse => Ok(false), + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + Ok(evaluate(left, spec, partition)? && evaluate(right, spec, partition)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + Ok(evaluate(left, spec, partition)? || evaluate(right, spec, partition)?) + } + // A projection never makes a NOT. Keep the file if one shows up anyway. + BoundPredicate::Not(_) => Ok(true), + BoundPredicate::Unary(expr) => { + let value = partition_value(expr.term(), spec, partition)?; + let is_nan = value.as_ref().is_some_and(Datum::is_nan); + Ok(match expr.op() { + PredicateOperator::IsNull => value.is_none(), + PredicateOperator::NotNull => value.is_some(), + PredicateOperator::IsNan => is_nan, + PredicateOperator::NotNan => !is_nan, + _ => true, + }) + } + BoundPredicate::Binary(expr) => { + let Some(value) = partition_value(expr.term(), spec, partition)? else { + return Ok(matches!( + expr.op(), + PredicateOperator::NotEq | PredicateOperator::NotStartsWith + )); + }; + let literal = expr.literal(); + Ok(match expr.op() { + PredicateOperator::StartsWith => starts_with(&value, literal)?, + PredicateOperator::NotStartsWith => !starts_with(&value, literal)?, + PredicateOperator::LessThan => compare(&value, literal)?.is_lt(), + PredicateOperator::LessThanOrEq => compare(&value, literal)?.is_le(), + PredicateOperator::GreaterThan => compare(&value, literal)?.is_gt(), + PredicateOperator::GreaterThanOrEq => compare(&value, literal)?.is_ge(), + PredicateOperator::Eq => compare(&value, literal)?.is_eq(), + PredicateOperator::NotEq => compare(&value, literal)?.is_ne(), + _ => true, + }) + } + BoundPredicate::Set(expr) => { + let Some(value) = partition_value(expr.term(), spec, partition)? else { + return Ok(matches!(expr.op(), PredicateOperator::NotIn)); + }; + if let Some(literal) = expr + .literals() + .iter() + .find(|literal| literal.data_type() != value.data_type()) + { + return Err(type_mismatch(&value, literal)); + } + let contains = expr.literals().contains(&value); + Ok(match expr.op() { + PredicateOperator::In => contains, + PredicateOperator::NotIn => !contains, + _ => true, + }) + } + } +} + +/// The partition value for a reference bound to the partition schema, with the partition +/// column's type. `None` is a null value. +fn partition_value( + column: &BoundReference, + spec: &PartitionSpec, + partition: &Struct, +) -> iceberg::Result> { + let field = column.field(); + let position = spec + .fields() + .iter() + .position(|partition_field| partition_field.field_id == field.id) + .ok_or_else(|| invalid(format!("partition field {} is not in the spec", field.id)))?; + let Some(value) = partition.fields().get(position) else { + return Err(invalid(format!("the partition has no value at {position}"))); + }; + let Some(value) = value else { + return Ok(None); + }; + let (Literal::Primitive(literal), Type::Primitive(r#type)) = (value, field.field_type.as_ref()) + else { + return Err(invalid(format!( + "partition field {} is not a primitive", + field.id + ))); + }; + datum(r#type, literal).map(Some) +} + +/// Builds a `Datum` of type `r#type`. Also widens values written before a column's type was +/// promoted, such as an int partition value of a column that is now a long. +fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> iceberg::Result { + Ok(match (r#type, literal) { + (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => Datum::bool(*value), + (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => Datum::int(*value), + (PrimitiveType::Long, PrimitiveLiteral::Int(value)) => Datum::long(*value), + (PrimitiveType::Long, PrimitiveLiteral::Long(value)) => Datum::long(*value), + (PrimitiveType::Float, PrimitiveLiteral::Float(value)) => Datum::float(value.0), + (PrimitiveType::Double, PrimitiveLiteral::Float(value)) => Datum::double(value.0), + (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => Datum::double(value.0), + (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => Datum::date(*value), + (PrimitiveType::Time, PrimitiveLiteral::Long(value)) => Datum::time_micros(*value)?, + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(value)) => { + Datum::timestamp_micros(*value) + } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(value)) => { + Datum::timestamptz_micros(*value) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(value)) => { + Datum::timestamp_nanos(*value) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(value)) => { + Datum::timestamptz_nanos(*value) + } + (PrimitiveType::String, PrimitiveLiteral::String(value)) => Datum::string(value), + (PrimitiveType::Binary, PrimitiveLiteral::Binary(value)) => { + Datum::binary(value.iter().copied()) + } + (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(value)) => { + Datum::fixed(value.iter().copied()) + } + // Decimal and uuid have no public constructor that takes the raw value, so these go + // through their byte encoding. + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(value)) => { + Datum::try_from_bytes(&value.to_be_bytes(), r#type.clone())? + } + (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => { + Datum::try_from_bytes(&value.to_be_bytes(), PrimitiveType::Uuid)? + } + // A value that doesn't fit its column's type. Keep the file. + _ => return Err(invalid(format!("can't read a {type} partition value"))), + }) +} + +/// Orders two values of the same type. Values of different types are an error, so the file is +/// kept. +fn compare(value: &Datum, literal: &Datum) -> iceberg::Result { + if value.data_type() != literal.data_type() { + return Err(type_mismatch(value, literal)); + } + value + .partial_cmp(literal) + .ok_or_else(|| type_mismatch(value, literal)) +} + +fn starts_with(value: &Datum, prefix: &Datum) -> iceberg::Result { + match (value.literal(), prefix.literal()) { + (PrimitiveLiteral::String(value), PrimitiveLiteral::String(prefix)) => { + Ok(value.starts_with(prefix.as_str())) + } + _ => Err(type_mismatch(value, prefix)), + } +} + +fn type_mismatch(value: &Datum, literal: &Datum) -> Error { + invalid(format!( + "can't compare a {} partition value with a {} value", + value.data_type(), + literal.data_type() + )) +} + +fn invalid(message: String) -> Error { + Error::new(ErrorKind::DataInvalid, message) +} diff --git a/datafusion/distributed-iceberg/src/common/mod.rs b/datafusion/distributed-iceberg/src/common/mod.rs index cce214161..984b811f8 100644 --- a/datafusion/distributed-iceberg/src/common/mod.rs +++ b/datafusion/distributed-iceberg/src/common/mod.rs @@ -2,7 +2,7 @@ mod error; pub(crate) use error::{df_err, iceberg_err}; +mod dynamic_filters; mod file_partition; -mod physical_predicate; pub(crate) mod scan_config; pub(crate) use file_partition::file_partition; diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 71eebfe19..58edd5206 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -1,3 +1,4 @@ +use crate::common::dynamic_filters::partition_skipper::PartitionSkipper; use crate::{IcebergConfig, common::df_err}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::stats::Precision; @@ -20,9 +21,10 @@ use futures::stream::poll_fn; use futures::{StreamExt, TryStreamExt}; use iceberg::arrow::{ArrowReader, ArrowReaderBuilder}; use iceberg::io::FileIO; -use iceberg::scan::FileScanTaskStream; +use iceberg::scan::{FileScanTask, FileScanTaskStream}; use iceberg::spec::SnapshotRef; use std::fmt; +use std::future::ready; use std::sync::Arc; /// Execution settings shared by assigned-file and feed-backed scans. @@ -149,9 +151,15 @@ impl ScanConfig { .with_category(MetricCategory::Bytes) .gauge("bytes_scanned", partition); - // The reader pulls a task when it starts reading that task's file. It reads ahead, - // so with a LIMIT this can count files that never produced output. - let tasks = tasks.inspect_ok(move |_| files_opened.add(1)).boxed(); + let dynamic_filters = self.dynamic_filters(); + let mut partition_skipper = PartitionSkipper::new(dynamic_filters); + + let tasks = tasks + .try_filter(move |task| ready(!partition_skipper.should_skip_task(task))) + .inspect_ok(move |_| { + files_opened.add(1); + }) + .boxed(); let result = self.reader.clone().read(tasks).map_err(df_err)?; let reader_metrics = result.metrics().clone(); @@ -173,6 +181,26 @@ impl ScanConfig { Ok(Box::pin(LimitStream::new(stream, 0, self.fetch, metrics))) } + + fn dynamic_filters(&self) -> Vec> { + if !self.dynamic_filtering_enabled { + return vec![]; + } + + let Some(filters) = &self.filters else { + return vec![]; + }; + + split_conjunction(filters) + .into_iter() + .filter(|filter| DynamicFilterTracking::classify(filter).contains_dynamic_filter()) + .cloned() + .collect() + } +} + +fn should_skip_task(task: &FileScanTask, dynamic_filters: &Vec>) -> bool { + return false; } impl fmt::Debug for ScanConfig { diff --git a/datafusion/distributed-iceberg/src/table_provider/mod.rs b/datafusion/distributed-iceberg/src/table_provider/mod.rs index 4f8a5471e..bc2762faa 100644 --- a/datafusion/distributed-iceberg/src/table_provider/mod.rs +++ b/datafusion/distributed-iceberg/src/table_provider/mod.rs @@ -1,6 +1,7 @@ mod catalog; mod column_statistics; mod factory; +pub(crate) mod physical_predicate; pub(crate) mod scan; mod r#static; diff --git a/datafusion/distributed-iceberg/src/common/physical_predicate.rs b/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs similarity index 100% rename from datafusion/distributed-iceberg/src/common/physical_predicate.rs rename to datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs From 9d08b2598270ecbcbd09da59bbecada1368ac3d3 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:16:29 +0000 Subject: [PATCH 03/17] nit: reword a comment the spell check flags --- .../src/table_provider/physical_predicate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs b/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs index b735d7692..fb6f927d1 100644 --- a/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs +++ b/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs @@ -138,7 +138,7 @@ fn convert_not(not: &NotExpr) -> Option { /// /// Partitioned hash joins build filters like this, with one branch per partition: /// `CASE hash(a) % 2 WHEN 0 THEN a IN (1, 2) WHEN 1 THEN a IN (3) ELSE false END`. -/// Iceberg can't compute the hash, so the WHENs are dropped, leaving +/// Iceberg can't compute the hash, so the WHEN conditions are dropped, leaving /// `a IN (1, 2) OR a IN (3)`. /// /// A missing ELSE means NULL, which matches nothing. From 9159ab9bf3df2325d1d06340f30e719b9e028090 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:01:10 +0000 Subject: [PATCH 04/17] nit: remove the unused skip stub, do less work per file, fix comments --- .../dynamic_filters/partition_skipper.rs | 42 ++++++++----- .../src/common/scan_config.rs | 63 ++++++++++--------- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs index 49f80bec1..957d4d2ec 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs @@ -17,7 +17,7 @@ use crate::table_provider::physical_predicate::convert_physical_filter_to_predic /// Skips files whose partition value can't match the scan's dynamic filters. pub(crate) struct PartitionSkipper { - /// The dynamic filters that will be used to evaluate whether a task can be skipped based on it's partition keys + /// The scan's dynamic filters, combined with AND. dynamic_filters: Arc, /// The generation of `dynamic_filters` that `predicate` was built from. @@ -27,7 +27,9 @@ pub(crate) struct PartitionSkipper { predicate: Option, /// `predicate` projected onto each partition spec and bound to its partition schema, by spec - /// id. `None` keeps every file written with that spec. Cleared when `predicate` changes. + /// id. `None` keeps every file written with that spec. That happens when the projection + /// matches every partition, or when the spec's partition values can't be checked. Cleared + /// when `predicate` changes. partition_predicates: HashMap>, } @@ -43,25 +45,37 @@ impl PartitionSkipper { /// Whether no row in the task's partition can match the filters. Any error keeps the file. pub(crate) fn should_skip_task(&mut self, task: &FileScanTask) -> bool { - self.refresh(task); - let Some(predicate) = &self.predicate else { - return false; - }; let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { return false; }; if spec.is_unpartitioned() { return false; } + self.refresh(task); + let Some(predicate) = &self.predicate else { + return false; + }; // Every task in a scan has the same table schema, so the spec id is enough as a key. let partition_predicate = self .partition_predicates .entry(spec.spec_id()) - .or_insert_with(|| project_onto_spec(predicate, spec, task.schema()).ok()); - let Some(partition_predicate) = partition_predicate else { + .or_insert_with(|| { + project_onto_spec(predicate, spec, task.schema()) + .ok() + .filter(|projected| !matches!(projected, BoundPredicate::AlwaysTrue)) + }); + let Some(projected) = partition_predicate.as_ref() else { return false; }; - !evaluate(partition_predicate, spec, partition).unwrap_or(true) + match evaluate(projected, spec, partition) { + Ok(matches) => !matches, + // Files with the same spec would most likely fail the same way, so stop checking + // them until the filters change. + Err(_) => { + *partition_predicate = None; + false + } + } } /// Rebuilds the predicate when any of the filters has changed. @@ -116,7 +130,8 @@ fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> iceberg::Result< let [left, right] = expr.inputs(); project(left, spec)?.or(project(right, spec)?) } - // `refresh` removes every NOT before binding. Keep the file if one shows up anyway. + // The filter translator turns each NOT into the opposite comparison, so none reach + // here. Keep the file if one shows up anyway. BoundPredicate::Not(_) => Predicate::AlwaysTrue, BoundPredicate::Unary(expr) => project_column(expr.term(), predicate, spec)?, BoundPredicate::Binary(expr) => project_column(expr.term(), predicate, spec)?, @@ -198,10 +213,9 @@ fn evaluate( let Some(value) = partition_value(expr.term(), spec, partition)? else { return Ok(matches!(expr.op(), PredicateOperator::NotIn)); }; - if let Some(literal) = expr - .literals() - .iter() - .find(|literal| literal.data_type() != value.data_type()) + // Binding gives every value in the set the column's type, so checking one is enough. + if let Some(literal) = expr.literals().iter().next() + && literal.data_type() != value.data_type() { return Err(type_mismatch(&value, literal)); } diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 58edd5206..580e3656e 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -6,10 +6,8 @@ use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::common::{ColumnStatistics, Statistics}; use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::expressions::{InListExpr, lit}; -use datafusion::physical_expr::{ - DynamicFilterTracking, Partitioning, PhysicalExpr, conjunction, split_conjunction, -}; +use datafusion::physical_expr::expressions::{DynamicFilterPhysicalExpr, InListExpr, lit}; +use datafusion::physical_expr::{Partitioning, PhysicalExpr, conjunction, split_conjunction}; use datafusion::physical_expr_common::physical_expr::snapshot_physical_expr; use datafusion::physical_plan::apply_expression_roots; use datafusion::physical_plan::limit::LimitStream; @@ -21,7 +19,7 @@ use futures::stream::poll_fn; use futures::{StreamExt, TryStreamExt}; use iceberg::arrow::{ArrowReader, ArrowReaderBuilder}; use iceberg::io::FileIO; -use iceberg::scan::{FileScanTask, FileScanTaskStream}; +use iceberg::scan::FileScanTaskStream; use iceberg::spec::SnapshotRef; use std::fmt; use std::future::ready; @@ -98,11 +96,7 @@ impl ScanConfig { let Some(filters) = &self.filters else { return Ok(()); }; - let dynamic_filters: Vec<_> = split_conjunction(filters) - .into_iter() - .filter(|filter| DynamicFilterTracking::classify(filter).contains_dynamic_filter()) - .cloned() - .collect(); + let dynamic_filters = dynamic_filters(filters); if dynamic_filters.is_empty() { return Ok(()); } @@ -151,14 +145,18 @@ impl ScanConfig { .with_category(MetricCategory::Bytes) .gauge("bytes_scanned", partition); - let dynamic_filters = self.dynamic_filters(); - let mut partition_skipper = PartitionSkipper::new(dynamic_filters); - + let mut partition_skipper = self.partition_skipper(); + // The reader pulls a task when it starts reading that task's file. It reads ahead, + // so with a LIMIT this can count files that never produced output. Skipped files + // are dropped before this, so they aren't counted. let tasks = tasks - .try_filter(move |task| ready(!partition_skipper.should_skip_task(task))) - .inspect_ok(move |_| { - files_opened.add(1); + .try_filter(move |task| { + let skip = partition_skipper + .as_mut() + .is_some_and(|skipper| skipper.should_skip_task(task)); + ready(!skip) }) + .inspect_ok(move |_| files_opened.add(1)) .boxed(); let result = self.reader.clone().read(tasks).map_err(df_err)?; @@ -182,25 +180,30 @@ impl ScanConfig { Ok(Box::pin(LimitStream::new(stream, 0, self.fetch, metrics))) } - fn dynamic_filters(&self) -> Vec> { + /// A skipper over the scan's dynamic filters. `None` when dynamic filtering is off or the + /// scan has no dynamic filter. + fn partition_skipper(&self) -> Option { if !self.dynamic_filtering_enabled { - return vec![]; + return None; } - - let Some(filters) = &self.filters else { - return vec![]; - }; - - split_conjunction(filters) - .into_iter() - .filter(|filter| DynamicFilterTracking::classify(filter).contains_dynamic_filter()) - .cloned() - .collect() + let dynamic_filters = dynamic_filters(self.filters.as_ref()?); + (!dynamic_filters.is_empty()).then(|| PartitionSkipper::new(dynamic_filters)) } } -fn should_skip_task(task: &FileScanTask, dynamic_filters: &Vec>) -> bool { - return false; +/// The parts of `filters` that contain a dynamic filter. It only looks at the expressions, so +/// it doesn't subscribe to the filters' updates. +fn dynamic_filters(filters: &Arc) -> Vec> { + split_conjunction(filters) + .into_iter() + .filter(|filter| { + matches!( + filter.exists(|expr| Ok(expr.is::())), + Ok(true) + ) + }) + .cloned() + .collect() } impl fmt::Debug for ScanConfig { From 07e0d1b725c93b415f91aec1561b95fae867720c Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:01:16 +0000 Subject: [PATCH 05/17] test: show join filters skipping partitions on workers --- .../tests/partition_skipping.rs | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 datafusion/distributed-iceberg/tests/partition_skipping.rs diff --git a/datafusion/distributed-iceberg/tests/partition_skipping.rs b/datafusion/distributed-iceberg/tests/partition_skipping.rs new file mode 100644 index 000000000..80accbd2b --- /dev/null +++ b/datafusion/distributed-iceberg/tests/partition_skipping.rs @@ -0,0 +1,87 @@ +mod tests { + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::common::Result; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::collect; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::IcebergTestHarness; + use datafusion_distributed_iceberg_test_tools::create_taxi_table; + use std::sync::Arc; + + // The taxi table has one file per day, for the 7 days from 2024-01-08. The join keeps two of + // them, so its filter lets the scan skip the other 5 files. + const TWO_DAYS_OF_TRIPS: &str = "SELECT COUNT(*) AS trips FROM taxi \ + JOIN (VALUES (DATE '2024-01-10'), (DATE '2024-01-12')) AS days(day) \ + ON taxi.pickup_date = days.day"; + + #[tokio::test] + async fn lazy_planning_workers_skip_files_the_join_filter_rules_out() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let plan = harness.physical_plan(TWO_DAYS_OF_TRIPS).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------+ + | trips | + +-------+ + | 50000 | + +-------+ + "); + // The first `DataSourceExec` is the join's `VALUES` table. The second is the taxi scan. + assert_eq!(node_metrics::(&plan, "files_opened", 1), 2); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_skip_files_the_join_filter_rules_out() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let plan = harness.physical_plan(TWO_DAYS_OF_TRIPS).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------+ + | trips | + +-------+ + | 50000 | + +-------+ + "); + assert_eq!( + node_metrics::(&plan, "files_opened", 0), + 2 + ); + Ok(()) + } +} From 10037c6f1c41902cbfb98716831651cf02eb201c Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:21:58 +0000 Subject: [PATCH 06/17] test: cover top-N, month partitions and null partitions in partition skipping --- .../tests/partition_skipping.rs | 172 +++++++++++++++++- 1 file changed, 171 insertions(+), 1 deletion(-) diff --git a/datafusion/distributed-iceberg/tests/partition_skipping.rs b/datafusion/distributed-iceberg/tests/partition_skipping.rs index 80accbd2b..c37ce9a94 100644 --- a/datafusion/distributed-iceberg/tests/partition_skipping.rs +++ b/datafusion/distributed-iceberg/tests/partition_skipping.rs @@ -1,4 +1,5 @@ mod tests { + use datafusion::arrow::array::{ArrayRef, Date32Array, Int32Array, RecordBatch}; use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::common::Result; use datafusion::datasource::source::DataSourceExec; @@ -9,8 +10,13 @@ mod tests { rewrite_distributed_plan_with_metrics, }; use datafusion_distributed_iceberg::IcebergExt; - use datafusion_distributed_iceberg_test_tools::IcebergTestHarness; use datafusion_distributed_iceberg_test_tools::create_taxi_table; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Transform, Type}; + use std::error::Error; use std::sync::Arc; // The taxi table has one file per day, for the 7 days from 2024-01-08. The join keeps two of @@ -84,4 +90,168 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn top_n_skips_files_once_it_fills_up() -> Result<()> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(create_taxi_table().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let plan = harness + .physical_plan("SELECT pickup_date FROM taxi ORDER BY pickup_date LIMIT 3") + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +-------------+ + | pickup_date | + +-------------+ + | 2024-01-08 | + | 2024-01-08 | + | 2024-01-08 | + +-------------+ + "); + // The first file, 2024-01-08, fills the top 3. The filter then becomes + // `pickup_date < 2024-01-08`, which rules out the other 6 days. + assert_eq!(node_metrics::(&plan, "files_opened", 0), 1); + Ok(()) + } + + #[tokio::test] + async fn month_partitions_skip_files_from_other_months() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_month().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let plan = harness + .physical_plan( + "SELECT events.day FROM events \ + JOIN (VALUES (DATE '2024-02-14')) AS days(day) ON events.day = days.day", + ) + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +------------+ + | day | + +------------+ + | 2024-02-14 | + +------------+ + "); + // The first `DataSourceExec` is the join's `VALUES` table. The second is the events scan, + // which opens only February's file. + assert_eq!(node_metrics::(&plan, "files_opened", 1), 1); + Ok(()) + } + + #[tokio::test] + async fn null_partitions_are_read_when_the_filter_can_match_null() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(groups_by_key().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let plan = harness + .physical_plan("SELECT k FROM groups ORDER BY k NULLS FIRST LIMIT 2") + .await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + + insta::assert_snapshot!(pretty_format_batches(&batches)?, @" + +---+ + | k | + +---+ + | | + | | + +---+ + "); + // The scan reads `k = 1` first, and top-N's filter becomes `k IS NULL OR k < 1`. That + // skips the `k = 2` file and keeps the null file. + assert_eq!(node_metrics::(&plan, "files_opened", 0), 2); + Ok(()) + } + + // Table `events`, partitioned by the month of `day`, with one file for each of January, + // February and March 2024. + async fn events_by_month() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "day", Type::Primitive(PrimitiveType::Date)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .partition_by("day", Transform::Month) + .create() + .await?; + // Days since 1970-01-01. + let january = [19727, 19742]; // 2024-01-05, 2024-01-20 + let february = [19756, 19767]; // 2024-02-03, 2024-02-14 + let march = [19783]; // 2024-03-01 + let mut append = table.append(); + for days in [&january[..], &february[..], &march[..]] { + let days: ArrayRef = Arc::new(Date32Array::from(days.to_vec())); + append = append.file(RecordBatch::try_from_iter([("day", days)])?); + } + append.commit().await?; + Ok(table) + } + + // Table `groups`, partitioned by `k`, with one file each for `k = 1`, `k = 2` and a null `k`, + // in that order. Each file has two rows. + async fn groups_by_key() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "k", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "groups"])?, schema) + .partition_by("k", Transform::Identity) + .create() + .await?; + let mut append = table.append(); + for k in [Some(1), Some(2), None] { + let keys: ArrayRef = Arc::new(Int32Array::from(vec![k, k])); + append = append.file(RecordBatch::try_from_iter([("k", keys)])?); + } + append.commit().await?; + Ok(table) + } } From 2311943c9f7918694e36f6e04db80d08dbc4c27e Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:19:05 +0000 Subject: [PATCH 07/17] refactor: move the physical filter converter under common/dynamic_filters The converter only depends on DataFusion and iceberg, so it belongs with the shared helpers. The partition skipper in common/ no longer imports from table_provider/. --- .../distributed-iceberg/src/common/dynamic_filters/mod.rs | 1 + .../src/common/dynamic_filters/partition_skipper.rs | 2 +- .../dynamic_filters}/physical_predicate.rs | 0 datafusion/distributed-iceberg/src/table_provider/mod.rs | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) rename datafusion/distributed-iceberg/src/{table_provider => common/dynamic_filters}/physical_predicate.rs (100%) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 3000e3b2f..1245e5ec7 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1 +1,2 @@ pub(crate) mod partition_skipper; +pub(crate) mod physical_predicate; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs index 957d4d2ec..aef6786de 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs @@ -13,7 +13,7 @@ use iceberg::spec::{ }; use iceberg::{Error, ErrorKind}; -use crate::table_provider::physical_predicate::convert_physical_filter_to_predicate; +use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; /// Skips files whose partition value can't match the scan's dynamic filters. pub(crate) struct PartitionSkipper { diff --git a/datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs similarity index 100% rename from datafusion/distributed-iceberg/src/table_provider/physical_predicate.rs rename to datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs diff --git a/datafusion/distributed-iceberg/src/table_provider/mod.rs b/datafusion/distributed-iceberg/src/table_provider/mod.rs index bc2762faa..4f8a5471e 100644 --- a/datafusion/distributed-iceberg/src/table_provider/mod.rs +++ b/datafusion/distributed-iceberg/src/table_provider/mod.rs @@ -1,7 +1,6 @@ mod catalog; mod column_statistics; mod factory; -pub(crate) mod physical_predicate; pub(crate) mod scan; mod r#static; From 4bfadd487e7470f6e21afdab415209683d80a30f Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:22:40 +0000 Subject: [PATCH 08/17] feat: count the files that partition skipping keeps and skips The partition skipper now reports files_pruned_by_partition, which shows how many files it checked and how many it kept. --- .../dynamic_filters/partition_skipper.rs | 25 ++++++++++++++++++- .../src/common/scan_config.rs | 9 ++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs index aef6786de..9c239a22a 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs @@ -6,6 +6,9 @@ use datafusion::physical_expr::{PhysicalExpr, conjunction}; use datafusion::physical_expr_common::physical_expr::{ snapshot_generation, snapshot_physical_expr, }; +use datafusion::physical_plan::metrics::{ + ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, +}; use iceberg::expr::{Bind, BoundPredicate, BoundReference, Predicate, PredicateOperator}; use iceberg::scan::FileScanTask; use iceberg::spec::{ @@ -31,20 +34,40 @@ pub(crate) struct PartitionSkipper { /// matches every partition, or when the spec's partition values can't be checked. Cleared /// when `predicate` changes. partition_predicates: HashMap>, + + /// The files checked. Pruned files were skipped and matched files were kept. + files: PruningMetrics, } impl PartitionSkipper { - pub(crate) fn new(dynamic_filters: Vec>) -> Self { + pub(crate) fn new( + dynamic_filters: Vec>, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + ) -> Self { Self { dynamic_filters: conjunction(dynamic_filters), cached_snapshot_generation: 0, predicate: None, partition_predicates: HashMap::new(), + files: MetricBuilder::new(metrics) + .with_type(MetricType::Summary) + .pruning_metrics("files_pruned_by_partition", partition), } } /// Whether no row in the task's partition can match the filters. Any error keeps the file. pub(crate) fn should_skip_task(&mut self, task: &FileScanTask) -> bool { + let skip = self.can_skip(task); + if skip { + self.files.add_pruned(1); + } else { + self.files.add_matched(1); + } + skip + } + + fn can_skip(&mut self, task: &FileScanTask) -> bool { let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { return false; }; diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 580e3656e..dcf4344c9 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -145,10 +145,10 @@ impl ScanConfig { .with_category(MetricCategory::Bytes) .gauge("bytes_scanned", partition); - let mut partition_skipper = self.partition_skipper(); + let mut partition_skipper = self.partition_skipper(partition); // The reader pulls a task when it starts reading that task's file. It reads ahead, // so with a LIMIT this can count files that never produced output. Skipped files - // are dropped before this, so they aren't counted. + // are dropped before this, and the skipper counts them. let tasks = tasks .try_filter(move |task| { let skip = partition_skipper @@ -182,12 +182,13 @@ impl ScanConfig { /// A skipper over the scan's dynamic filters. `None` when dynamic filtering is off or the /// scan has no dynamic filter. - fn partition_skipper(&self) -> Option { + fn partition_skipper(&self, partition: usize) -> Option { if !self.dynamic_filtering_enabled { return None; } let dynamic_filters = dynamic_filters(self.filters.as_ref()?); - (!dynamic_filters.is_empty()).then(|| PartitionSkipper::new(dynamic_filters)) + (!dynamic_filters.is_empty()) + .then(|| PartitionSkipper::new(dynamic_filters, &self.metrics, partition)) } } From 42a0ed731f96230c8e1fcfe748a58a6d74131f8f Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:22:41 +0000 Subject: [PATCH 09/17] test: compare query results with partition skipping on and off Runs joins and top-N queries over identity, truncate, month, bucket and day partitions, with dynamic filtering on and off, and checks they return the same rows. The cases use keys on partition edges, keys in no partition, and nulls. A snapshot shows how many files each case kept. --- .../tests/partition_skipping.rs | 261 +++++++++++++++++- 1 file changed, 259 insertions(+), 2 deletions(-) diff --git a/datafusion/distributed-iceberg/tests/partition_skipping.rs b/datafusion/distributed-iceberg/tests/partition_skipping.rs index c37ce9a94..bd6fc1f68 100644 --- a/datafusion/distributed-iceberg/tests/partition_skipping.rs +++ b/datafusion/distributed-iceberg/tests/partition_skipping.rs @@ -1,9 +1,14 @@ mod tests { - use datafusion::arrow::array::{ArrayRef, Date32Array, Int32Array, RecordBatch}; + use datafusion::arrow::array::{ + ArrayRef, Date32Array, Int32Array, Int64Array, RecordBatch, StringArray, + TimestampMicrosecondArray, + }; + use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::common::Result; use datafusion::datasource::source::DataSourceExec; - use datafusion::physical_plan::collect; + use datafusion::physical_plan::metrics::MetricValue; + use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion_distributed::test_utils::metrics::node_metrics; use datafusion_distributed::{ DistributedExt, DistributedLeafExec, DistributedMetricsFormat, @@ -205,6 +210,140 @@ mod tests { Ok(()) } + #[tokio::test] + async fn validate_predicate_translation() -> Result<(), Box> { + let skipping = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(readings().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so top-N's filter tightens before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + // A join with more than 2 keys filters only on their bounds, so the cases cover + // both kinds of join filter. + .with_config_option( + "datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values", + 2, + ) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + let reading_every_file = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(readings().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .with_config_option( + "datafusion.optimizer.hash_join_inlist_pushdown_max_distinct_values", + 2, + ) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + // Keys on partition edges, keys in no partition, and nulls, for each transform. + let schema = skipping.table().schema()?; + let cases = [ + join(&schema, "id", &["3"]), + join(&schema, "id", &["0"]), + join(&schema, "id", &["2", "6"]), + join(&schema, "id", &["2", "3", "4"]), + join(&schema, "id", &["NULL"]), + join(&schema, "id", &["1", "NULL"]), + join(&schema, "name", &["'avocado'"]), + join(&schema, "name", &["'b'"]), + join(&schema, "name", &["'éclair'"]), + join(&schema, "name", &["'é'"]), + join(&schema, "name", &["'zebra'"]), + join(&schema, "day", &["'2024-02-01'"]), + join(&schema, "day", &["'2024-01-15'"]), + join(&schema, "day", &["'2024-04-01'"]), + join(&schema, "day", &["'2023-12-31'", "'2024-03-01'"]), + join(&schema, "n", &["12"]), + join(&schema, "n", &["99"]), + join(&schema, "n", &["-1", "13"]), + join(&schema, "at", &["'2024-01-02T00:00:00'"]), + join(&schema, "at", &["'2024-01-01T23:59:59.999999'"]), + join(&schema, "at", &["'2024-01-04T00:00:00'"]), + top_n("id", 1), + top_n("id DESC", 1), + top_n("id NULLS FIRST", 2), + top_n("id DESC NULLS LAST", 2), + top_n("name", 1), + top_n("name DESC", 1), + top_n("day", 2), + top_n("day DESC", 1), + top_n("n", 1), + top_n("at", 1), + top_n("at DESC", 2), + ]; + + // Each case must return the same rows whether the scan skips files or reads them all. + // The snapshot shows how many files each case kept. + let mut skipped = Vec::new(); + for (case, sql) in cases { + let plan = skipping.physical_plan(&sql).await?; + let rows = collect(Arc::clone(&plan), skipping.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + let expected = reading_every_file.query(&sql).await?.1; + assert_eq!( + pretty_format_batches(&rows)?.to_string(), + expected, + "{case}" + ); + let files = scan_metric(&plan, "files_pruned_by_partition") + .unwrap_or_else(|| panic!("{case}: the scan has no dynamic filter")); + let rows: usize = rows.iter().map(RecordBatch::num_rows).sum(); + skipped.push(format!("{case}: rows: {rows}, files: {files}")); + } + insta::assert_snapshot!(skipped.join("\n"), @" + join on id with 3: rows: 1, files: 6 total → 1 matched + join on id with 0: rows: 0, files: 6 total → 0 matched + join on id with 2, 6: rows: 2, files: 6 total → 2 matched + join on id with 2, 3, 4: rows: 3, files: 6 total → 3 matched + join on id with NULL: rows: 0, files: 0 total → 0 matched + join on id with 1, NULL: rows: 1, files: 6 total → 1 matched + join on name with 'avocado': rows: 1, files: 6 total → 2 matched + join on name with 'b': rows: 0, files: 6 total → 1 matched + join on name with 'éclair': rows: 1, files: 6 total → 1 matched + join on name with 'é': rows: 0, files: 6 total → 1 matched + join on name with 'zebra': rows: 0, files: 6 total → 0 matched + join on day with '2024-02-01': rows: 1, files: 6 total → 2 matched + join on day with '2024-01-15': rows: 0, files: 6 total → 1 matched + join on day with '2024-04-01': rows: 0, files: 6 total → 0 matched + join on day with '2023-12-31', '2024-03-01': rows: 2, files: 6 total → 2 matched + join on n with 12: rows: 1, files: 6 total → 3 matched + join on n with 99: rows: 0, files: 6 total → 1 matched + join on n with -1, 13: rows: 2, files: 6 total → 4 matched + join on at with '2024-01-02T00:00:00': rows: 1, files: 6 total → 2 matched + join on at with '2024-01-01T23:59:59.999999': rows: 1, files: 6 total → 1 matched + join on at with '2024-01-04T00:00:00': rows: 0, files: 6 total → 0 matched + ORDER BY id LIMIT 1: rows: 1, files: 6 total → 1 matched + ORDER BY id DESC LIMIT 1: rows: 1, files: 6 total → 6 matched + ORDER BY id NULLS FIRST LIMIT 2: rows: 2, files: 6 total → 3 matched + ORDER BY id DESC NULLS LAST LIMIT 2: rows: 2, files: 6 total → 5 matched + ORDER BY name LIMIT 1: rows: 1, files: 6 total → 2 matched + ORDER BY name DESC LIMIT 1: rows: 1, files: 6 total → 5 matched + ORDER BY day LIMIT 2: rows: 2, files: 6 total → 3 matched + ORDER BY day DESC LIMIT 1: rows: 1, files: 6 total → 5 matched + ORDER BY n LIMIT 1: rows: 1, files: 6 total → 6 matched + ORDER BY at LIMIT 1: rows: 1, files: 6 total → 2 matched + ORDER BY at DESC LIMIT 2: rows: 2, files: 6 total → 5 matched + "); + Ok(()) + } + // Table `events`, partitioned by the month of `day`, with one file for each of January, // February and March 2024. async fn events_by_month() -> Result> { @@ -254,4 +393,122 @@ mod tests { append.commit().await?; Ok(table) } + + // Table `readings`, partitioned by `id`, the first letter of `name`, the month of `day`, 4 + // buckets of `n` and the day of `at`. Each row is its own file, and the last row is all + // nulls. + async fn readings() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "day", Type::Primitive(PrimitiveType::Date)).into(), + NestedField::optional(4, "n", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(5, "at", Type::Primitive(PrimitiveType::Timestamp)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "readings"])?, schema) + .partition_by("id", Transform::Identity) + .partition_by("name", Transform::Truncate(1)) + .partition_by("day", Transform::Month) + .partition_by("n", Transform::Bucket(4)) + .partition_by("at", Transform::Day) + .create() + .await?; + let id: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + Some(3), + Some(4), + Some(6), + None, + ])); + let name: ArrayRef = Arc::new(StringArray::from(vec![ + Some("apple"), + Some("avocado"), + Some("banana"), + Some("éclair"), + Some("cherry"), + None, + ])); + // Days since 1970-01-01: 2024-01-31, 2024-02-01, 2024-02-29, 2024-03-01, 2023-12-31. + let day: ArrayRef = Arc::new(Date32Array::from(vec![ + Some(19753), + Some(19754), + Some(19782), + Some(19783), + Some(19722), + None, + ])); + let n: ArrayRef = Arc::new(Int64Array::from(vec![ + Some(10), + Some(11), + Some(12), + Some(13), + Some(-1), + None, + ])); + // Microseconds since 1970-01-01: 2024-01-01 23:59:59.999999, 2024-01-02 00:00:00, + // 2024-01-02 12:00:00, 2024-01-03 00:00:00, 2023-12-31 23:59:59. + let at: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![ + Some(1704153599999999), + Some(1704153600000000), + Some(1704196800000000), + Some(1704240000000000), + Some(1704067199000000), + None, + ])); + let rows = RecordBatch::try_from_iter([ + ("id", id), + ("name", name), + ("day", day), + ("n", n), + ("at", at), + ])?; + let mut append = table.append(); + for row in 0..rows.num_rows() { + append = append.file(rows.slice(row, 1)); + } + append.commit().await?; + Ok(table) + } + + /// A case that joins `readings` with `keys` on `column`. Each key is cast to the column's + /// type, so the join filters the column itself, without a cast around it. + fn join(schema: &SchemaRef, column: &str, keys: &[&str]) -> (String, String) { + let data_type = schema.field_with_name(column).unwrap().data_type(); + let values = keys + .iter() + .map(|key| format!("(arrow_cast({key}, '{data_type}'))")) + .collect::>() + .join(", "); + ( + format!("join on {column} with {}", keys.join(", ")), + format!( + "SELECT t.* FROM readings t JOIN (VALUES {values}) AS k(x) \ + ON t.{column} = k.x ORDER BY t.id" + ), + ) + } + + /// A case that reads the first `limit` rows of `readings` in `order`. + fn top_n(order: &str, limit: usize) -> (String, String) { + ( + format!("ORDER BY {order} LIMIT {limit}"), + format!("SELECT * FROM readings ORDER BY {order} LIMIT {limit}"), + ) + } + + /// The metric `name` of the Iceberg scan in `plan`, summed over its partitions. + fn scan_metric(plan: &Arc, name: &str) -> Option { + plan.metrics() + .and_then(|metrics| metrics.sum_by_name(name)) + .or_else(|| { + plan.children() + .into_iter() + .find_map(|child| scan_metric(child, name)) + }) + } } From ae2d92eba975e8c2d590972e3329f0525c8e4102 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:26:26 +0000 Subject: [PATCH 10/17] docs: rewrite the comments in the physical filter converter Explain the two forms of CASE, what LIKE patterns convert, and why only ASCII prefixes are converted. --- .../dynamic_filters/physical_predicate.rs | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs index fb6f927d1..7e0e78606 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs @@ -10,7 +10,8 @@ use iceberg::expr::{Predicate, Reference}; use iceberg::spec::{Datum, PrimitiveType}; /// Turns a DataFusion filter into an iceberg predicate. The predicate matches every row that the -/// filter would match, and *can* match more if the DF expression cannot be translated into an Iceberg predicate completely. +/// filter would match, and *can* match more if the DF expression cannot be translated into an +/// Iceberg predicate completely. /// Returns `None` when no part of the filter can be converted. pub(crate) fn convert_physical_filter_to_predicate( filter: &Arc, @@ -128,25 +129,32 @@ fn convert_not(not: &NotExpr) -> Option { .map(|inner| Converted::exact(inner.predicate.negate())) } -/// In a CASE, each WHEN picks which THEN decides if a row is kept. So every row the CASE -/// keeps passed some WHEN and its THEN, and this matches it: +/// In a CASE statement, each WHEN/THEN pair applies a filter over the rows. +/// /// `CASE WHEN c1 THEN x WHEN c2 THEN y ELSE z END` becomes `(c1 AND x) OR (c2 AND y) OR z`. /// It ignores that the second branch only applies when `c1` is false, so it can match -/// extra rows. That's fine here, but a NOT over it could skip rows the filter keeps. +/// extra rows. /// /// `CASE a WHEN 1 THEN x ...` works the same way, with `a = 1` as the WHEN. /// -/// Partitioned hash joins build filters like this, with one branch per partition: -/// `CASE hash(a) % 2 WHEN 0 THEN a IN (1, 2) WHEN 1 THEN a IN (3) ELSE false END`. -/// Iceberg can't compute the hash, so the WHEN conditions are dropped, leaving -/// `a IN (1, 2) OR a IN (3)`. -/// /// A missing ELSE means NULL, which matches nothing. fn convert_case(case: &CaseExpr) -> Option { + // This handles the two variants for CASE + // 1. CASE WHEN a = 1 THEN ... + // In this variant, `case.expr()` is `None` because the expression is imbued in the WHEN clause. + // We pass the WHEN clause to our general predicate converter for this. + // + // 2. CASE a WHEN 0 THEN ... + // In this variant, `case.expr()` is `Some(a)`, and each WHEN clause compares its own + // expression with this expression. + // To handle this, we convert this into a comparison predicate. + // Iceberg predicates can't always represent a given expression; if that is the case, we drop + // the predicate for the WHEN clause. let convert_when = |when| match case.expr() { Some(value) => compare(value, Operator::Eq, when).map(Converted::exact), None => convert(when), }; + let predicate = case .when_then_expr() .iter() @@ -251,12 +259,26 @@ fn convert_like(like: &LikeExpr) -> Option { if like.case_insensitive() { return None; } + + // Iceberg can only represent prefix matches using plain columns. + // E.g. WHERE s LIKE ... + // It cannot represent complex expressions or functions on the left side of the match + // E.g WHERE upper(s) LIKE ... + // `column_name` will return None if the left side is not a plain column and we don't add a + // predicate for this LikeExpr let column = Reference::new(column_name(like.expr())?); + + // The pattern being matched *must* be a string literal that ends in a '%', which means that it + // is trying to match the string's prefix. + // + // Iceberg can only support `STARTS WITH` predicates - after stripping the final '%', + // if there are more symbols in the string that could represent a broader match, + // we don't add a predicate for this LikeExpr let prefix = string_literal(like.pattern())?.strip_suffix('%')?; - // Any other `%` or `_`, or a `\`, means it's more than a simple prefix. if prefix.contains(['%', '_', '\\']) { return None; } + let prefix = prefix_datum(prefix)?; Some(if like.negated() { column.not_starts_with(prefix) @@ -277,10 +299,12 @@ fn convert_function(function: &ScalarFunctionExpr) -> Option { } } -/// Iceberg counts bytes where it should count characters when it checks STARTS WITH against -/// a `truncate` partition. With `truncate(2)`, the prefix `é` is 2 bytes, so iceberg only -/// looks in the partition `é` and skips the partition `éa`, which also has matching rows. -/// ASCII text has one byte per character, so only ASCII prefixes are converted. +/// Converts a string used as the prefix in a 'STARTS WITH' clause into a Datum +/// +/// This only works for ASCII strings. This is because iceberg-rust currently has a bug +/// where it can skip rows unintentionally if a partition column truncates the underlying value +/// via truncate(N) and if that underlying value contains non-ascii characters +/// Ref: https://github.com/apache/iceberg-rust/issues/3626 fn prefix_datum(prefix: &str) -> Option { prefix.is_ascii().then(|| Datum::string(prefix)) } @@ -670,7 +694,6 @@ mod tests { } /// Turns SQL like `a < 5` into a DataFusion filter over [`schema`], then converts it. - /// DataFusion's simplify step is skipped, so `NOT (a < 5)` stays a NOT. fn convert_sql(filter: &str) -> String { let schema = DFSchema::try_from(schema()).unwrap(); let ctx = SessionContext::new(); From e598082fc50cfb95e05f5c353babaf708ab2eed5 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:37:57 +0000 Subject: [PATCH 11/17] refactor: split the copied iceberg-rust code out of the partition skipper The skipper becomes DynamicFilterPruner, with one prune method that returns the task to read, or None to skip it. Later uses of the dynamic filters on each file can go in the same place. The code copied from iceberg-rust moves to partition_filter.rs, behind the same PartitionFilter API that iceberg-rust is adding. Once the crate's iceberg-rust version has it, that file can be deleted and the import changed. --- .../src/common/dynamic_filters/mod.rs | 3 +- ...rtition_skipper.rs => partition_filter.rs} | 202 ++++++------------ .../src/common/dynamic_filters/pruner.rs | 118 ++++++++++ .../src/common/scan_config.rs | 23 +- 4 files changed, 196 insertions(+), 150 deletions(-) rename datafusion/distributed-iceberg/src/common/dynamic_filters/{partition_skipper.rs => partition_filter.rs} (62%) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 1245e5ec7..7cc259ed0 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,2 +1,3 @@ -pub(crate) mod partition_skipper; +pub(crate) mod partition_filter; pub(crate) mod physical_predicate; +pub(crate) mod pruner; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs similarity index 62% rename from datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs rename to datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs index 9c239a22a..534cccf33 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_skipper.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs @@ -1,147 +1,77 @@ +//! A copy of iceberg-rust's `PartitionFilter`, which the iceberg-rust version this crate uses +//! doesn't have. It has the same API, so moving to iceberg-rust's version means deleting this +//! file and changing the import. +//! +//! The projection mirrors iceberg-rust's `InclusiveProjection`, and the evaluation mirrors its +//! `ExpressionEvaluator`. Both are private in this version. + use std::cmp::Ordering; use std::sync::Arc; -use datafusion::common::HashMap; -use datafusion::physical_expr::{PhysicalExpr, conjunction}; -use datafusion::physical_expr_common::physical_expr::{ - snapshot_generation, snapshot_physical_expr, -}; -use datafusion::physical_plan::metrics::{ - ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, -}; use iceberg::expr::{Bind, BoundPredicate, BoundReference, Predicate, PredicateOperator}; -use iceberg::scan::FileScanTask; use iceberg::spec::{ - Datum, Literal, PartitionSpec, PrimitiveLiteral, PrimitiveType, Schema, Struct, Type, + Datum, Literal, PartitionSpec, PartitionSpecRef, PrimitiveLiteral, PrimitiveType, Schema, + SchemaRef, Struct, Type, }; -use iceberg::{Error, ErrorKind}; - -use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; - -/// Skips files whose partition value can't match the scan's dynamic filters. -pub(crate) struct PartitionSkipper { - /// The scan's dynamic filters, combined with AND. - dynamic_filters: Arc, - - /// The generation of `dynamic_filters` that `predicate` was built from. - cached_snapshot_generation: u64, - - /// The filters as an Iceberg predicate bound to the table schema. - predicate: Option, +use iceberg::{Error, ErrorKind, Result}; - /// `predicate` projected onto each partition spec and bound to its partition schema, by spec - /// id. `None` keeps every file written with that spec. That happens when the projection - /// matches every partition, or when the spec's partition values can't be checked. Cleared - /// when `predicate` changes. - partition_predicates: HashMap>, - - /// The files checked. Pruned files were skipped and matched files were kept. - files: PruningMetrics, +/// Checks whether a data file's partition can hold rows that match a filter, as a table scan +/// does when it plans files. +pub(crate) struct PartitionFilter { + /// The filter projected onto the spec's partition columns, bound to its partition schema. + predicate: BoundPredicate, + spec: PartitionSpecRef, } -impl PartitionSkipper { - pub(crate) fn new( - dynamic_filters: Vec>, - metrics: &ExecutionPlanMetricsSet, - partition: usize, - ) -> Self { - Self { - dynamic_filters: conjunction(dynamic_filters), - cached_snapshot_generation: 0, - predicate: None, - partition_predicates: HashMap::new(), - files: MetricBuilder::new(metrics) - .with_type(MetricType::Summary) - .pruning_metrics("files_pruned_by_partition", partition), - } - } - - /// Whether no row in the task's partition can match the filters. Any error keeps the file. - pub(crate) fn should_skip_task(&mut self, task: &FileScanTask) -> bool { - let skip = self.can_skip(task); - if skip { - self.files.add_pruned(1); +impl PartitionFilter { + /// Creates a partition filter for `filter` on data files written with `partition_spec`. + /// + /// If `partition_spec` uses a column that `schema` no longer has, every partition matches. + /// Returns an error if `filter` doesn't bind to `schema`, or if it can't be projected onto + /// `partition_spec`. + pub(crate) fn try_new( + filter: Predicate, + schema: SchemaRef, + partition_spec: &PartitionSpecRef, + case_sensitive: bool, + ) -> Result { + let filter = filter + .rewrite_not() + .bind(Arc::clone(&schema), case_sensitive)?; + let has_dropped_source_column = partition_spec + .fields() + .iter() + .any(|field| schema.field_by_id(field.source_id).is_none()); + let predicate = if has_dropped_source_column { + BoundPredicate::AlwaysTrue } else { - self.files.add_matched(1); - } - skip - } - - fn can_skip(&mut self, task: &FileScanTask) -> bool { - let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { - return false; + let partition_type = partition_spec.partition_type(&schema)?; + let partition_schema = Arc::new( + Schema::builder() + .with_schema_id(partition_spec.spec_id()) + .with_fields(partition_type.fields().to_owned()) + .build()?, + ); + project(&filter, partition_spec)? + .rewrite_not() + .bind(partition_schema, case_sensitive)? }; - if spec.is_unpartitioned() { - return false; - } - self.refresh(task); - let Some(predicate) = &self.predicate else { - return false; - }; - // Every task in a scan has the same table schema, so the spec id is enough as a key. - let partition_predicate = self - .partition_predicates - .entry(spec.spec_id()) - .or_insert_with(|| { - project_onto_spec(predicate, spec, task.schema()) - .ok() - .filter(|projected| !matches!(projected, BoundPredicate::AlwaysTrue)) - }); - let Some(projected) = partition_predicate.as_ref() else { - return false; - }; - match evaluate(projected, spec, partition) { - Ok(matches) => !matches, - // Files with the same spec would most likely fail the same way, so stop checking - // them until the filters change. - Err(_) => { - *partition_predicate = None; - false - } - } + Ok(Self { + predicate, + spec: Arc::clone(partition_spec), + }) } - /// Rebuilds the predicate when any of the filters has changed. - fn refresh(&mut self, task: &FileScanTask) { - // Read the generation before taking the snapshot. If a filter changes in between, the - // next file sees a newer generation and rebuilds. - let generation = snapshot_generation(&self.dynamic_filters); - if self.cached_snapshot_generation == generation { - return; - } - self.cached_snapshot_generation = generation; - self.partition_predicates.clear(); - self.predicate = snapshot_physical_expr(Arc::clone(&self.dynamic_filters)) - .ok() - .and_then(|snapshot| convert_physical_filter_to_predicate(&snapshot)) - .and_then(|predicate| { - predicate - .bind(task.schema_ref(), task.case_sensitive()) - .ok() - }); + /// Returns whether a data file with `partition` can hold rows that match the filter. + /// Returns an error if `partition` doesn't match the partition spec's fields. + pub(crate) fn can_match(&self, partition: &Struct) -> Result { + evaluate(&self.predicate, &self.spec, partition) } } -/// Projects a predicate on table columns onto a partition spec, then binds it to the spec's -/// partition schema. Binding gives its values the partition columns' types, so they compare -/// with the partition values. -fn project_onto_spec( - predicate: &BoundPredicate, - spec: &PartitionSpec, - table_schema: &Schema, -) -> iceberg::Result { - let partition_type = spec.partition_type(table_schema)?; - let partition_schema = Schema::builder() - .with_fields(partition_type.fields().iter().cloned()) - .build()?; - project(predicate, spec)?.bind(Arc::new(partition_schema), true) -} - /// Rewrites a predicate on table columns into one on the spec's partition columns. The result /// matches every partition that holds a matching row, and can match more. -/// -/// Mirrors iceberg-rust's `InclusiveProjection`, which is private. -fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> iceberg::Result { +fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> Result { Ok(match predicate { BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, @@ -153,8 +83,8 @@ fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> iceberg::Result< let [left, right] = expr.inputs(); project(left, spec)?.or(project(right, spec)?) } - // The filter translator turns each NOT into the opposite comparison, so none reach - // here. Keep the file if one shows up anyway. + // `PartitionFilter::try_new` rewrites every NOT away first, so none reach here. Keep + // the file if one shows up anyway. BoundPredicate::Not(_) => Predicate::AlwaysTrue, BoundPredicate::Unary(expr) => project_column(expr.term(), predicate, spec)?, BoundPredicate::Binary(expr) => project_column(expr.term(), predicate, spec)?, @@ -168,7 +98,7 @@ fn project_column( column: &BoundReference, predicate: &BoundPredicate, spec: &PartitionSpec, -) -> iceberg::Result { +) -> Result { spec.fields() .iter() .filter(|field| field.source_id == column.field().id) @@ -183,11 +113,7 @@ fn project_column( /// Evaluates a predicate bound to the partition schema against one partition value. Null values /// follow iceberg-rust's `ExpressionEvaluator`. -fn evaluate( - predicate: &BoundPredicate, - spec: &PartitionSpec, - partition: &Struct, -) -> iceberg::Result { +fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct) -> Result { match predicate { BoundPredicate::AlwaysTrue => Ok(true), BoundPredicate::AlwaysFalse => Ok(false), @@ -258,7 +184,7 @@ fn partition_value( column: &BoundReference, spec: &PartitionSpec, partition: &Struct, -) -> iceberg::Result> { +) -> Result> { let field = column.field(); let position = spec .fields() @@ -283,7 +209,7 @@ fn partition_value( /// Builds a `Datum` of type `r#type`. Also widens values written before a column's type was /// promoted, such as an int partition value of a column that is now a long. -fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> iceberg::Result { +fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { Ok(match (r#type, literal) { (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => Datum::bool(*value), (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => Datum::int(*value), @@ -328,7 +254,7 @@ fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> iceberg::Result< /// Orders two values of the same type. Values of different types are an error, so the file is /// kept. -fn compare(value: &Datum, literal: &Datum) -> iceberg::Result { +fn compare(value: &Datum, literal: &Datum) -> Result { if value.data_type() != literal.data_type() { return Err(type_mismatch(value, literal)); } @@ -337,7 +263,7 @@ fn compare(value: &Datum, literal: &Datum) -> iceberg::Result { .ok_or_else(|| type_mismatch(value, literal)) } -fn starts_with(value: &Datum, prefix: &Datum) -> iceberg::Result { +fn starts_with(value: &Datum, prefix: &Datum) -> Result { match (value.literal(), prefix.literal()) { (PrimitiveLiteral::String(value), PrimitiveLiteral::String(prefix)) => { Ok(value.starts_with(prefix.as_str())) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs new file mode 100644 index 000000000..c9212b109 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -0,0 +1,118 @@ +use std::sync::Arc; + +use datafusion::common::HashMap; +use datafusion::physical_expr::{PhysicalExpr, conjunction}; +use datafusion::physical_expr_common::physical_expr::{ + snapshot_generation, snapshot_physical_expr, +}; +use datafusion::physical_plan::metrics::{ + ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, +}; +use iceberg::expr::Predicate; +use iceberg::scan::FileScanTask; + +use crate::common::dynamic_filters::partition_filter::PartitionFilter; +use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; + +/// Uses the scan's dynamic filters on each file just before the reader starts it. Skips files +/// whose partition value can't match the filters. +pub(crate) struct DynamicFilterPruner { + /// The scan's dynamic filters, combined with AND. + dynamic_filters: Arc, + + /// The generation of `dynamic_filters` that `predicate` was built from. + cached_snapshot_generation: u64, + + /// The filters as an Iceberg predicate. `None` when no part of them converts. + predicate: Option, + + /// A partition filter for each partition spec, by spec id. `None` keeps every file written + /// with that spec, because its partition values can't be checked. Cleared when `predicate` + /// changes. + partition_filters: HashMap>, + + /// The files checked. Pruned files were skipped and matched files were kept. + files: PruningMetrics, +} + +impl DynamicFilterPruner { + pub(crate) fn new( + dynamic_filters: Vec>, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + ) -> Self { + Self { + dynamic_filters: conjunction(dynamic_filters), + cached_snapshot_generation: 0, + predicate: None, + partition_filters: HashMap::new(), + files: MetricBuilder::new(metrics) + .with_type(MetricType::Summary) + .pruning_metrics("files_pruned_by_partition", partition), + } + } + + /// The task to read, or `None` to skip it. Any error keeps the task. + pub(crate) fn prune(&mut self, task: FileScanTask) -> Option { + if self.can_skip(&task) { + self.files.add_pruned(1); + return None; + } + self.files.add_matched(1); + Some(task) + } + + /// Whether no row in the task's partition can match the filters. + fn can_skip(&mut self, task: &FileScanTask) -> bool { + let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { + return false; + }; + if spec.is_unpartitioned() { + return false; + } + self.refresh(); + let Some(predicate) = &self.predicate else { + return false; + }; + // Every task in a scan has the same table schema, so the spec id is enough as a key. + let partition_filter = self + .partition_filters + .entry(spec.spec_id()) + .or_insert_with(|| { + PartitionFilter::try_new( + predicate.clone(), + task.schema_ref(), + spec, + task.case_sensitive(), + ) + .ok() + }); + let Some(filter) = partition_filter.as_ref() else { + return false; + }; + match filter.can_match(partition) { + Ok(matches) => !matches, + // Files with the same spec would most likely fail the same way, so stop checking + // them until the filters change. + Err(_) => { + *partition_filter = None; + false + } + } + } + + /// Rebuilds the predicate when any of the filters has changed. + fn refresh(&mut self) { + // Read the generation before taking the snapshot. If a filter changes in between, the + // next file sees a newer generation and rebuilds. + let generation = snapshot_generation(&self.dynamic_filters); + if self.cached_snapshot_generation == generation { + return; + } + self.cached_snapshot_generation = generation; + self.partition_filters.clear(); + self.predicate = snapshot_physical_expr(Arc::clone(&self.dynamic_filters)) + .ok() + .and_then(|snapshot| convert_physical_filter_to_predicate(&snapshot)); + } +} diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index dcf4344c9..d9ec4843f 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -1,4 +1,4 @@ -use crate::common::dynamic_filters::partition_skipper::PartitionSkipper; +use crate::common::dynamic_filters::pruner::DynamicFilterPruner; use crate::{IcebergConfig, common::df_err}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::stats::Precision; @@ -145,16 +145,17 @@ impl ScanConfig { .with_category(MetricCategory::Bytes) .gauge("bytes_scanned", partition); - let mut partition_skipper = self.partition_skipper(partition); + let mut pruner = self.dynamic_filter_pruner(partition); // The reader pulls a task when it starts reading that task's file. It reads ahead, // so with a LIMIT this can count files that never produced output. Skipped files - // are dropped before this, and the skipper counts them. + // are dropped before this, and the pruner counts them. let tasks = tasks - .try_filter(move |task| { - let skip = partition_skipper - .as_mut() - .is_some_and(|skipper| skipper.should_skip_task(task)); - ready(!skip) + .try_filter_map(move |task| { + let task = match pruner.as_mut() { + Some(pruner) => pruner.prune(task), + None => Some(task), + }; + ready(Ok(task)) }) .inspect_ok(move |_| files_opened.add(1)) .boxed(); @@ -180,15 +181,15 @@ impl ScanConfig { Ok(Box::pin(LimitStream::new(stream, 0, self.fetch, metrics))) } - /// A skipper over the scan's dynamic filters. `None` when dynamic filtering is off or the + /// A pruner over the scan's dynamic filters. `None` when dynamic filtering is off or the /// scan has no dynamic filter. - fn partition_skipper(&self, partition: usize) -> Option { + fn dynamic_filter_pruner(&self, partition: usize) -> Option { if !self.dynamic_filtering_enabled { return None; } let dynamic_filters = dynamic_filters(self.filters.as_ref()?); (!dynamic_filters.is_empty()) - .then(|| PartitionSkipper::new(dynamic_filters, &self.metrics, partition)) + .then(|| DynamicFilterPruner::new(dynamic_filters, &self.metrics, partition)) } } From 53f291483b714778442bb59a7832c8c34c57c6fe Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:59:47 +0000 Subject: [PATCH 12/17] nit: doc cleanup + small refactor/simplification --- .../dynamic_filters/physical_predicate.rs | 141 +++++++++++++----- .../src/common/dynamic_filters/pruner.rs | 61 +++++--- .../src/common/scan_config.rs | 11 +- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs index 7e0e78606..3177052c2 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs @@ -1,5 +1,7 @@ use std::sync::Arc; +use datafusion::arrow::array::StructArray; +use datafusion::functions::core::r#struct::StructFunc; use datafusion::logical_expr::Operator; use datafusion::physical_expr::expressions::{ BinaryExpr, CaseExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NotExpr, @@ -12,6 +14,7 @@ use iceberg::spec::{Datum, PrimitiveType}; /// Turns a DataFusion filter into an iceberg predicate. The predicate matches every row that the /// filter would match, and *can* match more if the DF expression cannot be translated into an /// Iceberg predicate completely. +/// /// Returns `None` when no part of the filter can be converted. pub(crate) fn convert_physical_filter_to_predicate( filter: &Arc, @@ -23,8 +26,8 @@ pub(crate) fn convert_physical_filter_to_predicate( /// It can also match extra rows, so the caller must still apply the original filter. struct Converted { predicate: Predicate, - /// True if the predicate is false wherever the filter is false. - /// Only an exact predicate can go under a NOT. + + /// True if the DataFusion filter could be completely represented by this `iceberg::Predicate` exact: bool, } @@ -36,7 +39,7 @@ impl Converted { } } - fn wider(predicate: Predicate) -> Self { + fn inexact(predicate: Predicate) -> Self { Self { predicate, exact: false, @@ -58,6 +61,10 @@ impl Converted { } } +/// Converts a given `PhysicalExpr` into an `iceberg::Predicate` +/// wrapped in `Converted` which records whether the conversion was exact or inexact +/// +/// Returns None if the expression could not be converted fn convert(expr: &Arc) -> Option { if let Some(binary) = expr.downcast_ref::() { return convert_binary(binary); @@ -100,12 +107,16 @@ fn convert(expr: &Arc) -> Option { } } +/// Converts a binary expression into an `iceberg::Predicate` fn convert_binary(binary: &BinaryExpr) -> Option { let (left, right) = (binary.left(), binary.right()); match binary.op() { + // An AND expression can be converted if *either* side can be converted + // If one of the sides are dropped, iceberg would return extra rows + // which would then be filtered out by DataFusion Operator::And => both(convert(left), convert(right)), - // An OR needs both sides. Turning `a = 1 OR f(b)` into `a = 1` would skip rows where - // only `f(b)` is true. + // An OR expression can only be converted if *both* sides can be converted + // We cannot drop one of the sides, as that would lead to data being skipped incorrectly Operator::Or => Some(convert(left)?.or(convert(right)?)), op => compare(left, *op, right).map(Converted::exact), } @@ -116,19 +127,31 @@ fn convert_binary(binary: &BinaryExpr) -> Option { fn both(left: Option, right: Option) -> Option { match (left, right) { (Some(left), Some(right)) => Some(left.and(right)), - (Some(side), None) | (None, Some(side)) => Some(Converted::wider(side.predicate)), + (Some(side), None) | (None, Some(side)) => Some(Converted::inexact(side.predicate)), (None, None) => None, } } -/// A NOT needs an exact predicate. In `NOT (a = 1 AND f(b))`, the inside becomes `a = 1`, -/// and `NOT (a = 1)` would skip rows where `a = 1` but `f(b)` is false. +/// Converts a NOT expession into an `iceberg::Predicate` +/// +/// NOT is evaluated against an inner expression +/// It can be converted to a predicate *iff* the inner expression +/// can be converted *exactly*. +/// +/// Eg. `NOT (a = 1 AND f(b) = True)` +/// `f(b)` is not representable by an iceberg::Predicate, so it is dropped, and the inner +/// expression becomes `(a = 1)` upon conversion. +/// `NOT (a=1)` would skip rows where `a = 1` but `f(b)` is false, which is incorrect. +/// +/// Returns None if the expression cannot be converted to a predicate fn convert_not(not: &NotExpr) -> Option { convert(not.arg()) .filter(|inner| inner.exact) .map(|inner| Converted::exact(inner.predicate.negate())) } +/// Converts a CaseExpr into an `iceberg::Predicate` +/// /// In a CASE statement, each WHEN/THEN pair applies a filter over the rows. /// /// `CASE WHEN c1 THEN x WHEN c2 THEN y ELSE z END` becomes `(c1 AND x) OR (c2 AND y) OR z`. @@ -158,26 +181,49 @@ fn convert_case(case: &CaseExpr) -> Option { let predicate = case .when_then_expr() .iter() + // Converts each individual WHEN/THEN pair into a conjunctive pair + // WHEN x > 5 THEN y < 3 becomes (x > 5 AND y < 3) + // If either side is dropped because it can't be converted, the filter is still valid + // and will just match more rows .map(|(when, then)| both(convert_when(when), convert(then))) + // If there is an ELSE at the end, convert it into a predicate directly .chain(case.else_expr().map(convert)) .collect::>>()? .into_iter() .map(|branch| branch.predicate) + // Combines all predicates in a disjunction, i.e, OR + // Multiple WHEN/THEN statements are joined together with ORs + // E.g: + // WHEN x > 5 THEN y ELSE + // WHEN x > 10 THEN z ELSE ... + // becomes + // (x > 5 AND y) OR (x > 10 AND z) .reduce(Predicate::or)?; - Some(Converted::wider(predicate)) + + // Each WHEN/THEN pair in a CASE statement is mutually exclusive + // Subsequent pairs are only executed if the previous one didn't match + // However, in our translation, we `OR` all of them together; subsequent pairs don't check + // whether the previous one matched or not + // + // This means that the filter can return *more* rows than if DataFusion was doing the + // filtering. Due to this, we mark this Predicate as inexact + Some(Converted::inexact(predicate)) } -/// Converts a column compared with a value, like `a < 5`. -/// The value can come first too: `5 > a` becomes `a < 5`. +/// Converts a column <-> value comparison into an `iceberg::Predicate` fn compare( left: &Arc, op: Operator, right: &Arc, ) -> Option { + // Try to match ` ` first + // If unsuccessful, swap the order of the two inputs and check again + // If unsuccessful again, we can't convert this expression and we bail early let (column, op, datum) = match (column_name(left), literal_datum(right)) { (Some(column), Some(datum)) => (column, op, datum), _ => (column_name(right)?, op.swap()?, literal_datum(left)?), }; + let column = Reference::new(column); match op { Operator::Eq => Some(column.equal_to(datum)), @@ -190,69 +236,96 @@ fn compare( } } +/// Converts an InList into an `iceberg::Predicate` fn convert_in_list(in_list: &InListExpr) -> Option { // Something like `struct(a, b) IN (...)`, which checks several columns at once. let Some(column) = column_name(in_list.expr()) else { - return convert_struct_in_list(in_list).map(Converted::wider); + return convert_struct_in_list(in_list).map(Converted::inexact); }; + let column = Reference::new(column); let values = in_list .list() .iter() + // We expect every value in the list to be a literal, i.e, x IN (1, 2, 3) + // If it isn't, we can't convert this list, and bail early .map(literal_value) .collect::>>()?; - let (nulls, values): (Vec<_>, Vec<_>) = values.into_iter().partition(|value| value.is_null()); - let datums = values + let has_null = values.iter().any(|value| value.is_null()); + + // `x NOT IN (1, NULL)` would never match anything because x != NULL gives NULL for every row and matches nothing + if in_list.negated() && has_null { + return None; + } + + let literals = values .into_iter() + .filter(|value| !value.is_null()) .map(scalar_value_to_datum) .collect::>>()?; - let predicate = match (in_list.negated(), nulls.is_empty()) { - // `x IN (1, NULL)` matches the same rows as `x IN (1)`. - (false, _) => column.is_in(datums), - (true, true) => column.is_not_in(datums), - // `x NOT IN (1, NULL)` never matches anything. It's rare, so we don't convert it. - (true, false) => return None, + + let predicate = if in_list.negated() { + column.is_not_in(literals) + } else { + column.is_in(literals) }; + Some(Converted::exact(predicate)) } +/// Converts a multi-column InList into an `iceberg::Predicate` +/// /// Hash joins on two or more columns build filters like `struct(a, b) IN ({1, 3}, {2, 4})`. +/// /// Iceberg can't check pairs, so each column is checked on its own: -/// `a IN (1, 2) AND b IN (3, 4)`. This also matches pairs like `{1, 4}`, which is fine. +/// `a IN (1, 2) AND b IN (3, 4)`. This can result in extra rows from pairs like `{1, 4}`, +/// which is okay. +/// /// A column with a NULL value is left out, because the join decides if NULL matches NULL. fn convert_struct_in_list(in_list: &InListExpr) -> Option { + // We can't safely split multi-column pairs and check them individually for a `NOT IN` expr if in_list.negated() { return None; } - let columns = in_list - .expr() - .downcast_ref::() - .filter(|function| function.name() == "struct")? - .args(); + + let columns = + ScalarFunctionExpr::try_downcast_func::(in_list.expr().as_ref())?.args(); + let items = in_list .list() .iter() + // If any of the items in the list aren't struct literals, we can't convert this + // and we bail early .map(|item| match literal_value(item)? { ScalarValue::Struct(item) => Some(item), _ => None, }) .collect::>>()?; + + // One IN list per plain column. A column left out only makes the filter match more rows. columns .iter() .enumerate() .filter_map(|(index, column)| { - let datums = items - .iter() - .map(|item| { - let value = ScalarValue::try_from_array(item.columns().get(index)?, 0).ok()?; - scalar_value_to_datum(&value) - }) - .collect::>>()?; - Some(Reference::new(column_name(column)?).is_in(datums)) + let column = column_name(column)?; + let values = field_values(&items, index)?; + Some(Reference::new(column).is_in(values)) }) .reduce(Predicate::and) } +/// Field `index` of each struct as an Iceberg value, or `None` if any of them can't be converted. +/// Each struct is a one-row array, so its value is in row 0. +fn field_values(items: &[&Arc], index: usize) -> Option> { + items + .iter() + .map(|item| { + let field = item.columns().get(index)?; + scalar_value_to_datum(&ScalarValue::try_from_array(field, 0).ok()?) + }) + .collect() +} + /// Only simple prefix patterns: `a LIKE 'abc%'` becomes `a STARTS WITH 'abc'`. /// ILIKE can't be converted, because iceberg's STARTS WITH is case-sensitive. fn convert_like(like: &LikeExpr) -> Option { diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs index c9212b109..2f75995b8 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -23,15 +23,14 @@ pub(crate) struct DynamicFilterPruner { /// The generation of `dynamic_filters` that `predicate` was built from. cached_snapshot_generation: u64, - /// The filters as an Iceberg predicate. `None` when no part of them converts. + /// The dynamic filters are stored as iceberg Predicates after being converted + /// `None` if the filters can't be converted predicate: Option, - /// A partition filter for each partition spec, by spec id. `None` keeps every file written - /// with that spec, because its partition values can't be checked. Cleared when `predicate` - /// changes. + /// A map of partition spec IDs to their corresponding partition filter executors partition_filters: HashMap>, - /// The files checked. Pruned files were skipped and matched files were kept. + /// Metrics to track pruning files: PruningMetrics, } @@ -52,14 +51,16 @@ impl DynamicFilterPruner { } } - /// The task to read, or `None` to skip it. Any error keeps the task. - pub(crate) fn prune(&mut self, task: FileScanTask) -> Option { - if self.can_skip(&task) { + /// Whether the filters rule out every row in the task's partition. Counts the task as pruned + /// or matched + pub(crate) fn should_prune(&mut self, task: &FileScanTask) -> bool { + let prune = self.can_skip(task); + if prune { self.files.add_pruned(1); - return None; + } else { + self.files.add_matched(1); } - self.files.add_matched(1); - Some(task) + prune } /// Whether no row in the task's partition can match the filters. @@ -74,7 +75,19 @@ impl DynamicFilterPruner { let Some(predicate) = &self.predicate else { return false; }; - // Every task in a scan has the same table schema, so the spec id is enough as a key. + + // Every `FileScanTask` within a scan has the same Iceberg schema, taken from the selected + // Iceberg snapshot. + // The only difference in partition filters can come from the partition spec being different + // between data files. + // The PartitionFilter struct - + // 1. Binds the `Predicate`, which just contains column names and values, to the actual + // schema, resolving column names to Iceberg field IDs etc. + // 2. Projects the column filter into a partition filter. A partition spec is not limited + // to just direct column filters; it can have expressions like `bucket(2, x)` or + // `day(ts)`. A column filter like `ts >= '2024-03-15 10:00'` becomes + // `ts_day >= 2024-03-15` which can be checked against every data file's partition value. + // These steps can be expensive, so we maintain a cache of a `PartitionFilter` per spec ID. let partition_filter = self .partition_filters .entry(spec.spec_id()) @@ -90,21 +103,23 @@ impl DynamicFilterPruner { let Some(filter) = partition_filter.as_ref() else { return false; }; - match filter.can_match(partition) { - Ok(matches) => !matches, - // Files with the same spec would most likely fail the same way, so stop checking - // them until the filters change. - Err(_) => { - *partition_filter = None; - false - } - } + + // Check whether the file can be dropped, based on the partition values. + // If there's an error while evaluating the filter, + // we return `false` so that the file is not skipped + filter.can_match(partition).is_ok_and(|matches| !matches) } /// Rebuilds the predicate when any of the filters has changed. fn refresh(&mut self) { - // Read the generation before taking the snapshot. If a filter changes in between, the - // next file sees a newer generation and rebuilds. + // Read the current generation before snapshotting the filters. + // The two reads aren't atomic, so a filter can change between them. + // + // In this order, the snapshot can be newer than the generation we have stored + // which is okay - because we're still doing the most 'efficient' pruning + // + // In the opposite order, we could store a newer generation but an older filter, + // which would prune less until the filter updated. let generation = snapshot_generation(&self.dynamic_filters); if self.cached_snapshot_generation == generation { return; diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index d9ec4843f..51dd51a04 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -150,12 +150,11 @@ impl ScanConfig { // so with a LIMIT this can count files that never produced output. Skipped files // are dropped before this, and the pruner counts them. let tasks = tasks - .try_filter_map(move |task| { - let task = match pruner.as_mut() { - Some(pruner) => pruner.prune(task), - None => Some(task), - }; - ready(Ok(task)) + .try_filter(move |task| { + let prune = pruner + .as_mut() + .is_some_and(|pruner| pruner.should_prune(task)); + ready(!prune) }) .inspect_ok(move |_| files_opened.add(1)) .boxed(); From fc6451a1eb0acd147f845262917d31f98e253ad9 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:02:54 +0000 Subject: [PATCH 13/17] nit: fix a typo the spell check flags --- .../src/common/dynamic_filters/physical_predicate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs index 3177052c2..9ed9f50ff 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs @@ -132,7 +132,7 @@ fn both(left: Option, right: Option) -> Option } } -/// Converts a NOT expession into an `iceberg::Predicate` +/// Converts a NOT expression into an `iceberg::Predicate` /// /// NOT is evaluated against an inner expression /// It can be converted to a predicate *iff* the inner expression From dbca50504839bed64d96e67f6d0704964e81b808 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:23:56 +0000 Subject: [PATCH 14/17] fix: simplify partition_filter's docs and comments --- .../dynamic_filters/partition_filter.rs | 153 ++++++++++-------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs index 534cccf33..037a51561 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs @@ -1,11 +1,21 @@ -//! A copy of iceberg-rust's `PartitionFilter`, which the iceberg-rust version this crate uses -//! doesn't have. It has the same API, so moving to iceberg-rust's version means deleting this -//! file and changing the import. +//! A copy of the code iceberg-rust uses to skip data files by their partition values. That +//! code isn't part of iceberg-rust's public API yet. When it becomes public, this file should +//! be deleted and its callers should use the public API. +//! Ref: https://github.com/apache/iceberg-rust/issues/3675 //! -//! The projection mirrors iceberg-rust's `InclusiveProjection`, and the evaluation mirrors its -//! `ExpressionEvaluator`. Both are private in this version. +//! Each part of this file copies one part of iceberg-rust: +//! +//! | This file | iceberg-rust | +//! |------------------------------|--------------------------------------------------------------| +//! | `PartitionFilter::try_new` | `TableScanBuilder::with_filter`, `PartitionFilterCache::get` | +//! | `PartitionFilter::can_match` | `ExpressionEvaluator::eval` | +//! | `project`, `project_column` | `InclusiveProjection` | +//! | `evaluate` | `ExpressionEvaluatorVisitor` | +//! | `partition_value`, `datum` | `StructAccessor::get` | +//! +//! The copy works the same way as iceberg-rust, except for one case in `datum` that's +//! explained there. -use std::cmp::Ordering; use std::sync::Arc; use iceberg::expr::{Bind, BoundPredicate, BoundReference, Predicate, PredicateOperator}; @@ -83,9 +93,14 @@ fn project(predicate: &BoundPredicate, spec: &PartitionSpec) -> Result Predicate::AlwaysTrue, + // `PartitionFilter::try_new` removes every NOT first, so none should reach here. If one + // does, this returns an error, as iceberg-rust does. + BoundPredicate::Not(_) => { + return Err(Error::new( + ErrorKind::Unexpected, + "found a NOT that `PartitionFilter::try_new` should have removed", + )); + } BoundPredicate::Unary(expr) => project_column(expr.term(), predicate, spec)?, BoundPredicate::Binary(expr) => project_column(expr.term(), predicate, spec)?, BoundPredicate::Set(expr) => project_column(expr.term(), predicate, spec)?, @@ -111,22 +126,31 @@ fn project_column( }) } -/// Evaluates a predicate bound to the partition schema against one partition value. Null values -/// follow iceberg-rust's `ExpressionEvaluator`. +/// Checks whether one data file's partition values pass a filter on the partition columns. fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct) -> Result { match predicate { BoundPredicate::AlwaysTrue => Ok(true), BoundPredicate::AlwaysFalse => Ok(false), BoundPredicate::And(expr) => { let [left, right] = expr.inputs(); - Ok(evaluate(left, spec, partition)? && evaluate(right, spec, partition)?) + let (left, right) = ( + evaluate(left, spec, partition)?, + evaluate(right, spec, partition)?, + ); + Ok(left && right) } BoundPredicate::Or(expr) => { let [left, right] = expr.inputs(); - Ok(evaluate(left, spec, partition)? || evaluate(right, spec, partition)?) + let (left, right) = ( + evaluate(left, spec, partition)?, + evaluate(right, spec, partition)?, + ); + Ok(left || right) } - // A projection never makes a NOT. Keep the file if one shows up anyway. - BoundPredicate::Not(_) => Ok(true), + BoundPredicate::Not(_) => Err(Error::new( + ErrorKind::Unexpected, + "found a NOT that `PartitionFilter::try_new` should have removed", + )), BoundPredicate::Unary(expr) => { let value = partition_value(expr.term(), spec, partition)?; let is_nan = value.as_ref().is_some_and(Datum::is_nan); @@ -147,14 +171,14 @@ fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct }; let literal = expr.literal(); Ok(match expr.op() { - PredicateOperator::StartsWith => starts_with(&value, literal)?, - PredicateOperator::NotStartsWith => !starts_with(&value, literal)?, - PredicateOperator::LessThan => compare(&value, literal)?.is_lt(), - PredicateOperator::LessThanOrEq => compare(&value, literal)?.is_le(), - PredicateOperator::GreaterThan => compare(&value, literal)?.is_gt(), - PredicateOperator::GreaterThanOrEq => compare(&value, literal)?.is_ge(), - PredicateOperator::Eq => compare(&value, literal)?.is_eq(), - PredicateOperator::NotEq => compare(&value, literal)?.is_ne(), + PredicateOperator::StartsWith => starts_with(&value, literal), + PredicateOperator::NotStartsWith => !starts_with(&value, literal), + PredicateOperator::LessThan => &value < literal, + PredicateOperator::LessThanOrEq => &value <= literal, + PredicateOperator::GreaterThan => &value > literal, + PredicateOperator::GreaterThanOrEq => &value >= literal, + PredicateOperator::Eq => &value == literal, + PredicateOperator::NotEq => &value != literal, _ => true, }) } @@ -162,12 +186,6 @@ fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct let Some(value) = partition_value(expr.term(), spec, partition)? else { return Ok(matches!(expr.op(), PredicateOperator::NotIn)); }; - // Binding gives every value in the set the column's type, so checking one is enough. - if let Some(literal) = expr.literals().iter().next() - && literal.data_type() != value.data_type() - { - return Err(type_mismatch(&value, literal)); - } let contains = expr.literals().contains(&value); Ok(match expr.op() { PredicateOperator::In => contains, @@ -178,8 +196,11 @@ fn evaluate(predicate: &BoundPredicate, spec: &PartitionSpec, partition: &Struct } } -/// The partition value for a reference bound to the partition schema, with the partition -/// column's type. `None` is a null value. +/// Returns the partition value that `column` points to, with the partition column's type. +/// Returns `None` when the value is null. +/// +/// iceberg-rust reads this value with `StructAccessor::get`, which isn't public. So this finds +/// the value's position by looking up its partition field ID in the spec. fn partition_value( column: &BoundReference, spec: &PartitionSpec, @@ -190,25 +211,37 @@ fn partition_value( .fields() .iter() .position(|partition_field| partition_field.field_id == field.id) - .ok_or_else(|| invalid(format!("partition field {} is not in the spec", field.id)))?; + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("partition field {} is not in the spec", field.id), + ) + })?; let Some(value) = partition.fields().get(position) else { - return Err(invalid(format!("the partition has no value at {position}"))); + return Err(Error::new( + ErrorKind::DataInvalid, + format!("the partition has no value at {position}"), + )); }; let Some(value) = value else { return Ok(None); }; let (Literal::Primitive(literal), Type::Primitive(r#type)) = (value, field.field_type.as_ref()) else { - return Err(invalid(format!( - "partition field {} is not a primitive", - field.id - ))); + return Err(Error::new( + ErrorKind::DataInvalid, + format!("partition field {} is not a primitive", field.id), + )); }; datum(r#type, literal).map(Some) } -/// Builds a `Datum` of type `r#type`. Also widens values written before a column's type was -/// promoted, such as an int partition value of a column that is now a long. +/// Builds a `Datum` of type `r#type` from a stored partition value. iceberg-rust uses +/// `Datum::new` for this, which isn't public, so this uses the public constructors. +/// +/// If a column changed from int to long, or from float to double, older files still store +/// values of the old type. This converts them to the new type. Newer iceberg-rust versions do +/// the same conversion when they read manifests. fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { Ok(match (r#type, literal) { (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => Datum::bool(*value), @@ -247,39 +280,25 @@ fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => { Datum::try_from_bytes(&value.to_be_bytes(), PrimitiveType::Uuid)? } - // A value that doesn't fit its column's type. Keep the file. - _ => return Err(invalid(format!("can't read a {type} partition value"))), + // The stored value doesn't fit its column's type, which we don't expect in a valid + // table. iceberg-rust builds the value anyway. None of its comparisons then match, so + // it skips the file. The public constructors can't build a value like that, so this + // returns an error and we don't skip this file. + _ => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("can't read a {type} partition value"), + )); + } }) } -/// Orders two values of the same type. Values of different types are an error, so the file is -/// kept. -fn compare(value: &Datum, literal: &Datum) -> Result { - if value.data_type() != literal.data_type() { - return Err(type_mismatch(value, literal)); - } - value - .partial_cmp(literal) - .ok_or_else(|| type_mismatch(value, literal)) -} - -fn starts_with(value: &Datum, prefix: &Datum) -> Result { +/// Returns whether `value` starts with `prefix` +fn starts_with(value: &Datum, prefix: &Datum) -> bool { match (value.literal(), prefix.literal()) { (PrimitiveLiteral::String(value), PrimitiveLiteral::String(prefix)) => { - Ok(value.starts_with(prefix.as_str())) + value.starts_with(prefix.as_str()) } - _ => Err(type_mismatch(value, prefix)), + _ => false, } } - -fn type_mismatch(value: &Datum, literal: &Datum) -> Error { - invalid(format!( - "can't compare a {} partition value with a {} value", - value.data_type(), - literal.data_type() - )) -} - -fn invalid(message: String) -> Error { - Error::new(ErrorKind::DataInvalid, message) -} From 249fd05a8060d25de9b5249ff30ba8ecfa2d1dda Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:46:31 +0000 Subject: [PATCH 15/17] refactor: convert only what dynamic filters use DataFusion's dynamic filters use comparisons, IN lists, IS NULL, IS NOT NULL, AND, OR and CASE. This removes the conversions for NOT, NOT IN, LIKE, isnan, starts_with and bare boolean columns. Without NOT, nothing reads the `exact` flag, so the `Converted` wrapper goes too. `datum` no longer converts partition values from int to long or from float to double. Files written before a column was widened now hit the error arm, so we don't skip them. --- .../dynamic_filters/partition_filter.rs | 15 +- .../dynamic_filters/physical_predicate.rs | 328 +++--------------- 2 files changed, 45 insertions(+), 298 deletions(-) diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs index 037a51561..baaf5f1d6 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/partition_filter.rs @@ -238,18 +238,12 @@ fn partition_value( /// Builds a `Datum` of type `r#type` from a stored partition value. iceberg-rust uses /// `Datum::new` for this, which isn't public, so this uses the public constructors. -/// -/// If a column changed from int to long, or from float to double, older files still store -/// values of the old type. This converts them to the new type. Newer iceberg-rust versions do -/// the same conversion when they read manifests. fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { Ok(match (r#type, literal) { (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => Datum::bool(*value), (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => Datum::int(*value), - (PrimitiveType::Long, PrimitiveLiteral::Int(value)) => Datum::long(*value), (PrimitiveType::Long, PrimitiveLiteral::Long(value)) => Datum::long(*value), (PrimitiveType::Float, PrimitiveLiteral::Float(value)) => Datum::float(value.0), - (PrimitiveType::Double, PrimitiveLiteral::Float(value)) => Datum::double(value.0), (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => Datum::double(value.0), (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => Datum::date(*value), (PrimitiveType::Time, PrimitiveLiteral::Long(value)) => Datum::time_micros(*value)?, @@ -280,10 +274,11 @@ fn datum(r#type: &PrimitiveType, literal: &PrimitiveLiteral) -> Result { (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => { Datum::try_from_bytes(&value.to_be_bytes(), PrimitiveType::Uuid)? } - // The stored value doesn't fit its column's type, which we don't expect in a valid - // table. iceberg-rust builds the value anyway. None of its comparisons then match, so - // it skips the file. The public constructors can't build a value like that, so this - // returns an error and we don't skip this file. + // The stored value doesn't fit its column's type. This happens in files written before + // a column was widened, for example from int to long. iceberg-rust builds the value + // anyway. None of its comparisons then match, so it skips the file. The public + // constructors can't build a value like that, so we return an error and don't skip + // the file. _ => { return Err(Error::new( ErrorKind::DataInvalid, diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs index 9ed9f50ff..a38adc9f0 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/physical_predicate.rs @@ -4,7 +4,7 @@ use datafusion::arrow::array::StructArray; use datafusion::functions::core::r#struct::StructFunc; use datafusion::logical_expr::Operator; use datafusion::physical_expr::expressions::{ - BinaryExpr, CaseExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NotExpr, + BinaryExpr, CaseExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, Literal, }; use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; use datafusion::scalar::ScalarValue; @@ -19,59 +19,19 @@ use iceberg::spec::{Datum, PrimitiveType}; pub(crate) fn convert_physical_filter_to_predicate( filter: &Arc, ) -> Option { - convert(filter).map(|converted| converted.predicate) -} - -/// An iceberg predicate that matches every row the DataFusion filter matches. -/// It can also match extra rows, so the caller must still apply the original filter. -struct Converted { - predicate: Predicate, - - /// True if the DataFusion filter could be completely represented by this `iceberg::Predicate` - exact: bool, -} - -impl Converted { - fn exact(predicate: Predicate) -> Self { - Self { - predicate, - exact: true, - } - } - - fn inexact(predicate: Predicate) -> Self { - Self { - predicate, - exact: false, - } - } - - fn and(self, other: Self) -> Self { - Self { - predicate: self.predicate.and(other.predicate), - exact: self.exact && other.exact, - } - } - - fn or(self, other: Self) -> Self { - Self { - predicate: self.predicate.or(other.predicate), - exact: self.exact && other.exact, - } - } + convert(filter) } /// Converts a given `PhysicalExpr` into an `iceberg::Predicate` -/// wrapped in `Converted` which records whether the conversion was exact or inexact +/// +/// We only convert the expressions DataFusion's dynamic filters use: comparisons, IN lists, +/// IS NULL, IS NOT NULL, AND, OR and CASE. /// /// Returns None if the expression could not be converted -fn convert(expr: &Arc) -> Option { +fn convert(expr: &Arc) -> Option { if let Some(binary) = expr.downcast_ref::() { return convert_binary(binary); } - if let Some(not) = expr.downcast_ref::() { - return convert_not(not); - } if let Some(case) = expr.downcast_ref::() { return convert_case(case); } @@ -80,35 +40,22 @@ fn convert(expr: &Arc) -> Option { } if let Some(is_null) = expr.downcast_ref::() { let column = column_name(is_null.arg())?; - return Some(Converted::exact(Reference::new(column).is_null())); + return Some(Reference::new(column).is_null()); } if let Some(is_not_null) = expr.downcast_ref::() { let column = column_name(is_not_null.arg())?; - return Some(Converted::exact(Reference::new(column).is_not_null())); - } - if let Some(like) = expr.downcast_ref::() { - return convert_like(like).map(Converted::exact); - } - if let Some(function) = expr.downcast_ref::() { - return convert_function(function).map(Converted::exact); - } - // A bare boolean column like `WHERE flag` becomes `flag = true`. - if let Some(column) = expr.downcast_ref::() { - let predicate = Reference::new(column.name()).equal_to(Datum::bool(true)); - return Some(Converted::exact(predicate)); + return Some(Reference::new(column).is_not_null()); } // A bare `true`, `false` or NULL. NULL matches nothing. match literal_value(expr)? { - ScalarValue::Boolean(Some(true)) => Some(Converted::exact(Predicate::AlwaysTrue)), - ScalarValue::Boolean(_) | ScalarValue::Null => { - Some(Converted::exact(Predicate::AlwaysFalse)) - } + ScalarValue::Boolean(Some(true)) => Some(Predicate::AlwaysTrue), + ScalarValue::Boolean(_) | ScalarValue::Null => Some(Predicate::AlwaysFalse), _ => None, } } /// Converts a binary expression into an `iceberg::Predicate` -fn convert_binary(binary: &BinaryExpr) -> Option { +fn convert_binary(binary: &BinaryExpr) -> Option { let (left, right) = (binary.left(), binary.right()); match binary.op() { // An AND expression can be converted if *either* side can be converted @@ -118,38 +65,20 @@ fn convert_binary(binary: &BinaryExpr) -> Option { // An OR expression can only be converted if *both* sides can be converted // We cannot drop one of the sides, as that would lead to data being skipped incorrectly Operator::Or => Some(convert(left)?.or(convert(right)?)), - op => compare(left, *op, right).map(Converted::exact), + op => compare(left, *op, right), } } /// ANDs two converted sides. An AND can drop a side it can't convert: /// `a = 1 AND f(b)` becomes `a = 1`. The result matches extra rows, which is fine. -fn both(left: Option, right: Option) -> Option { +fn both(left: Option, right: Option) -> Option { match (left, right) { (Some(left), Some(right)) => Some(left.and(right)), - (Some(side), None) | (None, Some(side)) => Some(Converted::inexact(side.predicate)), + (Some(side), None) | (None, Some(side)) => Some(side), (None, None) => None, } } -/// Converts a NOT expression into an `iceberg::Predicate` -/// -/// NOT is evaluated against an inner expression -/// It can be converted to a predicate *iff* the inner expression -/// can be converted *exactly*. -/// -/// Eg. `NOT (a = 1 AND f(b) = True)` -/// `f(b)` is not representable by an iceberg::Predicate, so it is dropped, and the inner -/// expression becomes `(a = 1)` upon conversion. -/// `NOT (a=1)` would skip rows where `a = 1` but `f(b)` is false, which is incorrect. -/// -/// Returns None if the expression cannot be converted to a predicate -fn convert_not(not: &NotExpr) -> Option { - convert(not.arg()) - .filter(|inner| inner.exact) - .map(|inner| Converted::exact(inner.predicate.negate())) -} - /// Converts a CaseExpr into an `iceberg::Predicate` /// /// In a CASE statement, each WHEN/THEN pair applies a filter over the rows. @@ -161,7 +90,7 @@ fn convert_not(not: &NotExpr) -> Option { /// `CASE a WHEN 1 THEN x ...` works the same way, with `a = 1` as the WHEN. /// /// A missing ELSE means NULL, which matches nothing. -fn convert_case(case: &CaseExpr) -> Option { +fn convert_case(case: &CaseExpr) -> Option { // This handles the two variants for CASE // 1. CASE WHEN a = 1 THEN ... // In this variant, `case.expr()` is `None` because the expression is imbued in the WHEN clause. @@ -174,12 +103,11 @@ fn convert_case(case: &CaseExpr) -> Option { // Iceberg predicates can't always represent a given expression; if that is the case, we drop // the predicate for the WHEN clause. let convert_when = |when| match case.expr() { - Some(value) => compare(value, Operator::Eq, when).map(Converted::exact), + Some(value) => compare(value, Operator::Eq, when), None => convert(when), }; - let predicate = case - .when_then_expr() + case.when_then_expr() .iter() // Converts each individual WHEN/THEN pair into a conjunctive pair // WHEN x > 5 THEN y < 3 becomes (x > 5 AND y < 3) @@ -190,7 +118,6 @@ fn convert_case(case: &CaseExpr) -> Option { .chain(case.else_expr().map(convert)) .collect::>>()? .into_iter() - .map(|branch| branch.predicate) // Combines all predicates in a disjunction, i.e, OR // Multiple WHEN/THEN statements are joined together with ORs // E.g: @@ -198,16 +125,7 @@ fn convert_case(case: &CaseExpr) -> Option { // WHEN x > 10 THEN z ELSE ... // becomes // (x > 5 AND y) OR (x > 10 AND z) - .reduce(Predicate::or)?; - - // Each WHEN/THEN pair in a CASE statement is mutually exclusive - // Subsequent pairs are only executed if the previous one didn't match - // However, in our translation, we `OR` all of them together; subsequent pairs don't check - // whether the previous one matched or not - // - // This means that the filter can return *more* rows than if DataFusion was doing the - // filtering. Due to this, we mark this Predicate as inexact - Some(Converted::inexact(predicate)) + .reduce(Predicate::or) } /// Converts a column <-> value comparison into an `iceberg::Predicate` @@ -237,40 +155,31 @@ fn compare( } /// Converts an InList into an `iceberg::Predicate` -fn convert_in_list(in_list: &InListExpr) -> Option { +fn convert_in_list(in_list: &InListExpr) -> Option { + // Dynamic filters only build IN lists, so we don't convert NOT IN. + if in_list.negated() { + return None; + } + // Something like `struct(a, b) IN (...)`, which checks several columns at once. let Some(column) = column_name(in_list.expr()) else { - return convert_struct_in_list(in_list).map(Converted::inexact); + return convert_struct_in_list(in_list); }; - let column = Reference::new(column); - let values = in_list + let literals = in_list .list() .iter() // We expect every value in the list to be a literal, i.e, x IN (1, 2, 3) // If it isn't, we can't convert this list, and bail early .map(literal_value) - .collect::>>()?; - let has_null = values.iter().any(|value| value.is_null()); - - // `x NOT IN (1, NULL)` would never match anything because x != NULL gives NULL for every row and matches nothing - if in_list.negated() && has_null { - return None; - } - - let literals = values + .collect::>>()? .into_iter() + // NULL never equals anything, so `x IN (1, NULL)` matches the same rows as `x IN (1)` .filter(|value| !value.is_null()) .map(scalar_value_to_datum) .collect::>>()?; - let predicate = if in_list.negated() { - column.is_not_in(literals) - } else { - column.is_in(literals) - }; - - Some(Converted::exact(predicate)) + Some(Reference::new(column).is_in(literals)) } /// Converts a multi-column InList into an `iceberg::Predicate` @@ -283,11 +192,6 @@ fn convert_in_list(in_list: &InListExpr) -> Option { /// /// A column with a NULL value is left out, because the join decides if NULL matches NULL. fn convert_struct_in_list(in_list: &InListExpr) -> Option { - // We can't safely split multi-column pairs and check them individually for a `NOT IN` expr - if in_list.negated() { - return None; - } - let columns = ScalarFunctionExpr::try_downcast_func::(in_list.expr().as_ref())?.args(); @@ -326,62 +230,6 @@ fn field_values(items: &[&Arc], index: usize) -> Option> .collect() } -/// Only simple prefix patterns: `a LIKE 'abc%'` becomes `a STARTS WITH 'abc'`. -/// ILIKE can't be converted, because iceberg's STARTS WITH is case-sensitive. -fn convert_like(like: &LikeExpr) -> Option { - if like.case_insensitive() { - return None; - } - - // Iceberg can only represent prefix matches using plain columns. - // E.g. WHERE s LIKE ... - // It cannot represent complex expressions or functions on the left side of the match - // E.g WHERE upper(s) LIKE ... - // `column_name` will return None if the left side is not a plain column and we don't add a - // predicate for this LikeExpr - let column = Reference::new(column_name(like.expr())?); - - // The pattern being matched *must* be a string literal that ends in a '%', which means that it - // is trying to match the string's prefix. - // - // Iceberg can only support `STARTS WITH` predicates - after stripping the final '%', - // if there are more symbols in the string that could represent a broader match, - // we don't add a predicate for this LikeExpr - let prefix = string_literal(like.pattern())?.strip_suffix('%')?; - if prefix.contains(['%', '_', '\\']) { - return None; - } - - let prefix = prefix_datum(prefix)?; - Some(if like.negated() { - column.not_starts_with(prefix) - } else { - column.starts_with(prefix) - }) -} - -/// Functions are matched by name: `isnan(a)` and `starts_with(a, 'abc')`. -fn convert_function(function: &ScalarFunctionExpr) -> Option { - match (function.name(), function.args()) { - ("isnan", [arg]) => Some(Reference::new(column_name(arg)?).is_nan()), - ("starts_with", [arg, prefix]) => { - let prefix = prefix_datum(string_literal(prefix)?)?; - Some(Reference::new(column_name(arg)?).starts_with(prefix)) - } - _ => None, - } -} - -/// Converts a string used as the prefix in a 'STARTS WITH' clause into a Datum -/// -/// This only works for ASCII strings. This is because iceberg-rust currently has a bug -/// where it can skip rows unintentionally if a partition column truncates the underlying value -/// via truncate(N) and if that underlying value contains non-ascii characters -/// Ref: https://github.com/apache/iceberg-rust/issues/3626 -fn prefix_datum(prefix: &str) -> Option { - prefix.is_ascii().then(|| Datum::string(prefix)) -} - fn column_name(expr: &Arc) -> Option<&str> { expr.downcast_ref::().map(Column::name) } @@ -394,10 +242,6 @@ fn literal_datum(expr: &Arc) -> Option { scalar_value_to_datum(literal_value(expr)?) } -fn string_literal(expr: &Arc) -> Option<&str> { - literal_value(expr)?.try_as_str().flatten() -} - /// Converts a DataFusion value to an iceberg value, for the types iceberg columns use. /// /// Floats aren't converted. DataFusion sorts NaN above every number and -0.0 below 0.0. @@ -442,7 +286,7 @@ mod tests { use datafusion::functions::core::r#struct; use datafusion::logical_expr::Operator; use datafusion::physical_expr::expressions::{ - BinaryExpr, Column, InListExpr, NotExpr, col, in_list, lit, + BinaryExpr, Column, InListExpr, col, in_list, lit, }; use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; use datafusion::prelude::SessionContext; @@ -490,7 +334,11 @@ mod tests { #[test] fn converts_in_lists() { assert_snapshot!(convert_sql("a IN (1, 2)"), @"a IN (1, 2)"); - assert_snapshot!(convert_sql("a NOT IN (1, 2)"), @"a NOT IN (1, 2)"); + } + + #[test] + fn leaves_not_in_lists_unconverted() { + assert_snapshot!(convert_sql("a NOT IN (1, 2)"), @"None"); } #[test] @@ -507,16 +355,6 @@ mod tests { Ok(()) } - #[test] - fn leaves_not_in_with_a_null_unconverted() -> Result<()> { - // Built by hand, the way a hash join builds it. SQL would wrap the NULL in a cast. - let schema = schema(); - let values = vec![lit(1i64), lit(ScalarValue::Int64(None))]; - let filter = in_list(col("a", &schema)?, values, &true, &schema)?; - assert_snapshot!(convert_filter(&filter), @"None"); - Ok(()) - } - #[test] fn leaves_in_lists_with_a_column_item_unconverted() { assert_snapshot!(convert_sql("a IN (b, 1)"), @"None"); @@ -528,35 +366,10 @@ mod tests { } #[test] - fn converts_like_prefixes() { - assert_snapshot!(convert_sql("s LIKE 'ab%'"), @r#"s STARTS WITH "ab""#); - assert_snapshot!(convert_sql("s NOT LIKE 'ab%'"), @r#"s NOT STARTS WITH "ab""#); - } - - #[test] - fn leaves_other_like_patterns_unconverted() { - assert_snapshot!(convert_sql("s LIKE 'a_b%'"), @"None"); - assert_snapshot!(convert_sql("s LIKE 'a%b%'"), @"None"); - assert_snapshot!(convert_sql(r"s LIKE 'a\b%'"), @"None"); - assert_snapshot!(convert_sql("s LIKE 'ab'"), @"None"); - assert_snapshot!(convert_sql("s ILIKE 'ab%'"), @"None"); - } - - #[test] - fn leaves_non_ascii_prefixes_unconverted() { - assert_snapshot!(convert_sql("s LIKE 'éa%'"), @"None"); - assert_snapshot!(convert_sql("starts_with(s, 'éa')"), @"None"); - } - - #[test] - fn converts_isnan_and_starts_with() { - assert_snapshot!(convert_sql("isnan(f)"), @"f IS NAN"); - assert_snapshot!(convert_sql("starts_with(s, 'ab')"), @r#"s STARTS WITH "ab""#); - } - - #[test] - fn converts_a_boolean_column() { - assert_snapshot!(convert_sql("flag"), @"flag = true"); + fn leaves_filters_that_dynamic_filters_dont_use_unconverted() { + assert_snapshot!(convert_sql("NOT (a < 5)"), @"None"); + assert_snapshot!(convert_sql("s LIKE 'ab%'"), @"None"); + assert_snapshot!(convert_sql("flag"), @"None"); } #[test] @@ -587,34 +400,6 @@ mod tests { assert_snapshot!(convert_sql("a = 1 OR a < b"), @"None"); } - #[test] - fn negates_exact_predicates() { - assert_snapshot!(convert_sql("NOT (a < 5)"), @"a >= 5"); - assert_snapshot!(convert_sql("NOT (a IS NULL)"), @"a IS NOT NULL"); - // DataFusion turns `NOT (a IN (1, 2))` into `a NOT IN (1, 2)` itself, and does the - // same for LIKE. Putting them inside an OR keeps the NOT. - assert_snapshot!( - convert_sql("NOT (a IN (1, 2) OR b = 2)"), - @"(a NOT IN (1, 2)) AND (b != 2)" - ); - assert_snapshot!( - convert_sql("NOT (s LIKE 'ab%' OR b = 2)"), - @r#"(s NOT STARTS WITH "ab") AND (b != 2)"# - ); - assert_snapshot!(convert_sql("NOT starts_with(s, 'ab')"), @r#"s NOT STARTS WITH "ab""#); - assert_snapshot!(convert_sql("NOT isnan(f)"), @"f IS NOT NAN"); - assert_snapshot!(convert_sql("NOT flag"), @"flag != true"); - assert_snapshot!(convert_sql("NOT (a = 1 AND b = 2)"), @"(a != 1) OR (b != 2)"); - assert_snapshot!(convert_sql("NOT (a = 1 OR b = 2)"), @"(a != 1) AND (b != 2)"); - } - - #[test] - fn leaves_not_over_a_partly_converted_filter_unconverted() { - assert_snapshot!(convert_sql("NOT (a = 1 AND a < b)"), @"None"); - assert_snapshot!(convert_sql("NOT (a = 1 AND (b = 2 AND a < b))"), @"None"); - assert_snapshot!(convert_sql("NOT (a = 1 OR (b = 2 AND a < b))"), @"None"); - } - #[test] fn converts_a_case_to_each_when_and_its_then_ored_together() { assert_snapshot!( @@ -660,11 +445,6 @@ mod tests { assert_snapshot!(convert_sql("CASE WHEN a = 1 THEN b = 2 ELSE a < b END"), @"None"); } - #[test] - fn leaves_not_over_a_case_unconverted() { - assert_snapshot!(convert_sql("NOT (CASE WHEN a = 1 THEN b = 2 ELSE false END)"), @"None"); - } - #[test] fn converts_a_multi_column_join_filter_to_an_in_list_per_column() -> Result<()> { let filter = struct_in(vec![Some(1), Some(2)], vec![Some(5), Some(6)])?; @@ -679,21 +459,6 @@ mod tests { Ok(()) } - #[test] - fn leaves_a_multi_column_not_in_unconverted() -> Result<()> { - let filter = struct_not_in(vec![Some(1), Some(2)], vec![Some(3), Some(4)])?; - assert_snapshot!(convert_filter(&filter), @"None"); - Ok(()) - } - - #[test] - fn leaves_not_over_a_multi_column_join_filter_unconverted() -> Result<()> { - let filter = struct_in(vec![Some(1), Some(2)], vec![Some(3), Some(4)])?; - let filter: Arc = Arc::new(NotExpr::new(filter)); - assert_snapshot!(convert_filter(&filter), @"None"); - Ok(()) - } - #[test] fn converts_literals_to_iceberg_values() { // Each value is compared with `a`. The translator doesn't check that the types match. @@ -786,19 +551,6 @@ mod tests { /// Builds the filter a hash join on `a` and `b` makes: `struct(a, b) IN (...)`. /// Each pair takes the values at the same position in `a` and `b`. fn struct_in(a: Vec>, b: Vec>) -> Result> { - struct_in_list(a, b, false) - } - - /// Like [`struct_in`], but `NOT IN`. Hash joins never build this. - fn struct_not_in(a: Vec>, b: Vec>) -> Result> { - struct_in_list(a, b, true) - } - - fn struct_in_list( - a: Vec>, - b: Vec>, - negated: bool, - ) -> Result> { let schema = schema(); let fields = Fields::from(vec![ Field::new("a", DataType::Int64, true), @@ -822,7 +574,7 @@ mod tests { Ok(Arc::new(InListExpr::try_new_from_array( Arc::new(columns), Arc::new(values), - negated, + false, &schema, )?)) } From c9ccbd1bbaa853ca7e3430700fd9a772ca319f87 Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:51:35 +0000 Subject: [PATCH 16/17] feat: add a way to add a filter to an Iceberg file scan task The next commit gives the Iceberg reader the scan's dynamic filters, so it can skip row groups. The reader only uses the filter stored on each FileScanTask, and iceberg-rust has no way to change that filter. and_predicate works around this. It turns the task's filter back into an unbound predicate, ANDs the new predicate onto it, binds the result and builds a copy of the task with it. Nothing calls it yet. The copy sets each of the task's fields by hand. A snapshot test lists the task's fields, so it fails if iceberg-rust adds a field the copy would leave out. --- .../src/common/dynamic_filters/mod.rs | 1 + .../src/common/dynamic_filters/task_filter.rs | 232 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 7cc259ed0..0ef97b583 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod partition_filter; pub(crate) mod physical_predicate; pub(crate) mod pruner; +pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs new file mode 100644 index 000000000..2794d59ed --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/task_filter.rs @@ -0,0 +1,232 @@ +//! Adds a filter to a `FileScanTask`. +//! +//! iceberg-rust has no API for this. A task's filter can't be changed, and iceberg-rust has no +//! public way to AND two bound filters. So we turn the task's filter back into an unbound one, +//! AND ours onto it, bind the result, and build a new task with it. +//! +//! If iceberg-rust adds an API for this, delete this file and use the API. + +use iceberg::Result; +use iceberg::expr::{ + BinaryExpression, Bind, BoundPredicate, BoundReference, Predicate, Reference, SetExpression, + UnaryExpression, +}; +use iceberg::scan::FileScanTask; +use iceberg::spec::Schema; + +/// Returns a copy of `task` whose filter is its existing filter AND `predicate`. Returns `None` +/// if that fails. +pub(crate) fn and_predicate(task: &FileScanTask, predicate: Predicate) -> Option { + let combined = match task.predicate() { + Some(existing) => unbind(existing, task.schema())?.and(predicate), + None => predicate, + }; + let combined = combined + .bind(task.schema_ref(), task.case_sensitive()) + .ok()?; + with_predicate(task, combined).ok() +} + +/// Turns a bound predicate back into an unbound one, so we can AND it with another predicate and +/// bind it again. Each column gets its full name from `schema`, like `location.lat`. Returns +/// `None` if a column isn't in `schema`. +fn unbind(predicate: &BoundPredicate, schema: &Schema) -> Option { + let reference = |column: &BoundReference| { + schema + .name_by_field_id(column.field().id) + .map(Reference::new) + }; + Some(match predicate { + BoundPredicate::AlwaysTrue => Predicate::AlwaysTrue, + BoundPredicate::AlwaysFalse => Predicate::AlwaysFalse, + BoundPredicate::And(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.and(unbind(right, schema)?) + } + BoundPredicate::Or(expr) => { + let [left, right] = expr.inputs(); + unbind(left, schema)?.or(unbind(right, schema)?) + } + BoundPredicate::Not(expr) => { + let [inner] = expr.inputs(); + !unbind(inner, schema)? + } + BoundPredicate::Unary(expr) => { + Predicate::Unary(UnaryExpression::new(expr.op(), reference(expr.term())?)) + } + BoundPredicate::Binary(expr) => Predicate::Binary(BinaryExpression::new( + expr.op(), + reference(expr.term())?, + expr.literal().clone(), + )), + BoundPredicate::Set(expr) => Predicate::Set(SetExpression::new( + expr.op(), + reference(expr.term())?, + expr.literals().clone(), + )), + }) +} + +/// Returns a copy of `task` with `predicate` as its filter. Every other field is copied as is. +/// This is only required because iceberg-rust doesn't provide a way to add predicates to an existing FileScanTask +fn with_predicate(task: &FileScanTask, predicate: BoundPredicate) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(task.file_size_in_bytes()) + .with_start(task.start()) + .with_length(task.length()) + .with_record_count(task.record_count()) + .with_first_row_id(task.first_row_id()) + .with_data_sequence_number(task.data_sequence_number()) + .with_data_file_path(task.data_file_path().to_string()) + .with_data_file_format(task.data_file_format()) + .with_schema(task.schema_ref()) + .with_project_field_ids(task.project_field_ids().to_vec()) + .with_predicate(Some(predicate)) + .with_deletes(task.deletes().to_vec()) + .with_partition(task.partition().cloned()) + .with_partition_spec(task.partition_spec().cloned()) + .with_name_mapping(task.name_mapping().cloned()) + .with_unified_partition_type(task.unified_partition_type().cloned()) + .with_sort_order_id(task.sort_order_id()) + .with_sort_order(task.sort_order().cloned()) + .with_case_sensitive(task.case_sensitive()) + .with_key_metadata(task.key_metadata().map(Box::from)) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::scan::FileScanTaskDeleteFile; + use iceberg::spec::{ + DataContentType, DataFileFormat, Datum, Literal, MappedField, NameMapping, NestedField, + PartitionSpec, PrimitiveType, SortOrder, Struct, StructType, Transform, Type, + }; + use std::error::Error; + use std::sync::Arc; + + #[test] + fn unbinding_and_binding_again_gives_the_same_predicate() + -> std::result::Result<(), Box> { + let location = StructType::new(vec![ + NestedField::optional(3, "lat", Type::Primitive(PrimitiveType::Int)).into(), + ]); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "location", Type::Struct(location)).into(), + ]) + .build()?, + ); + let id = || Reference::new("id"); + let predicate = id() + .less_than(Datum::long(5)) + .and(!id().is_in([Datum::long(1), Datum::long(2)])) + .or(Reference::new("name").starts_with(Datum::string("a"))) + .or(Reference::new("location.lat").is_null()) + .bind(Arc::clone(&schema), true)?; + + let unbound = unbind(&predicate, &schema).ok_or("a column isn't in the schema")?; + + assert_eq!(unbound.bind(schema, true)?, predicate); + Ok(()) + } + + /// `with_predicate` copies a task one field at a time. Every field of the task below differs + /// from its default, so the `assert_eq!` fails if the copy leaves a field out. If iceberg-rust + /// adds a field, the copy would silently leave it out. The snapshot lists the task's fields, + /// so it fails when a field is added. + #[test] + fn the_copy_of_a_task_keeps_every_field() -> std::result::Result<(), Box> { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(Arc::clone(&schema)) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let deletes = FileScanTaskDeleteFile::builder() + .with_file_path("deletes.parquet".to_string()) + .with_file_size_in_bytes(1) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build()?; + let name_mapping = NameMapping::new(vec![MappedField::new( + Some(1), + vec!["id".to_string()], + vec![], + )]); + let task = FileScanTask::builder() + .with_file_size_in_bytes(1) + .with_start(2) + .with_length(3) + .with_record_count(Some(4)) + .with_first_row_id(Some(5)) + .with_data_sequence_number(Some(6)) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&schema)) + .with_project_field_ids(vec![1]) + .with_predicate(Some( + Reference::new("id") + .is_null() + .bind(Arc::clone(&schema), true)?, + )) + .with_deletes(vec![deletes]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(7))]))) + .with_unified_partition_type(Some(Arc::new(spec.partition_type(&schema)?))) + .with_partition_spec(Some(Arc::new(spec))) + .with_name_mapping(Some(Arc::new(name_mapping))) + .with_sort_order_id(Some(8)) + .with_sort_order(Some(Arc::new(SortOrder::unsorted_order()))) + .with_case_sensitive(true) + .with_key_metadata(Some(Box::from(b"key".as_slice()))) + .build()?; + let predicate = task.predicate().ok_or("the task has no filter")?; + + let copy = with_predicate(&task, predicate.clone())?; + + assert_eq!(copy, task); + insta::assert_snapshot!(field_names(&task).join("\n"), @" + file_size_in_bytes + start + length + record_count + first_row_id + data_sequence_number + data_file_path + data_file_format + schema + project_field_ids + predicate + deletes + partition + partition_spec + name_mapping + unified_partition_type + sort_order_id + sort_order + case_sensitive + key_metadata + "); + Ok(()) + } + + /// Reads a struct's field names from its pretty-printed `Debug` output. The top-level fields + /// are the lines indented by exactly four spaces. + fn field_names(value: &impl std::fmt::Debug) -> Vec { + format!("{value:#?}") + .lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter(|line| !line.starts_with(' ')) + .filter_map(|line| line.split_once(':')) + .map(|(name, _)| name.to_string()) + .collect() + } +} From f0b5fb39e4e7a2860d674085c0ec80a1652476ce Mon Sep 17 00:00:00 2001 From: Anirudh <6716458+Perseus@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:51:44 +0000 Subject: [PATCH 17/17] feat: give the Iceberg reader dynamic filters to skip row groups With iceberg.dynamic_filtering_enabled on, the scan now gives the reader the newest dynamic filters as it starts each file. The reader uses them to skip row groups, and to drop the rows that don't match. This helps joins on columns that aren't partition columns, and top-N. The reader checks every row once per IN list value. When the IN lists hold more than 200 values in total, we leave them out and keep the min/max bounds. The reader reads a column the file doesn't store as NULL. So a filter on an identity partition column, or on a column added later with a default value, can drop every row of such a file. We accept this for now. Any error leaves the file's task as it was. --- datafusion/distributed-iceberg/README.md | 7 +- .../src/common/dynamic_filters/mod.rs | 1 + .../src/common/dynamic_filters/pruner.rs | 25 +- .../dynamic_filters/row_group_filter.rs | 101 ++++++++ .../src/common/scan_config.rs | 13 +- datafusion/distributed-iceberg/src/config.rs | 8 +- .../tests/row_group_skipping.rs | 234 ++++++++++++++++++ 7 files changed, 378 insertions(+), 11 deletions(-) create mode 100644 datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs create mode 100644 datafusion/distributed-iceberg/tests/row_group_skipping.rs diff --git a/datafusion/distributed-iceberg/README.md b/datafusion/distributed-iceberg/README.md index 9c5b6507b..5c3ddfdbd 100644 --- a/datafusion/distributed-iceberg/README.md +++ b/datafusion/distributed-iceberg/README.md @@ -96,7 +96,12 @@ return `PushedDown::No` for these filters, so DataFusion still checks every row. or aggregate only computes a dynamic filter when a scan below it is a consumer of that filter. Enable `iceberg.dynamic_filtering_enabled` to make Iceberg scans consumers of the dynamic filters pushed into them. With it on, `EXPLAIN` shows them on the scan as -`dynamic_filters=`. Scans don't skip data with these filters yet. +`dynamic_filters=`. Just before iceberg-rust's Parquet reader opens each file, the scan +checks the latest version of the filters: + +- It skips the file if the file's partition value can't match them. +- Otherwise, it gives the filters to the Parquet reader, which uses them to skip row groups, + and to drop the rows that don't match. ```bash cargo test -p datafusion-distributed-iceberg -p datafusion-distributed-iceberg-test-tools diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs index 0ef97b583..38a122662 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod partition_filter; pub(crate) mod physical_predicate; pub(crate) mod pruner; +pub(crate) mod row_group_filter; pub(crate) mod task_filter; diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs index 2f75995b8..e26bf5bc0 100644 --- a/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/pruner.rs @@ -13,9 +13,12 @@ use iceberg::scan::FileScanTask; use crate::common::dynamic_filters::partition_filter::PartitionFilter; use crate::common::dynamic_filters::physical_predicate::convert_physical_filter_to_predicate; +use crate::common::dynamic_filters::row_group_filter::reader_filter; +use crate::common::dynamic_filters::task_filter::and_predicate; -/// Uses the scan's dynamic filters on each file just before the reader starts it. Skips files -/// whose partition value can't match the filters. +/// Applies the scan's dynamic filters to each file just before iceberg-rust's `ArrowReader` opens +/// it. It skips files whose partition value can't match the filters. For the other files, it can +/// add the filters to the file's own filter, so `ArrowReader` can skip row groups. pub(crate) struct DynamicFilterPruner { /// The scan's dynamic filters, combined with AND. dynamic_filters: Arc, @@ -63,6 +66,24 @@ impl DynamicFilterPruner { prune } + /// Adds the dynamic filters to the task's own filter, so iceberg-rust's `ArrowReader` can use + /// them to skip row groups. Returns the task unchanged if adding them fails. + pub(crate) fn with_reader_filter(&mut self, task: FileScanTask) -> FileScanTask { + self.refresh(); + let Some(predicate) = &self.predicate else { + return task; + }; + + // Some data files may not contain identity partition columns in their rows. + // `ArrowReader` reads a column the file doesn't store as NULL, so a filter on it drops + // every row in the file. The same goes for columns added later with a default value. + // We accept this until iceberg-rust fixes it. + // + // We AND onto the task's existing filter, so `ArrowReader` still applies the query's + // WHERE clause. + and_predicate(&task, reader_filter(predicate)).unwrap_or(task) + } + /// Whether no row in the task's partition can match the filters. fn can_skip(&mut self, task: &FileScanTask) -> bool { let (Some(spec), Some(partition)) = (task.partition_spec(), task.partition()) else { diff --git a/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs new file mode 100644 index 000000000..870852512 --- /dev/null +++ b/datafusion/distributed-iceberg/src/common/dynamic_filters/row_group_filter.rs @@ -0,0 +1,101 @@ +use iceberg::expr::Predicate; + +/// The maximum number of values we give iceberg-rust's `ArrowReader` across all the IN lists in the filter. +/// +/// `ArrowReader` checks an IN list by comparing each row with every value in it, one value at a +/// time. So the more values there are, the slower every row gets. A join's filter can have +/// thousands of values. DataFusion keeps each IN list to at most 150 values, but a partitioned +/// join has one list per partition, and the filter combines them with OR. +/// +/// When the filter has more values than this, we leave out all of its IN lists. `ArrowReader` +/// still gets the join's min and max values, so it can still skip row groups outside that range. +/// We use 200 because Iceberg's own row group checks ignore any IN list longer than that. This +/// limit can be removed once `ArrowReader` checks IN lists more optimally (eg. with a hash set) +const MAX_IN_LIST_VALUES: usize = 200; + +/// Returns the dynamic filters to give iceberg-rust's `ArrowReader`. `ArrowReader` uses them to +/// skip row groups, and to drop the rows that don't match. +/// +/// `predicate` is the dynamic filters, converted to Iceberg's filter type. +/// +/// If the filters have too many IN list values, we leave out every IN list. See +/// [`MAX_IN_LIST_VALUES`]. +pub(crate) fn reader_filter(predicate: &Predicate) -> Predicate { + if count_in_list_values(predicate) <= MAX_IN_LIST_VALUES { + return predicate.clone(); + } + keep_filters(predicate, &|filter| !matches!(filter, Predicate::Set(_))) +} + +/// Walks the whole filter and adds up the values in every IN and NOT IN list it finds. +fn count_in_list_values(predicate: &Predicate) -> usize { + match predicate { + Predicate::And(expr) | Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + count_in_list_values(left) + count_in_list_values(right) + } + Predicate::Not(expr) => { + let [inner] = expr.inputs(); + count_in_list_values(inner) + } + Predicate::Set(expr) => expr.literals().len(), + _ => 0, + } +} + +/// Returns a copy of `predicate` with only the filters that `keep` accepts. We replace every +/// other filter with "always true". +/// +/// This is safe because it can only make `predicate` match more rows. So iceberg-rust's +/// `ArrowReader` may read some rows it didn't need to, but it never drops a row that matches. +/// +/// A NOT is the exception. Inside a NOT, "always true" turns into "always false", which would +/// drop rows that match. So we replace the whole NOT with "always true". Our filter converter +/// never produces a NOT, so this doesn't come up in practice. +fn keep_filters(predicate: &Predicate, keep: &dyn Fn(&Predicate) -> bool) -> Predicate { + match predicate { + Predicate::And(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).and(keep_filters(right, keep)) + } + Predicate::Or(expr) => { + let [left, right] = expr.inputs(); + keep_filters(left, keep).or(keep_filters(right, keep)) + } + Predicate::Not(_) => Predicate::AlwaysTrue, + filter if keep(filter) => filter.clone(), + _ => Predicate::AlwaysTrue, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iceberg::expr::Reference; + use iceberg::spec::Datum; + + #[test] + fn leaves_out_long_lists_and_keeps_their_bounds() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(id().less_than_or_equal_to(Datum::long(1000))) + .and(id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"(id >= 0) AND (id <= 1000)"); + } + + #[test] + fn leaves_out_a_not_around_a_long_list() { + let values = (0..=MAX_IN_LIST_VALUES as i64).map(Datum::long); + let predicate = id() + .greater_than_or_equal_to(Datum::long(0)) + .and(!id().is_in(values)); + + insta::assert_snapshot!(reader_filter(&predicate), @"id >= 0"); + } + + fn id() -> Reference { + Reference::new("id") + } +} diff --git a/datafusion/distributed-iceberg/src/common/scan_config.rs b/datafusion/distributed-iceberg/src/common/scan_config.rs index 51dd51a04..a212ed60b 100644 --- a/datafusion/distributed-iceberg/src/common/scan_config.rs +++ b/datafusion/distributed-iceberg/src/common/scan_config.rs @@ -150,11 +150,14 @@ impl ScanConfig { // so with a LIMIT this can count files that never produced output. Skipped files // are dropped before this, and the pruner counts them. let tasks = tasks - .try_filter(move |task| { - let prune = pruner - .as_mut() - .is_some_and(|pruner| pruner.should_prune(task)); - ready(!prune) + .try_filter_map(move |mut task| { + if let Some(pruner) = pruner.as_mut() { + if pruner.should_prune(&task) { + return ready(Ok(None)); + } + task = pruner.with_reader_filter(task); + } + ready(Ok(Some(task))) }) .inspect_ok(move |_| files_opened.add(1)) .boxed(); diff --git a/datafusion/distributed-iceberg/src/config.rs b/datafusion/distributed-iceberg/src/config.rs index 2f9e38c53..3c7f6f94e 100644 --- a/datafusion/distributed-iceberg/src/config.rs +++ b/datafusion/distributed-iceberg/src/config.rs @@ -18,9 +18,11 @@ extensions_options! { pub column_stats_enabled: bool, default = false /// Discover and retain file tasks during physical planning instead of execution. pub eager_file_planning: bool, default = false - /// (Experimental) Whether Iceberg scans are consumers of the dynamic filters pushed - /// into them, so joins and aggregates compute those filters. Scans don't skip data - /// with them yet. + /// (Experimental) Whether Iceberg scans use the dynamic filters pushed into them. When + /// on, joins, sorts with a limit and aggregates build these filters while the query + /// runs. Scans use them to skip files whose partition value can't match. Scans also give + /// them to iceberg-rust's Parquet reader, which uses them to skip row groups and to drop + /// the rows that don't match. pub dynamic_filtering_enabled: bool, default = false } } diff --git a/datafusion/distributed-iceberg/tests/row_group_skipping.rs b/datafusion/distributed-iceberg/tests/row_group_skipping.rs new file mode 100644 index 000000000..933691b8d --- /dev/null +++ b/datafusion/distributed-iceberg/tests/row_group_skipping.rs @@ -0,0 +1,234 @@ +mod tests { + use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::datasource::source::DataSourceExec; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion_distributed::test_utils::metrics::node_metrics; + use datafusion_distributed::{ + DistributedExt, DistributedLeafExec, DistributedMetricsFormat, + rewrite_distributed_plan_with_metrics, + }; + use datafusion_distributed_iceberg::IcebergExt; + use datafusion_distributed_iceberg_test_tools::{ + FixtureCatalog, FixtureTable, IcebergTestHarness, + }; + use iceberg::TableIdent; + use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; + use parquet::file::properties::WriterProperties; + use std::error::Error; + use std::sync::Arc; + + // The join keeps one id from each of the two `events` files. + const TWO_IDS: &str = "SELECT events.id FROM events \ + JOIN (VALUES (5), (150005)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + + #[tokio::test] + async fn the_reader_skips_row_groups_the_join_filter_rules_out() -> Result<(), Box> { + let query = "SELECT events.id, events.payload FROM events \ + JOIN (VALUES (150005), (150010)) AS ids(id) ON events.id = ids.id \ + ORDER BY events.id"; + let with_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + let without_filter = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", false) + .build() + .await?; + + let (with_filter_results, with_filter_plan) = run(&with_filter, query).await?; + let (without_filter_results, without_filter_plan) = run(&without_filter, query).await?; + + insta::assert_snapshot!(with_filter_results, @" + +--------+----------------------------------------+ + | id | payload | + +--------+----------------------------------------+ + | 150005 | event 150005 | + | 150010 | event 150010 | + +--------+----------------------------------------+ + "); + assert_eq!(with_filter_results, without_filter_results); + // `DataSourceExec` 0 is the join's `VALUES` table, and 1 is the `events` scan. + // Both ids are in one row group of the second file, so with the filter iceberg-rust's + // `ArrowReader` reads 1 of the 40 row groups. Both runs also read the last 512 KiB of each + // file to get its footer, so we only check that the filter cuts the bytes read by more + // than 5x. + let with_filter_bytes = + node_metrics::(&with_filter_plan, "bytes_scanned", 1); + let without_filter_bytes = + node_metrics::(&without_filter_plan, "bytes_scanned", 1); + assert!( + with_filter_bytes * 5 < without_filter_bytes, + "read {with_filter_bytes} bytes with the join filter and {without_filter_bytes} without" + ); + Ok(()) + } + + #[tokio::test] + async fn eager_planning_workers_give_the_reader_the_join_filter() -> Result<(), Box> + { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .with_workers(2) + .configure_session(|state| { + state.with_distributed_file_scan_config_bytes_per_partition(1) + })? + .build() + .await?; + + let (results, plan) = run(&harness, TWO_IDS).await?; + + insta::assert_snapshot!(results, @" + +--------+ + | id | + +--------+ + | 5 | + | 150005 | + +--------+ + "); + // The coordinator plans the files and sends them to the workers. Each worker still gives + // `ArrowReader` the join filter, so `ArrowReader` drops the rows it rules out and the scan + // returns only the two ids. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 2 + ); + Ok(()) + } + + #[tokio::test] + async fn the_top_n_filter_reaches_files_that_start_later() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", true) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + // Read one file at a time, so the top-N filter can tighten before the next file starts. + .with_config_option("iceberg.data_file_concurrency_limit", 1) + .configure_session(|mut state| { + let session = state.config().get_or_insert_default(); + session.options_mut().execution.target_partitions = 1; + Ok(state) + })? + .build() + .await?; + + let (results, plan) = run(&harness, "SELECT id FROM events ORDER BY id LIMIT 3").await?; + + insta::assert_snapshot!(results, @" + +----+ + | id | + +----+ + | 0 | + | 1 | + | 2 | + +----+ + "); + // When the first file starts, the top 3 aren't known yet, so the scan returns all 100,000 + // of its rows. By the time the second file starts, they are, and `ArrowReader` drops + // every row in it. + assert_eq!( + node_metrics::(&plan, "output_rows", 0), + 100_000 + ); + Ok(()) + } + + #[tokio::test] + async fn the_reader_keeps_the_where_filter_too() -> Result<(), Box> { + let harness = IcebergTestHarness::builder() + .with_session_extension(|state| state.with_iceberg_integration(Default::default())) + .with_table(events_by_id().await?) + .with_config_option("iceberg.eager_file_planning", false) + .with_config_option("iceberg.dynamic_filtering_enabled", true) + .build() + .await?; + + let (results, plan) = run( + &harness, + "SELECT events.id, events.kind FROM events \ + JOIN (VALUES (5), (60000), (150004)) AS ids(id) ON events.id = ids.id \ + WHERE events.kind = 'click' ORDER BY events.id", + ) + .await?; + + insta::assert_snapshot!(results, @" + +--------+-------+ + | id | kind | + +--------+-------+ + | 60000 | click | + | 150004 | click | + +--------+-------+ + "); + // `ArrowReader` applies both filters. The join filter keeps ids 5, 60000 and 150004. The + // `WHERE` filter keeps only clicks, which drops the odd id 5. If adding the join filter + // lost the `WHERE` filter, the scan would return 3 rows. + assert_eq!(node_metrics::(&plan, "output_rows", 1), 2); + Ok(()) + } + + /// Runs `sql`, and returns its results and its plan with the metrics from the run. + async fn run( + harness: &IcebergTestHarness, + sql: &str, + ) -> Result<(String, Arc), Box> { + let plan = harness.physical_plan(sql).await?; + let batches = collect(Arc::clone(&plan), harness.state().task_ctx()).await?; + let plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .await?; + Ok((pretty_format_batches(&batches)?.to_string(), plan)) + } + + // The `events` table has two files of 100,000 rows each, sorted by `id`: + // - The first file has ids 0 to 99,999, and the second has 100,000 to 199,999. + // - `kind` is "click" for even ids and "view" for odd ones. + // - Each file has 20 row groups of 5,000 rows. + // - Each file is larger than the 512 KiB `ArrowReader` reads first to find the footer, so + // skipping row groups reads fewer bytes. + async fn events_by_id() -> Result> { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "kind", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "payload", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?; + let fixtures = FixtureCatalog::try_new().await?; + let mut table = fixtures + .table(TableIdent::from_strs(["default", "events"])?, schema) + .create() + .await?; + let writer_properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(5_000)) + .build(); + let mut append = table.append().writer_properties(writer_properties); + for ids in [0..100_000, 100_000..200_000] { + let kinds: Vec<&str> = ids + .clone() + .map(|id| if id % 2 == 0 { "click" } else { "view" }) + .collect(); + let payloads: Vec = ids.clone().map(|id| format!("event {id:>32}")).collect(); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(ids)); + let kinds: ArrayRef = Arc::new(StringArray::from(kinds)); + let payloads: ArrayRef = Arc::new(StringArray::from(payloads)); + append = append.file(RecordBatch::try_from_iter([ + ("id", ids), + ("kind", kinds), + ("payload", payloads), + ])?); + } + append.commit().await?; + Ok(table) + } +}