Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 19 additions & 38 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -408,3 +408,24 @@ debug = false
debug-assertions = false
strip = "debuginfo"
incremental = false

# Pin arrow/parquet to arrow-rs main for pre-release testing
[patch.crates-io]
arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-arith = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-avro = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-cmp = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-csv = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-data = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-json = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-ord = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-row = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-select = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-string = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "dd240116c75a15a2a048a041c808d67f3f32a375" }
15 changes: 14 additions & 1 deletion datafusion/common/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1370,7 +1370,7 @@ config_namespace! {

/// (reading) If true, filter expressions are be applied during the parquet decoding operation to
/// reduce the number of rows decoded. This optimization is sometimes called "late materialization".
pub pushdown_filters: bool, default = false
pub pushdown_filters: bool, default = true

/// (reading) If true, filter expressions evaluated during the parquet decoding operation
/// will be reordered heuristically to minimize the cost of evaluation. If false,
Expand All @@ -1383,6 +1383,19 @@ config_namespace! {
/// pattern of selected rows.
pub force_filter_selections: bool, default = false

/// (reading) Controls the I/O pattern used when `pushdown_filters` is
/// enabled. If false, all data pages needed to read a row group (for
/// both filter evaluation and the output projection) are fetched in a
/// single request, the same I/O pattern used when `pushdown_filters` is
/// disabled. If true (the default), data is fetched progressively:
/// first the columns needed by each filter, then, after the filters are
/// evaluated, the remaining projected columns for the rows that passed.
/// Progressive fetching can reduce the total bytes read when the file
/// has a Parquet offset index, at the cost of additional I/O requests
/// per row group; files without an offset index are always read with a
/// single request per row group.
pub progressive_io: bool, default = true

/// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`,
/// and `Binary/BinaryLarge` with `BinaryView`.
///
Expand Down
3 changes: 3 additions & 0 deletions datafusion/common/src/file_options/parquet_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ impl ParquetOptions {
pushdown_filters: _,
reorder_filters: _,
force_filter_selections: _, // not used for writer props
progressive_io: _, // not used for writer props
allow_single_file_parallelism: _,
maximum_parallel_row_group_writers: _,
maximum_buffered_record_batches_per_stream: _,
Expand Down Expand Up @@ -423,6 +424,7 @@ mod tests {
pushdown_filters: defaults.pushdown_filters,
reorder_filters: defaults.reorder_filters,
force_filter_selections: defaults.force_filter_selections,
progressive_io: defaults.progressive_io,
allow_single_file_parallelism: defaults.allow_single_file_parallelism,
maximum_parallel_row_group_writers: defaults
.maximum_parallel_row_group_writers,
Expand Down Expand Up @@ -551,6 +553,7 @@ mod tests {
pushdown_filters: global_options_defaults.pushdown_filters,
reorder_filters: global_options_defaults.reorder_filters,
force_filter_selections: global_options_defaults.force_filter_selections,
progressive_io: global_options_defaults.progressive_io,
allow_single_file_parallelism: global_options_defaults
.allow_single_file_parallelism,
maximum_parallel_row_group_writers: global_options_defaults
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
//! Two properties of the planner shape these queries, and both are easy to get
//! wrong:
//!
//! * `execution.parquet.pushdown_filters` must be enabled. It defaults to
//! `false`, in which case no row filter is built and every case below
//! * `execution.parquet.pushdown_filters` must be enabled (it is by
//! default). If it is disabled no row filter is built and every case below
//! degenerates to a plain scan.
//! * The predicate must be a *single* conjunct. `build_row_filter` calls
//! `split_conjunction` before building filter candidates, and each candidate
Expand Down
6 changes: 4 additions & 2 deletions datafusion/core/src/dataframe/parquet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,11 @@ mod tests {
.select_columns(&["bool_col", "int_col"])?;

let plan = df.explain(false, false)?.collect().await?;
// Filters all the way to Parquet
// Filters all the way to Parquet: with `pushdown_filters` enabled by
// default the predicate is evaluated by the scan itself
let formatted = pretty::pretty_format_batches(&plan)?.to_string();
assert!(formatted.contains("FilterExec: id@0 = 1"), "{formatted}");
assert!(formatted.contains("predicate=id@0 = 1"), "{formatted}");
assert!(!formatted.contains("FilterExec"), "{formatted}");

Ok(())
}
Expand Down
6 changes: 4 additions & 2 deletions datafusion/core/src/datasource/view_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,11 +322,13 @@ mod tests {

let plan = df.explain(false, false)?.collect().await?;

// Filters all the way to Parquet
// Filters all the way to Parquet: with `pushdown_filters` enabled by
// default the predicate is evaluated by the scan itself
let formatted = arrow::util::pretty::pretty_format_batches(&plan)
.unwrap()
.to_string();
assert!(formatted.contains("FilterExec: id@0 = 1"));
assert!(formatted.contains("predicate=id@0 = 1"), "{formatted}");
assert!(!formatted.contains("FilterExec"), "{formatted}");
Ok(())
}

Expand Down
Loading