Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions datafusion/catalog-listing/src/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ pub fn evaluate_partition_prefix<'a>(
}
}

pub fn filter_partitioned_file(
fn filter_partitions(
pf: PartitionedFile,
filters: &[Expr],
df_schema: &DFSchema,
Expand Down Expand Up @@ -447,7 +447,7 @@ pub async fn pruned_partition_list<'a>(
))
})
.try_filter_map(move |pf| {
futures::future::ready(filter_partitioned_file(pf, filters, &df_schema))
futures::future::ready(filter_partitions(pf, filters, &df_schema))
})
.boxed())
}
Expand Down
54 changes: 1 addition & 53 deletions datafusion/catalog-listing/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use datafusion_common::plan_err;
use datafusion_datasource::ListingTableUrl;
use datafusion_datasource::file_format::FileFormat;
use datafusion_execution::config::SessionConfig;
use datafusion_expr::{Partitioning, SortExpr};
use datafusion_expr::SortExpr;
use futures::StreamExt;
use futures::TryStreamExt;
use itertools::Itertools;
Expand Down Expand Up @@ -61,46 +61,6 @@ pub struct ListingOptions {
/// multiple equivalent orderings, the outer `Vec` will have a
/// single element.
pub file_sort_order: Vec<Vec<SortExpr>>,
/// Declared output partitioning for scans from this table.
///
/// Expressions are logical expressions over the full table schema. When set,
/// [`ListingTable`](crate::ListingTable) creates one file group per
/// declared output partition. When unset, file grouping uses the scan-time
/// [`SessionConfig::target_partitions`](datafusion_execution::config::SessionConfig::target_partitions).
///
/// Files are listed in path order, split into whole-file groups across the
/// declared partition count, and then padded with trailing empty groups when
/// needed. DataFusion does not route files by partition values or validate
/// row placement, so callers must ensure file group `i` contains rows for
/// partition `i`. Layouts that require explicit file-to-partition assignment
/// are not supported.
///
/// For example, range partitioning on column `a` with split points
/// `[10, 20, 30]` declares four output partitions. With three path-ordered
/// files, the trailing partition is preserved as empty:
///
/// ```text
/// files in path order: f0, f1, f2
///
/// file groups:
/// partition 0: [f0]
/// partition 1: [f1]
/// partition 2: [f2]
/// partition 3: []
/// ```
///
/// With five path-ordered files, a partition can contain multiple files:
///
/// ```text
/// files in path order: f0, f1, f2, f3, f4
///
/// file groups:
/// partition 0: [f0, f1]
/// partition 1: [f2, f3]
/// partition 2: [f4]
/// partition 3: []
/// ```
pub output_partitioning: Option<Partitioning>,
}

impl ListingOptions {
Expand All @@ -118,7 +78,6 @@ impl ListingOptions {
collect_stat: false,
target_partitions: 1,
file_sort_order: vec![],
output_partitioning: None,
}
}

Expand Down Expand Up @@ -177,17 +136,6 @@ impl ListingOptions {
self
}

/// Set declared output partitioning.
///
/// See [`Self::output_partitioning`] for the contract.
pub fn with_output_partitioning(
mut self,
output_partitioning: Option<Partitioning>,
) -> Self {
self.output_partitioning = output_partitioning;
self
}

/// Set `table partition columns` on [`ListingOptions`] and returns self.
///
/// "partition columns," used to support [Hive Partitioning], are
Expand Down
Loading
Loading