Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions datafusion/catalog-listing/src/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};
use datafusion_common::pruning::PrunableStatistics;
use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_common::{
Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err,
plan_err, project_schema,
Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, TableReadSelector,
internal_datafusion_err, plan_err, project_schema,
};
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_groups::FileGroup;
Expand Down Expand Up @@ -199,6 +199,7 @@ pub struct ListingTable {
/// validation. Constant for the table, so computed once here instead of per
/// file.
file_schema_fingerprint: Arc<SchemaFingerprint>,
read_selector: Option<TableReadSelector>,
}

impl ListingTable {
Expand Down Expand Up @@ -272,6 +273,7 @@ impl ListingTable {
column_defaults: HashMap::new(),
expr_adapter_factory: config.expr_adapter_factory,
file_schema_fingerprint,
read_selector: None,
};

Ok(table)
Expand Down Expand Up @@ -313,6 +315,20 @@ impl ListingTable {
self
}

/// Sets the [`TableReadSelector`] reported by
/// [`TableProvider::read_selector`], which identifies which read of the
/// table this provider performs, such as a version of the table.
///
/// This only labels the provider, it does not change the data it reads:
/// the provider must already be set up for that read.
pub fn with_read_selector(
mut self,
read_selector: Option<TableReadSelector>,
) -> Self {
self.read_selector = read_selector;
self
}

/// Get paths ref
pub fn table_paths(&self) -> &Vec<ListingTableUrl> {
&self.table_paths
Expand Down Expand Up @@ -534,6 +550,10 @@ impl TableProvider for ListingTable {
Some(&self.constraints)
}

fn read_selector(&self) -> Option<&TableReadSelector> {
self.read_selector.as_ref()
}

fn table_type(&self) -> TableType {
TableType::Base
}
Expand Down
6 changes: 5 additions & 1 deletion datafusion/catalog/src/default_table_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use std::sync::Arc;
use crate::TableProvider;

use arrow::datatypes::SchemaRef;
use datafusion_common::{Constraints, internal_err};
use datafusion_common::{Constraints, TableReadSelector, internal_err};
use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource, TableType};

/// Implements [`TableSource`] for a [`TableProvider`]
Expand Down Expand Up @@ -68,6 +68,10 @@ impl TableSource for DefaultTableSource {
fn get_column_default(&self, column: &str) -> Option<&Expr> {
self.table_provider.get_column_default(column)
}

fn read_selector(&self) -> Option<&TableReadSelector> {
self.table_provider.read_selector()
}
}

/// Wrap TableProvider in TableSource
Expand Down
9 changes: 9 additions & 0 deletions datafusion/catalog/src/dynamic_file/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

use crate::{CatalogProvider, CatalogProviderList, SchemaProvider, TableProvider};
use async_trait::async_trait;
use datafusion_common::TableReadSelector;
use std::fmt::Debug;
use std::sync::Arc;

Expand Down Expand Up @@ -138,6 +139,14 @@ impl SchemaProvider for DynamicFileSchemaProvider {
self.factory.try_new(name).await
}

async fn table_with_read_selector(
&self,
name: &str,
selector: &TableReadSelector,
) -> datafusion_common::Result<Option<Arc<dyn TableProvider>>> {
self.inner.table_with_read_selector(name, selector).await
}

fn register_table(
&self,
name: String,
Expand Down
22 changes: 21 additions & 1 deletion datafusion/catalog/src/empty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::sync::Arc;

use arrow::datatypes::*;
use async_trait::async_trait;
use datafusion_common::{Result, project_schema};
use datafusion_common::{Result, TableReadSelector, project_schema};
use datafusion_expr::{Expr, TableType};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
Expand All @@ -22,6 +22,7 @@ use crate::TableProvider;
pub struct EmptyTable {
schema: SchemaRef,
partitions: usize,
read_selector: Option<TableReadSelector>,
}

impl EmptyTable {
Expand All @@ -30,9 +31,24 @@ impl EmptyTable {
Self {
schema,
partitions: 1,
read_selector: None,
}
}

/// Sets the [`TableReadSelector`] reported by
/// [`TableProvider::read_selector`], which identifies which read of the
/// table this provider performs, such as a version of the table.
///
/// This only labels the provider, it does not change the data it reads:
/// the provider must already be set up for that read.
pub fn with_read_selector(
mut self,
read_selector: Option<TableReadSelector>,
) -> Self {
self.read_selector = read_selector;
self
}

/// Creates a new EmptyTable with specified partition number.
pub fn with_partitions(mut self, partitions: usize) -> Self {
self.partitions = partitions;
Expand All @@ -50,6 +66,10 @@ impl TableProvider for EmptyTable {
TableType::Base
}

fn read_selector(&self) -> Option<&TableReadSelector> {
self.read_selector.as_ref()
}

async fn scan(
&self,
_state: &dyn Session,
Expand Down
24 changes: 22 additions & 2 deletions datafusion/catalog/src/memory/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::error::Result;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraints, DFSchema, SchemaExt, internal_datafusion_err, internal_err,
not_impl_err, plan_err,
Constraints, DFSchema, SchemaExt, TableReadSelector, internal_datafusion_err,
internal_err, not_impl_err, plan_err,
};
use datafusion_datasource::memory::{MemSink, MemorySourceConfig};
use datafusion_datasource::sink::DataSinkExec;
Expand Down Expand Up @@ -70,6 +70,7 @@ pub struct MemTable {
/// Optional pre-known sort order(s). Must be `SortExpr`s.
/// inserting data into this table removes the order
pub sort_order: Arc<Mutex<Vec<Vec<SortExpr>>>>,
read_selector: Option<TableReadSelector>,
}

impl MemTable {
Expand Down Expand Up @@ -103,9 +104,24 @@ impl MemTable {
constraints: Constraints::default(),
column_defaults: HashMap::new(),
sort_order: Arc::new(Mutex::new(vec![])),
read_selector: None,
})
}

/// Sets the [`TableReadSelector`] reported by
/// [`TableProvider::read_selector`], which identifies which read of the
/// table this provider performs, such as a version of the table.
///
/// This only labels the provider, it does not change the data it reads:
/// the provider must already be set up for that read.
pub fn with_read_selector(
mut self,
read_selector: Option<TableReadSelector>,
) -> Self {
self.read_selector = read_selector;
self
}

/// Assign constraints
pub fn with_constraints(mut self, constraints: Constraints) -> Self {
self.constraints = constraints;
Expand Down Expand Up @@ -178,6 +194,10 @@ impl TableProvider for MemTable {
Some(&self.constraints)
}

fn read_selector(&self) -> Option<&TableReadSelector> {
self.read_selector.as_ref()
}

fn table_type(&self) -> TableType {
TableType::Base
}
Expand Down
22 changes: 21 additions & 1 deletion datafusion/catalog/src/view.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ use crate::TableProvider;

use arrow::datatypes::SchemaRef;
use async_trait::async_trait;
use datafusion_common::Column;
use datafusion_common::error::Result;
use datafusion_common::{Column, TableReadSelector};
use datafusion_expr::TableType;
use datafusion_expr::{Expr, LogicalPlan};
use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown};
Expand All @@ -28,6 +28,7 @@ pub struct ViewTable {
table_schema: SchemaRef,
/// SQL used to create the view, if available
definition: Option<String>,
read_selector: Option<TableReadSelector>,
}

impl ViewTable {
Expand All @@ -44,9 +45,24 @@ impl ViewTable {
logical_plan,
table_schema,
definition,
read_selector: None,
}
}

/// Sets the [`TableReadSelector`] reported by
/// [`TableProvider::read_selector`], which identifies which read of the
/// table this provider performs, such as a version of the table.
///
/// This only labels the provider, it does not change the data it reads:
/// the provider must already be set up for that read.
pub fn with_read_selector(
mut self,
read_selector: Option<TableReadSelector>,
) -> Self {
self.read_selector = read_selector;
self
}

/// Get definition ref
pub fn definition(&self) -> Option<&String> {
self.definition.as_ref()
Expand All @@ -72,6 +88,10 @@ impl TableProvider for ViewTable {
TableType::View
}

fn read_selector(&self) -> Option<&TableReadSelector> {
self.read_selector.as_ref()
}

fn get_table_definition(&self) -> Option<&str> {
self.definition.as_deref()
}
Expand Down
2 changes: 2 additions & 0 deletions datafusion/common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ mod join_type;
mod param_value;
mod partitioning;
mod schema_reference;
mod table_read_selector;
mod table_reference;
mod unnest;

Expand Down Expand Up @@ -89,6 +90,7 @@ pub use scalar::{ScalarType, ScalarValue};
pub use schema_reference::SchemaReference;
pub use spans::{Location, Span, Spans};
pub use stats::{ColumnStatistics, Statistics};
pub use table_read_selector::{TableReadSelector, TableReadSelectorExtension};
pub use table_reference::{ResolvedTableReference, TableReference};
pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions};
pub use utils::project_schema;
Expand Down
Loading
Loading