From 362b3240bdb1cb423168987584cb76f6da89972f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 7 Nov 2026 16:02:57 +0100 Subject: [PATCH 1/3] feat: support time travel queries with VERSION AS OF / TIMESTAMP AS OF Plan time travel clauses such as `VERSION AS OF 3`, `TIMESTAMP AS OF '2024-01-01'`, `FOR SYSTEM_TIME AS OF ...` and `AT (VERSION => ...)`. All syntaxes are normalized to a `TableVersion`, either a `Version` holding any scalar value or a `Timestamp`. The version expression is evaluated during planning, so catalogs only see constants. - `SchemaProvider::table_at` returns a specific version of a table. The default returns "not implemented" rather than the current version. - `TableProvider::version` / `TableSource::version` report the version a table reads, which may differ from the requested version (e.g. a timestamp resolving to a snapshot). `ListingTable`, `MemTable`, `ViewTable` and `EmptyTable` gain `with_version`. - `TableScan` equality, ordering and hashing include the version, so subqueries over different versions of a table are not merged. EXPLAIN shows the version as `t@3`. - `RelationPlanner` extensions that handle time travel themselves keep working: the current version of versioned tables is still fetched when relation planners are registered. - Unparser, protobuf and Substrait support; parameters and execution time dependent functions (`now()`) are rejected in prepared statements. Co-Authored-By: Claude Opus 5.5 --- datafusion/catalog-listing/src/table.rs | 18 +- .../catalog/src/default_table_source.rs | 6 +- .../catalog/src/dynamic_file/catalog.rs | 9 + datafusion/catalog/src/empty.rs | 16 +- datafusion/catalog/src/memory/table.rs | 18 +- datafusion/catalog/src/view.rs | 16 +- datafusion/common/src/lib.rs | 2 + datafusion/common/src/table_version.rs | 101 +++++ .../core/src/execution/session_state.rs | 156 +++++++- .../tests/user_defined/relation_planner.rs | 177 ++++++++- datafusion/expr/src/logical_plan/display.rs | 5 + datafusion/expr/src/logical_plan/plan.rs | 17 +- datafusion/expr/src/planner.rs | 37 +- datafusion/expr/src/table_source.rs | 16 +- .../proto-models/proto/datafusion.proto | 18 + datafusion/proto-models/src/from_proto.rs | 20 +- .../proto-models/src/generated/pbjson.rs | 178 +++++++++ .../proto-models/src/generated/prost.rs | 30 ++ datafusion/proto-models/src/to_proto.rs | 17 +- datafusion/proto/src/logical_plan/mod.rs | 43 +- .../tests/cases/roundtrip_logical_plan.rs | 38 +- datafusion/session/src/schema.rs | 26 +- datafusion/session/src/table.rs | 20 +- datafusion/sql/src/relation/mod.rs | 222 ++++++++++- datafusion/sql/src/resolve.rs | 276 ++++++++++++- datafusion/sql/src/statement.rs | 12 +- datafusion/sql/src/unparser/dialect.rs | 41 ++ datafusion/sql/src/unparser/plan.rs | 55 ++- datafusion/sql/tests/cases/plan_to_sql.rs | 86 +++- datafusion/sql/tests/common/mod.rs | 53 ++- datafusion/sql/tests/sql_integration.rs | 23 ++ datafusion/sqllogictest/src/test_context.rs | 5 + .../src/test_context/time_travel.rs | 149 +++++++ .../sqllogictest/test_files/time_travel.slt | 368 ++++++++++++++++++ .../src/logical_plan/producer/rel/read_rel.rs | 8 +- docs/source/library-user-guide/catalogs.md | 71 ++++ .../library-user-guide/upgrading/57.0.0.md | 23 ++ docs/source/user-guide/sql/select.md | 49 ++- 38 files changed, 2359 insertions(+), 66 deletions(-) create mode 100644 datafusion/common/src/table_version.rs create mode 100644 datafusion/sqllogictest/src/test_context/time_travel.rs create mode 100644 datafusion/sqllogictest/test_files/time_travel.slt diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index 05e3c6c9d1674..e00e59ec494c2 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -13,8 +13,8 @@ use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::pruning::PrunableStatistics; use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err, - plan_err, project_schema, + Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, TableVersion, + internal_datafusion_err, plan_err, project_schema, }; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_groups::FileGroup; @@ -199,6 +199,8 @@ pub struct ListingTable { /// validation. Constant for the table, so computed once here instead of per /// file. file_schema_fingerprint: Arc, + /// The version of the table this provider reads, if versioned + version: Option, } impl ListingTable { @@ -272,6 +274,7 @@ impl ListingTable { column_defaults: HashMap::new(), expr_adapter_factory: config.expr_adapter_factory, file_schema_fingerprint, + version: None, }; Ok(table) @@ -313,6 +316,13 @@ impl ListingTable { self } + /// Set the version of the table this provider reads, for time travel + /// queries. See [`TableProvider::version`]. + pub fn with_version(mut self, version: Option) -> Self { + self.version = version; + self + } + /// Get paths ref pub fn table_paths(&self) -> &Vec { &self.table_paths @@ -534,6 +544,10 @@ impl TableProvider for ListingTable { Some(&self.constraints) } + fn version(&self) -> Option<&TableVersion> { + self.version.as_ref() + } + fn table_type(&self) -> TableType { TableType::Base } diff --git a/datafusion/catalog/src/default_table_source.rs b/datafusion/catalog/src/default_table_source.rs index 6e889e55a856b..f9117648f600d 100644 --- a/datafusion/catalog/src/default_table_source.rs +++ b/datafusion/catalog/src/default_table_source.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use crate::TableProvider; use arrow::datatypes::SchemaRef; -use datafusion_common::{Constraints, internal_err}; +use datafusion_common::{Constraints, TableVersion, internal_err}; use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource, TableType}; /// Implements [`TableSource`] for a [`TableProvider`] @@ -68,6 +68,10 @@ impl TableSource for DefaultTableSource { fn get_column_default(&self, column: &str) -> Option<&Expr> { self.table_provider.get_column_default(column) } + + fn version(&self) -> Option<&TableVersion> { + self.table_provider.version() + } } /// Wrap TableProvider in TableSource diff --git a/datafusion/catalog/src/dynamic_file/catalog.rs b/datafusion/catalog/src/dynamic_file/catalog.rs index 689c2d6c0d4f1..69f253017e916 100644 --- a/datafusion/catalog/src/dynamic_file/catalog.rs +++ b/datafusion/catalog/src/dynamic_file/catalog.rs @@ -6,6 +6,7 @@ use crate::{CatalogProvider, CatalogProviderList, SchemaProvider, TableProvider}; use async_trait::async_trait; +use datafusion_common::TableVersion; use std::fmt::Debug; use std::sync::Arc; @@ -138,6 +139,14 @@ impl SchemaProvider for DynamicFileSchemaProvider { self.factory.try_new(name).await } + async fn table_at( + &self, + name: &str, + version: &TableVersion, + ) -> datafusion_common::Result>> { + self.inner.table_at(name, version).await + } + fn register_table( &self, name: String, diff --git a/datafusion/catalog/src/empty.rs b/datafusion/catalog/src/empty.rs index e4721aaaf5e8f..48a4bb2cb4562 100644 --- a/datafusion/catalog/src/empty.rs +++ b/datafusion/catalog/src/empty.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use arrow::datatypes::*; use async_trait::async_trait; -use datafusion_common::{Result, project_schema}; +use datafusion_common::{Result, TableVersion, project_schema}; use datafusion_expr::{Expr, TableType}; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::empty::EmptyExec; @@ -22,6 +22,8 @@ use crate::TableProvider; pub struct EmptyTable { schema: SchemaRef, partitions: usize, + /// The version of the table this provider reads, if versioned + version: Option, } impl EmptyTable { @@ -30,9 +32,17 @@ impl EmptyTable { Self { schema, partitions: 1, + version: None, } } + /// Set the version of the table this provider reads, for time travel + /// queries. See [`TableProvider::version`]. + pub fn with_version(mut self, version: Option) -> Self { + self.version = version; + self + } + /// Creates a new EmptyTable with specified partition number. pub fn with_partitions(mut self, partitions: usize) -> Self { self.partitions = partitions; @@ -50,6 +60,10 @@ impl TableProvider for EmptyTable { TableType::Base } + fn version(&self) -> Option<&TableVersion> { + self.version.as_ref() + } + async fn scan( &self, _state: &dyn Session, diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 6b7d69dbc631e..3c9ae696693bb 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -19,8 +19,8 @@ use arrow::record_batch::RecordBatch; use datafusion_common::error::Result; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, internal_datafusion_err, internal_err, - not_impl_err, plan_err, + Constraints, DFSchema, SchemaExt, TableVersion, internal_datafusion_err, + internal_err, not_impl_err, plan_err, }; use datafusion_datasource::memory::{MemSink, MemorySourceConfig}; use datafusion_datasource::sink::DataSinkExec; @@ -70,6 +70,8 @@ pub struct MemTable { /// Optional pre-known sort order(s). Must be `SortExpr`s. /// inserting data into this table removes the order pub sort_order: Arc>>>, + /// The version of the table this provider reads, if versioned + version: Option, } impl MemTable { @@ -103,9 +105,17 @@ impl MemTable { constraints: Constraints::default(), column_defaults: HashMap::new(), sort_order: Arc::new(Mutex::new(vec![])), + version: None, }) } + /// Set the version of the table this provider reads, for time travel + /// queries. See [`TableProvider::version`]. + pub fn with_version(mut self, version: Option) -> Self { + self.version = version; + self + } + /// Assign constraints pub fn with_constraints(mut self, constraints: Constraints) -> Self { self.constraints = constraints; @@ -178,6 +188,10 @@ impl TableProvider for MemTable { Some(&self.constraints) } + fn version(&self) -> Option<&TableVersion> { + self.version.as_ref() + } + fn table_type(&self) -> TableType { TableType::Base } diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs index 5c84943c4f48e..fa56dc3f4d098 100644 --- a/datafusion/catalog/src/view.rs +++ b/datafusion/catalog/src/view.rs @@ -11,8 +11,8 @@ use crate::TableProvider; use arrow::datatypes::SchemaRef; use async_trait::async_trait; -use datafusion_common::Column; use datafusion_common::error::Result; +use datafusion_common::{Column, TableVersion}; use datafusion_expr::TableType; use datafusion_expr::{Expr, LogicalPlan}; use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown}; @@ -28,6 +28,8 @@ pub struct ViewTable { table_schema: SchemaRef, /// SQL used to create the view, if available definition: Option, + /// The version of the table this provider reads, if versioned + version: Option, } impl ViewTable { @@ -44,9 +46,17 @@ impl ViewTable { logical_plan, table_schema, definition, + version: None, } } + /// Set the version of the table this provider reads, for time travel + /// queries. See [`TableProvider::version`]. + pub fn with_version(mut self, version: Option) -> Self { + self.version = version; + self + } + /// Get definition ref pub fn definition(&self) -> Option<&String> { self.definition.as_ref() @@ -72,6 +82,10 @@ impl TableProvider for ViewTable { TableType::View } + fn version(&self) -> Option<&TableVersion> { + self.version.as_ref() + } + fn get_table_definition(&self) -> Option<&str> { self.definition.as_deref() } diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index b37ee729c9e63..27d40fb47f0ab 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -20,6 +20,7 @@ mod param_value; mod partitioning; mod schema_reference; mod table_reference; +mod table_version; mod unnest; pub mod alias; @@ -90,6 +91,7 @@ pub use schema_reference::SchemaReference; pub use spans::{Location, Span, Spans}; pub use stats::{ColumnStatistics, Statistics}; pub use table_reference::{ResolvedTableReference, TableReference}; +pub use table_version::TableVersion; pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; diff --git a/datafusion/common/src/table_version.rs b/datafusion/common/src/table_version.rs new file mode 100644 index 0000000000000..9d1cc9e7ff8b4 --- /dev/null +++ b/datafusion/common/src/table_version.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::fmt::{Display, Formatter}; +use std::sync::Arc; + +use arrow::temporal_conversions::timestamp_ns_to_datetime; +use chrono::SecondsFormat; + +use crate::ScalarValue; + +/// A specific version of a table, as requested by a time travel query. +/// +/// Time travel queries read a table as it existed at some point in the past. +/// SQL dialects use a variety of syntaxes for this, all of which are +/// normalized to one of the two variants below. For example: +/// +/// | SQL | `TableVersion` | +/// |----------------------------------------------|----------------| +/// | `t VERSION AS OF 3` | `Version` | +/// | `t AT (VERSION => 'v1')` | `Version` | +/// | `t TIMESTAMP AS OF '2024-01-01'` | `Timestamp` | +/// | `t FOR SYSTEM_TIME AS OF now()` | `Timestamp` | +/// | `t AT (TIMESTAMP => now() - INTERVAL '1 h')` | `Timestamp` | +/// +/// The version expression is evaluated during planning, so catalogs only +/// ever see the resulting constant value. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] +pub enum TableVersion { + /// A catalog specific version identifier, such as a snapshot id, a + /// version number, a tag or a branch name. + Version(ScalarValue), + /// A point in time, in nanoseconds since the Unix epoch (UTC). + Timestamp(i64), +} + +impl TableVersion { + /// Returns the value of this version as a [`ScalarValue`]. + /// + /// For [`TableVersion::Timestamp`] this is a + /// [`ScalarValue::TimestampNanosecond`] in the UTC time zone. + pub fn value(&self) -> ScalarValue { + match self { + TableVersion::Version(v) => v.clone(), + TableVersion::Timestamp(ns) => { + ScalarValue::TimestampNanosecond(Some(*ns), Some(Arc::from("UTC"))) + } + } + } +} + +impl Display for TableVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + // Strings are quoted, to tell them apart from numbers and + // timestamps + TableVersion::Version(v) => match v.try_as_str() { + Some(Some(s)) => write!(f, "{s:?}"), + _ => write!(f, "{v}"), + }, + TableVersion::Timestamp(ns) => match timestamp_ns_to_datetime(*ns) { + Some(dt) => write!( + f, + "{}", + dt.and_utc().to_rfc3339_opts(SecondsFormat::AutoSi, true) + ), + None => write!(f, "{ns}"), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + assert_eq!( + TableVersion::Version(ScalarValue::Int64(Some(3))).to_string(), + "3" + ); + assert_eq!( + TableVersion::Version(ScalarValue::from("v1")).to_string(), + "\"v1\"" + ); + assert_eq!( + TableVersion::Timestamp(1_704_067_200_000_000_000).to_string(), + "2024-01-01T00:00:00Z" + ); + assert_eq!( + TableVersion::Version(ScalarValue::from("a \"quoted\" tag")).to_string(), + r#""a \"quoted\" tag""# + ); + assert_eq!( + TableVersion::Version(ScalarValue::Utf8View(Some("3".into()))).to_string(), + "\"3\"" + ); + } +} diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index aeaf80984f8e3..bcaf99458a2a9 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -19,9 +19,7 @@ use crate::datasource::provider_as_source; use crate::execution::SessionStateDefaults; use crate::execution::context::{EmptySerializerRegistry, FunctionFactory, QueryPlanner}; use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; -#[cfg(feature = "sql")] -use arrow_schema::DataType; -use arrow_schema::FieldRef; +use arrow_schema::{DataType, FieldRef, TimeUnit}; use datafusion_catalog::MemoryCatalogProviderList; use datafusion_catalog::information_schema::{ INFORMATION_SCHEMA, InformationSchemaProvider, @@ -34,8 +32,8 @@ use datafusion_common::config::{ConfigExtension, ConfigOptions, TableOptions}; use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan}; use datafusion_common::tree_node::TreeNode; use datafusion_common::{ - DFSchema, DataFusionError, ResolvedTableReference, TableReference, config_err, - exec_err, plan_datafusion_err, plan_err, + DFSchema, DataFusionError, ResolvedTableReference, ScalarValue, TableReference, + TableVersion, config_err, exec_err, internal_err, plan_datafusion_err, plan_err, }; use datafusion_execution::TaskContext; use datafusion_execution::config::SessionConfig; @@ -45,7 +43,7 @@ use datafusion_expr::TableSource; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr_rewriter::FunctionRewrite; use datafusion_expr::physical_planning_context::PhysicalPlanningContext; -use datafusion_expr::planner::ExprPlanner; +use datafusion_expr::planner::{ExprPlanner, TableVersionExpr}; #[cfg(feature = "sql")] use datafusion_expr::planner::{RelationPlanner, TypePlanner}; use datafusion_expr::registry::{ @@ -54,7 +52,7 @@ use datafusion_expr::registry::{ }; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::{ - AggregateUDF, Explain, Expr, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, + AggregateUDF, Explain, Expr, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, cast, }; use datafusion_optimizer::simplify_expressions::ExprSimplifier; use datafusion_optimizer::{ @@ -640,14 +638,29 @@ impl SessionState { &self, statement: Statement, ) -> datafusion_common::Result { - let references = self.resolve_table_references(&statement)?; + let references = datafusion_sql::resolve::resolve_table_references_with_versions( + &statement, + self.config() + .options() + .sql_parser + .enable_ident_normalization, + )?; let mut provider = SessionContextProvider { state: self, - tables: HashMap::with_capacity(references.len()), + tables: HashMap::with_capacity(references.tables.len()), + versioned_tables: HashMap::with_capacity(references.versioned_tables.len()), }; - for reference in references { + // A `RelationPlanner` may handle a time travel clause itself, starting + // from the current version of the table, so also fetch that version. + // Without relation planners, the current version would go unused. + let mut tables = references.tables; + if !self.relation_planners().is_empty() { + tables.extend(references.versioned_tables.iter().map(|(t, _)| t.clone())); + } + + for reference in tables { let resolved = self.resolve_table_ref(reference); if let Entry::Vacant(v) = provider.tables.entry(resolved) { let resolved = v.key(); @@ -659,10 +672,101 @@ impl SessionState { } } + let versions = + self.evaluate_table_versions(&provider, references.versioned_tables); + for key in versions { + if let Entry::Vacant(v) = provider.versioned_tables.entry(key) { + let (resolved, version) = v.key(); + if let Ok(schema) = self.schema_for_ref(resolved.clone()) { + match schema.table_at(&resolved.table, version).await { + Ok(Some(table)) => { + v.insert(Ok(provider_as_source(table))); + } + Ok(None) => {} + Err(e) => { + v.insert(Err(Arc::new(e))); + } + } + } + } + } + let query = SqlToRel::new_with_options(&provider, self.get_parser_options()); query.statement_to_plan(statement) } + /// Evaluates the time travel clauses of `versioned_tables` ahead of + /// planning, so the referenced table versions can be fetched from the + /// catalog. + /// + /// Clauses that fail to evaluate are skipped: a [`RelationPlanner`] may + /// handle them, and otherwise the SQL planner reports the same error. + #[cfg(feature = "sql")] + fn evaluate_table_versions( + &self, + provider: &SessionContextProvider<'_>, + versioned_tables: Vec<(TableReference, sqlparser::ast::TableVersion)>, + ) -> Vec<(ResolvedTableReference, TableVersion)> { + let query = SqlToRel::new_with_options(provider, self.get_parser_options()); + versioned_tables + .into_iter() + .filter_map(|(reference, version)| { + let version = query + .sql_table_version_to_expr(version, &mut PlannerContext::new()) + .and_then(|version| self.evaluate_table_version(version)) + .ok()?; + Some((self.resolve_table_ref(reference), version)) + }) + .collect() + } + + /// Evaluates the expression of a time travel clause, such as + /// `VERSION AS OF `, to a constant [`TableVersion`]. + /// + /// The expression must not reference any columns. Functions such as + /// `now()` are evaluated using this session's query execution start time. + /// Timestamps are converted to nanoseconds in the UTC time zone. + pub fn evaluate_table_version( + &self, + version: TableVersionExpr, + ) -> datafusion_common::Result { + let simplify_context = SimplifyContext::builder() + .with_config_options(Arc::clone(self.config_options())) + .with_query_execution_start_time( + self.execution_props().query_execution_start_time, + ) + .build(); + let simplifier = ExprSimplifier::new(simplify_context); + let (expr, is_timestamp) = match version { + TableVersionExpr::Version(expr) => (expr, false), + TableVersionExpr::Timestamp(expr) => ( + cast( + expr, + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ), + true, + ), + }; + let expr = simplifier.coerce(expr, &DFSchema::empty())?; + let value = match simplifier.simplify(expr)? { + Expr::Literal(value, _) if !value.is_null() => value, + expr => { + return plan_err!( + "Time travel version must evaluate to a non-null constant, got: {expr}" + ); + } + }; + match value { + ScalarValue::TimestampNanosecond(Some(ns), _) if is_timestamp => { + Ok(TableVersion::Timestamp(ns)) + } + value if !is_timestamp => Ok(TableVersion::Version(value)), + value => { + internal_err!("Expected a timestamp time travel version, got: {value:?}") + } + } + } + #[cfg(feature = "sql")] fn get_parser_options(&self) -> ParserOptions { let sql_parser_options = &self.config().options().sql_parser; @@ -731,6 +835,7 @@ impl SessionState { let provider = SessionContextProvider { state: self, tables: HashMap::new(), + versioned_tables: HashMap::new(), }; let query = SqlToRel::new_with_options(&provider, self.get_parser_options()); @@ -2117,8 +2222,17 @@ impl From for SessionStateBuilder { struct SessionContextProvider<'a> { state: &'a SessionState, tables: HashMap>, + /// Tables referenced with a time travel clause. Lookup errors are kept + /// and only reported if the planner actually needs the table, as a + /// [`RelationPlanner`] may handle the time travel clause itself. + versioned_tables: + HashMap<(ResolvedTableReference, TableVersion), VersionedTableSource>, } +/// The result of looking up a specific version of a table +#[cfg(feature = "sql")] +type VersionedTableSource = Result, Arc>; + #[cfg(feature = "sql")] impl ContextProvider for SessionContextProvider<'_> { fn get_expr_planners(&self) -> &[Arc] { @@ -2148,6 +2262,26 @@ impl ContextProvider for SessionContextProvider<'_> { .ok_or_else(|| plan_datafusion_err!("table '{name}' not found")) } + fn evaluate_table_version( + &self, + version: TableVersionExpr, + ) -> datafusion_common::Result { + self.state.evaluate_table_version(version) + } + + fn get_table_source_at( + &self, + name: TableReference, + version: &TableVersion, + ) -> datafusion_common::Result> { + let name = self.state.resolve_table_ref(name); + match self.versioned_tables.get(&(name.clone(), version.clone())) { + Some(Ok(table)) => Ok(Arc::clone(table)), + Some(Err(e)) => Err(DataFusionError::Shared(Arc::clone(e))), + None => plan_err!("table '{name}@{version}' not found"), + } + } + fn get_table_function_source( &self, name: &str, @@ -2613,6 +2747,7 @@ mod tests { let provider = SessionContextProvider { state, tables: HashMap::new(), + versioned_tables: HashMap::new(), }; let sql = "[1,2,3]"; @@ -2643,6 +2778,7 @@ mod tests { let provider = SessionContextProvider { state: &state, tables: HashMap::new(), + versioned_tables: HashMap::new(), }; let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); diff --git a/datafusion/core/tests/user_defined/relation_planner.rs b/datafusion/core/tests/user_defined/relation_planner.rs index 5bdb251bc3f1e..fee216192c5e2 100644 --- a/datafusion/core/tests/user_defined/relation_planner.rs +++ b/datafusion/core/tests/user_defined/relation_planner.rs @@ -5,19 +5,24 @@ //! Tests for the RelationPlanner extension point use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arrow::array::{Int64Array, RecordBatch, StringArray}; use arrow::datatypes::{DataType, Field, Schema}; +use async_trait::async_trait; use datafusion::catalog::memory::MemTable; +use datafusion::catalog::{SchemaProvider, TableProvider}; use datafusion::common::test_util::batches_to_string; use datafusion::prelude::*; -use datafusion_common::{Result, ScalarValue}; +use datafusion_common::{DFSchema, Result, ScalarValue, TableVersion, not_impl_err}; use datafusion_expr::Expr; use datafusion_expr::logical_plan::builder::LogicalPlanBuilder; use datafusion_expr::planner::{ PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning, }; -use datafusion_sql::sqlparser::ast::{TableAlias, TableFactor}; +use datafusion_sql::sqlparser::ast::{ + TableAlias, TableFactor, TableVersion as SQLTableVersion, +}; use insta::assert_snapshot; // ============================================================================ @@ -337,6 +342,51 @@ fn create_context_with_catalog_table( ctx } +/// A schema with a single table `t`, which counts how often the current and +/// a specific version of `t` are looked up. +#[derive(Debug)] +struct CountingSchema { + table: Arc, + /// Whether `table_at` is implemented, or returns "not implemented" like + /// catalogs that predate it + supports_time_travel: bool, + table_calls: AtomicUsize, + table_at_calls: AtomicUsize, +} + +#[async_trait] +impl SchemaProvider for CountingSchema { + fn table_names(&self) -> Vec { + vec!["t".to_string()] + } + + async fn table(&self, name: &str) -> Result>> { + self.table_calls.fetch_add(1, Ordering::SeqCst); + Ok((name == "t").then(|| Arc::clone(&self.table))) + } + + async fn table_at( + &self, + name: &str, + version: &TableVersion, + ) -> Result>> { + self.table_at_calls.fetch_add(1, Ordering::SeqCst); + if !self.supports_time_travel { + return not_impl_err!("Time travel is not supported"); + } + if name != "t" { + return Ok(None); + } + let table = MemTable::try_new(self.table.schema(), vec![vec![]])? + .with_version(Some(version.clone())); + Ok(Some(Arc::new(table))) + } + + fn table_exist(&self, name: &str) -> bool { + name == "t" + } +} + /// Create a SessionContext with a simple single-column Int64 table. /// /// Useful for basic tests that need a real catalog table. @@ -366,6 +416,44 @@ fn create_context_with_simple_table( // TESTS: Ordered from Basic to Complex // ============================================================================ +/// Example planner that implements time travel itself, for tables whose +/// catalog does not support [`SchemaProvider::table_at`]. It emulates version +/// `n` of a table by only returning the rows with an `id` of at most `n`. +/// +/// [`SchemaProvider::table_at`]: datafusion::catalog::SchemaProvider::table_at +#[derive(Debug)] +struct EmulatedVersionPlanner; + +impl RelationPlanner for EmulatedVersionPlanner { + fn plan_relation( + &self, + relation: TableFactor, + context: &mut dyn RelationPlannerContext, + ) -> Result { + let TableFactor::Table { + name, + alias, + args: None, + version: Some(SQLTableVersion::VersionAsOf(version)), + .. + } = relation + else { + return Ok(RelationPlanning::Original(Box::new(relation))); + }; + let table_ref = context.object_name_to_table_reference(name)?; + let version = context.sql_to_expr(version, &DFSchema::empty())?; + let source = context + .context_provider() + .get_table_source(table_ref.clone())?; + let plan = LogicalPlanBuilder::scan(table_ref, source, None)? + .filter(col("id").lt_eq(version))? + .build()?; + Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new( + plan, alias, + )))) + } +} + /// Comprehensive test suite for RelationPlanner extension point. /// Tests are ordered from simplest smoke test to most complex scenarios. #[cfg(test)] @@ -704,6 +792,91 @@ mod tests { ); } + // A RelationPlanner can handle time travel clauses itself, for tables + // whose catalog does not support time travel. + #[tokio::test] + async fn planner_handles_time_travel() { + let ctx = + create_context_with_catalog_table("t", vec![1, 2, 3], vec!["a", "b", "c"]) + .with_planner(EmulatedVersionPlanner); + ctx.sql("SET datafusion.sql_parser.dialect = 'Databricks'") + .await + .unwrap(); + + let result = + execute_sql_to_string(&ctx, "SELECT * FROM t VERSION AS OF 2 ORDER BY id") + .await; + assert_snapshot!(result, @r" + +----+------+ + | id | name | + +----+------+ + | 1 | a | + | 2 | b | + +----+------+ + "); + + // Without the planner, the catalog's lack of time travel support is + // reported + let ctx = + create_context_with_catalog_table("t", vec![1, 2, 3], vec!["a", "b", "c"]); + ctx.sql("SET datafusion.sql_parser.dialect = 'Databricks'") + .await + .unwrap(); + let error = execute_sql(&ctx, "SELECT * FROM t VERSION AS OF 2") + .await + .expect_err("time travel should not be supported"); + assert_snapshot!(error.strip_backtrace(), @"This feature is not implemented: Time travel is not supported for table 't@2'"); + } + + // The current version of a table referenced with a time travel clause is + // only fetched when a RelationPlanner may handle the time travel clause + // itself, whether or not the catalog implements `table_at`. + #[tokio::test] + async fn time_travel_fetches_current_version_only_for_planners() { + for (supports_time_travel, with_planner) in + [(true, false), (true, true), (false, false), (false, true)] + { + let case = format!( + "supports_time_travel={supports_time_travel}, with_planner={with_planner}" + ); + let mut ctx = + create_context_with_catalog_table("unused", vec![1, 2], vec!["a", "b"]); + if with_planner { + ctx = ctx.with_planner(EmulatedVersionPlanner); + } + ctx.sql("SET datafusion.sql_parser.dialect = 'Databricks'") + .await + .unwrap(); + let table = ctx.table_provider("unused").await.unwrap(); + let schema = Arc::new(CountingSchema { + table, + supports_time_travel, + table_calls: AtomicUsize::new(0), + table_at_calls: AtomicUsize::new(0), + }); + ctx.catalog("datafusion") + .unwrap() + .register_schema("counting", Arc::clone(&schema) as _) + .unwrap(); + + let result = + execute_sql(&ctx, "SELECT * FROM counting.t VERSION AS OF 1").await; + // Planned by the planner if registered, and by DataFusion + // otherwise + assert_eq!( + result.is_ok(), + supports_time_travel || with_planner, + "{case}: {result:?}" + ); + assert_eq!( + schema.table_calls.load(Ordering::SeqCst), + usize::from(with_planner), + "{case}" + ); + assert_eq!(schema.table_at_calls.load(Ordering::SeqCst), 1, "{case}"); + } + } + // SamplingJoinPlanner recursively calls `context.plan()` on both sides of a // nested join before sampling, exercising recursive relation planning. #[tokio::test] diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs index bc3ee916123af..79cf0ba0ea4ec 100644 --- a/datafusion/expr/src/logical_plan/display.rs +++ b/datafusion/expr/src/logical_plan/display.rs @@ -387,6 +387,11 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> { "Relation Name": table_name.table(), ); + if let Some(version) = source.version() { + object + .push("Version", serde_json::Value::String(version.to_string())); + } + if let Some(s) = table_name.schema() { object.push("Schema", serde_json::Value::String(s.to_string())); } diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 6da0734ab0aeb..2d356891f0bcc 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -58,7 +58,7 @@ use datafusion_common::tree_node::{ use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, - ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions, + ScalarValue, Spans, SplitPoint, TableReference, TableVersion, UnnestOptions, aggregate_functional_dependencies_with_computed_inputs, assert_eq_or_internal_err, assert_or_internal_err, internal_err, plan_datafusion_err, plan_err, validate_range_split_points, @@ -2229,7 +2229,11 @@ impl LogicalPlan { _ => "".to_string(), }; - write!(f, "TableScan: {table_name}{projected_fields}")?; + write!(f, "TableScan: {table_name}")?; + if let Some(version) = source.version() { + write!(f, "@{version}")?; + } + write!(f, "{projected_fields}")?; if !filters.is_empty() { let mut full_filter = vec![]; @@ -3291,6 +3295,7 @@ impl Debug for TableScan { fn fmt(&self, f: &mut Formatter) -> fmt::Result { f.debug_struct("TableScan") .field("table_name", &self.table_name) + .field("version", &self.source.version()) .field("source", &"...") .field("projection", &self.projection) .field("projected_schema", &self.projected_schema) @@ -3308,13 +3313,14 @@ impl PartialEq for TableScan { && self.projected_schema == other.projected_schema && self.filters == other.filters && self.fetch == other.fetch + && self.source.version() == other.source.version() } } impl Eq for TableScan {} // Manual implementation needed because of `source` and `projected_schema` fields. -// Comparison excludes these field. +// Comparison excludes these fields, except for the version of the `source`. impl PartialOrd for TableScan { fn partial_cmp(&self, other: &Self) -> Option { #[derive(PartialEq, PartialOrd)] @@ -3327,18 +3333,22 @@ impl PartialOrd for TableScan { pub filters: &'a Vec, /// Optional number of rows to read pub fetch: &'a Option, + /// The version of the table to read + pub version: Option<&'a TableVersion>, } let comparable_self = ComparableTableScan { table_name: &self.table_name, projection: &self.projection, filters: &self.filters, fetch: &self.fetch, + version: self.source.version(), }; let comparable_other = ComparableTableScan { table_name: &other.table_name, projection: &other.projection, filters: &other.filters, fetch: &other.fetch, + version: other.source.version(), }; comparable_self .partial_cmp(&comparable_other) @@ -3354,6 +3364,7 @@ impl Hash for TableScan { self.projected_schema.hash(state); self.filters.hash(state); self.fetch.hash(state); + self.source.version().hash(state); } } diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rs index 33426920787b5..fb793ccdcafd1 100644 --- a/datafusion/expr/src/planner.rs +++ b/datafusion/expr/src/planner.rs @@ -17,7 +17,7 @@ use crate::{ use arrow::datatypes::{DataType, Field, FieldRef, SchemaRef}; use datafusion_common::datatype::DataTypeExt; use datafusion_common::{ - DFSchema, Result, TableReference, config::ConfigOptions, + DFSchema, Result, TableReference, TableVersion, config::ConfigOptions, file_options::file_type::FileType, not_impl_err, }; #[cfg(feature = "sql")] @@ -32,6 +32,29 @@ pub trait ContextProvider { /// Returns a table by reference, if it exists fn get_table_source(&self, name: TableReference) -> Result>; + /// Evaluates the expression of a time travel clause, such as + /// `VERSION AS OF `, to a constant [`TableVersion`]. + /// + /// The expression must not reference any columns, but may call functions + /// such as `now()`. + fn evaluate_table_version(&self, _version: TableVersionExpr) -> Result { + not_impl_err!("Time travel queries are not supported") + } + + /// Returns a specific version of a table by reference, if it exists. + /// + /// The returned source must report the version it reads from + /// [`TableSource::version`], otherwise planning fails. This may differ + /// from the requested `version`. See [`TableVersion`] for the time travel + /// syntaxes that resolve to this. + fn get_table_source_at( + &self, + name: TableReference, + version: &TableVersion, + ) -> Result> { + not_impl_err!("Time travel is not supported for table '{name}@{version}'") + } + /// Return the type of a file based on its extension (e.g. `.parquet`) /// /// This is used to plan `COPY` statements @@ -135,6 +158,18 @@ pub trait ContextProvider { fn udwf_names(&self) -> Vec; } +/// The unevaluated form of a [`TableVersion`], as written in a time travel +/// clause such as `VERSION AS OF ` or `TIMESTAMP AS OF `. +/// +/// See [`ContextProvider::evaluate_table_version`]. +#[derive(Debug, Clone, PartialEq)] +pub enum TableVersionExpr { + /// Evaluates to a [`TableVersion::Version`] + Version(Expr), + /// Evaluates to a [`TableVersion::Timestamp`] + Timestamp(Expr), +} + /// Customize planning of SQL AST expressions to [`Expr`]s /// /// For more background, please also see the [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog] diff --git a/datafusion/expr/src/table_source.rs b/datafusion/expr/src/table_source.rs index 2d05121e11143..f565acaa7924f 100644 --- a/datafusion/expr/src/table_source.rs +++ b/datafusion/expr/src/table_source.rs @@ -7,7 +7,7 @@ use crate::{Expr, LogicalPlan}; use arrow::datatypes::SchemaRef; -use datafusion_common::{Constraints, Result}; +use datafusion_common::{Constraints, Result, TableVersion}; use std::{any::Any, borrow::Cow}; @@ -121,6 +121,20 @@ pub trait TableSource: Any + Sync + Send { fn get_column_default(&self, _column: &str) -> Option<&Expr> { None } + + /// The version of the table this source reads, for time travel queries + /// such as `SELECT * FROM t VERSION AS OF 3`. `None` means the version is + /// unknown or the table is not versioned. + /// + /// Sources returned by [`ContextProvider::get_table_source_at`] must + /// report the version they read, which may differ from the requested + /// version. See [`TableProvider::version`]. + /// + /// [`TableProvider::version`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.version + /// [`ContextProvider::get_table_source_at`]: crate::planner::ContextProvider::get_table_source_at + fn version(&self) -> Option<&TableVersion> { + None + } } impl dyn TableSource { diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 64e4be982ed59..e87f2a5e9650d 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -93,6 +93,8 @@ message ListingTableScanNode { optional uint64 fetch = 17; // Optional number of rows to skip. optional uint64 skip = 18; + // The version of the table to read, for time travel queries. + TableVersion version = 19; } message ViewTableScanNode { @@ -106,6 +108,8 @@ message ViewTableScanNode { optional uint64 fetch = 7; // Optional number of rows to skip. optional uint64 skip = 8; + // The version of the table to read, for time travel queries. + TableVersion version = 9; } // Logical Plan to Scan a CustomTableProvider registered at runtime @@ -120,6 +124,8 @@ message CustomTableScanNode { optional uint64 fetch = 7; // Optional number of rows to skip. optional uint64 skip = 8; + // The version of the table to read, for time travel queries. + TableVersion version = 9; } message ProjectionNode { @@ -883,6 +889,16 @@ message TableReference { } } +// A version of a table, for time travel queries +message TableVersion { + oneof table_version_enum { + // A catalog specific version identifier, such as a snapshot id or a tag + datafusion_common.ScalarValue version = 1; + // A point in time, in nanoseconds since the Unix epoch (UTC) + int64 timestamp = 2; + } +} + ///////////////////////////////////////////////////////////////////////////////////////////////// // PhysicalPlanNode is a nested type @@ -1772,6 +1788,8 @@ message EmptyTableScanNode { datafusion_common.Schema schema = 2; ProjectionColumns projection = 3; repeated LogicalExprNode filters = 4; + // The version of the table to read, for time travel queries. + TableVersion version = 5; } enum GenerateSeriesName { diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs index 2efba9779bc8a..6f2d8dc752818 100644 --- a/datafusion/proto-models/src/from_proto.rs +++ b/datafusion/proto-models/src/from_proto.rs @@ -22,7 +22,7 @@ use datafusion_common::parsers::{CompressionTypeVariant, CsvQuoteStyle}; use datafusion_common::utils::usize_from_wire; use datafusion_common::{ JoinConstraint, JoinType, NullEquality, RecursionUnnestOption, TableReference, - UnnestOptions, + TableVersion, UnnestOptions, }; use datafusion_proto_common::FromProtoError as Error; @@ -99,6 +99,24 @@ impl TryFrom for TableReference { } } +impl TryFrom<&protobuf::TableVersion> for TableVersion { + type Error = Error; + + fn try_from(value: &protobuf::TableVersion) -> Result { + use protobuf::table_version::TableVersionEnum; + match value + .table_version_enum + .as_ref() + .ok_or_else(|| Error::required("table_version_enum"))? + { + TableVersionEnum::Version(value) => { + Ok(TableVersion::Version(value.try_into()?)) + } + TableVersionEnum::Timestamp(ns) => Ok(TableVersion::Timestamp(*ns)), + } + } +} + impl From<&protobuf::StringifiedPlan> for StringifiedPlan { fn from(stringified_plan: &protobuf::StringifiedPlan) -> Self { Self { diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index f7814a2780ea1..4516ebd02598c 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -5728,6 +5728,9 @@ impl serde::Serialize for CustomTableScanNode { if self.skip.is_some() { len += 1; } + if self.version.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.CustomTableScanNode", len)?; if let Some(v) = self.table_name.as_ref() { struct_ser.serialize_field("tableName", v)?; @@ -5756,6 +5759,9 @@ impl serde::Serialize for CustomTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } + if let Some(v) = self.version.as_ref() { + struct_ser.serialize_field("version", v)?; + } struct_ser.end() } } @@ -5775,6 +5781,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { "customTableData", "fetch", "skip", + "version", ]; #[allow(clippy::enum_variant_names)] @@ -5786,6 +5793,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { CustomTableData, Fetch, Skip, + Version, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -5814,6 +5822,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { "customTableData" | "custom_table_data" => Ok(GeneratedField::CustomTableData), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), + "version" => Ok(GeneratedField::Version), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -5840,6 +5849,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { let mut custom_table_data__ = None; let mut fetch__ = None; let mut skip__ = None; + let mut version__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -5890,6 +5900,12 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } + GeneratedField::Version => { + if version__.is_some() { + return Err(serde::de::Error::duplicate_field("version")); + } + version__ = map_.next_value()?; + } } } Ok(CustomTableScanNode { @@ -5900,6 +5916,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { custom_table_data: custom_table_data__.unwrap_or_default(), fetch: fetch__, skip: skip__, + version: version__, }) } } @@ -6831,6 +6848,9 @@ impl serde::Serialize for EmptyTableScanNode { if !self.filters.is_empty() { len += 1; } + if self.version.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?; if let Some(v) = self.table_name.as_ref() { struct_ser.serialize_field("tableName", v)?; @@ -6844,6 +6864,9 @@ impl serde::Serialize for EmptyTableScanNode { if !self.filters.is_empty() { struct_ser.serialize_field("filters", &self.filters)?; } + if let Some(v) = self.version.as_ref() { + struct_ser.serialize_field("version", v)?; + } struct_ser.end() } } @@ -6859,6 +6882,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { "schema", "projection", "filters", + "version", ]; #[allow(clippy::enum_variant_names)] @@ -6867,6 +6891,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { Schema, Projection, Filters, + Version, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6892,6 +6917,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { "schema" => Ok(GeneratedField::Schema), "projection" => Ok(GeneratedField::Projection), "filters" => Ok(GeneratedField::Filters), + "version" => Ok(GeneratedField::Version), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6915,6 +6941,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { let mut schema__ = None; let mut projection__ = None; let mut filters__ = None; + let mut version__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -6941,6 +6968,12 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { } filters__ = Some(map_.next_value()?); } + GeneratedField::Version => { + if version__.is_some() { + return Err(serde::de::Error::duplicate_field("version")); + } + version__ = map_.next_value()?; + } } } Ok(EmptyTableScanNode { @@ -6948,6 +6981,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { schema: schema__, projection: projection__, filters: filters__.unwrap_or_default(), + version: version__, }) } } @@ -13228,6 +13262,9 @@ impl serde::Serialize for ListingTableScanNode { if self.skip.is_some() { len += 1; } + if self.version.is_some() { + len += 1; + } if self.file_format_type.is_some() { len += 1; } @@ -13266,6 +13303,9 @@ impl serde::Serialize for ListingTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } + if let Some(v) = self.version.as_ref() { + struct_ser.serialize_field("version", v)?; + } if let Some(v) = self.file_format_type.as_ref() { match v { listing_table_scan_node::FileFormatType::Csv(v) => { @@ -13309,6 +13349,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "fileSortOrder", "fetch", "skip", + "version", "csv", "parquet", "avro", @@ -13328,6 +13369,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { FileSortOrder, Fetch, Skip, + Version, Csv, Parquet, Avro, @@ -13364,6 +13406,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "fileSortOrder" | "file_sort_order" => Ok(GeneratedField::FileSortOrder), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), + "version" => Ok(GeneratedField::Version), "csv" => Ok(GeneratedField::Csv), "parquet" => Ok(GeneratedField::Parquet), "avro" => Ok(GeneratedField::Avro), @@ -13398,6 +13441,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { let mut file_sort_order__ = None; let mut fetch__ = None; let mut skip__ = None; + let mut version__ = None; let mut file_format_type__ = None; while let Some(k) = map_.next_key()? { match k { @@ -13465,6 +13509,12 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } + GeneratedField::Version => { + if version__.is_some() { + return Err(serde::de::Error::duplicate_field("version")); + } + version__ = map_.next_value()?; + } GeneratedField::Csv => { if file_format_type__.is_some() { return Err(serde::de::Error::duplicate_field("csv")); @@ -13513,6 +13563,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { file_sort_order: file_sort_order__.unwrap_or_default(), fetch: fetch__, skip: skip__, + version: version__, file_format_type: file_format_type__, }) } @@ -28941,6 +28992,116 @@ impl<'de> serde::Deserialize<'de> for TableReference { deserializer.deserialize_struct("datafusion.TableReference", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for TableVersion { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.table_version_enum.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("datafusion.TableVersion", len)?; + if let Some(v) = self.table_version_enum.as_ref() { + match v { + table_version::TableVersionEnum::Version(v) => { + struct_ser.serialize_field("version", v)?; + } + table_version::TableVersionEnum::Timestamp(v) => { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("timestamp", ToString::to_string(&v).as_str())?; + } + } + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for TableVersion { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "version", + "timestamp", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Version, + Timestamp, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "version" => Ok(GeneratedField::Version), + "timestamp" => Ok(GeneratedField::Timestamp), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = TableVersion; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct datafusion.TableVersion") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut table_version_enum__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Version => { + if table_version_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("version")); + } + table_version_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_version::TableVersionEnum::Version) +; + } + GeneratedField::Timestamp => { + if table_version_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("timestamp")); + } + table_version_enum__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| table_version::TableVersionEnum::Timestamp(x.0)); + } + } + } + Ok(TableVersion { + table_version_enum: table_version_enum__, + }) + } + } + deserializer.deserialize_struct("datafusion.TableVersion", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for TryCastNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -30162,6 +30323,9 @@ impl serde::Serialize for ViewTableScanNode { if self.skip.is_some() { len += 1; } + if self.version.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.ViewTableScanNode", len)?; if let Some(v) = self.table_name.as_ref() { struct_ser.serialize_field("tableName", v)?; @@ -30188,6 +30352,9 @@ impl serde::Serialize for ViewTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } + if let Some(v) = self.version.as_ref() { + struct_ser.serialize_field("version", v)?; + } struct_ser.end() } } @@ -30206,6 +30373,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { "definition", "fetch", "skip", + "version", ]; #[allow(clippy::enum_variant_names)] @@ -30217,6 +30385,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { Definition, Fetch, Skip, + Version, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -30245,6 +30414,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { "definition" => Ok(GeneratedField::Definition), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), + "version" => Ok(GeneratedField::Version), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -30271,6 +30441,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { let mut definition__ = None; let mut fetch__ = None; let mut skip__ = None; + let mut version__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -30319,6 +30490,12 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } + GeneratedField::Version => { + if version__.is_some() { + return Err(serde::de::Error::duplicate_field("version")); + } + version__ = map_.next_value()?; + } } } Ok(ViewTableScanNode { @@ -30329,6 +30506,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { definition: definition__.unwrap_or_default(), fetch: fetch__, skip: skip__, + version: version__, }) } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 66b28437ee614..20ceaa431b5be 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -129,6 +129,9 @@ pub struct ListingTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "18")] pub skip: ::core::option::Option, + /// The version of the table to read, for time travel queries. + #[prost(message, optional, tag = "19")] + pub version: ::core::option::Option, #[prost( oneof = "listing_table_scan_node::FileFormatType", tags = "10, 11, 12, 15, 16" @@ -171,6 +174,9 @@ pub struct ViewTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "8")] pub skip: ::core::option::Option, + /// The version of the table to read, for time travel queries. + #[prost(message, optional, tag = "9")] + pub version: ::core::option::Option, } /// Logical Plan to Scan a CustomTableProvider registered at runtime #[derive(Clone, PartialEq, ::prost::Message)] @@ -191,6 +197,9 @@ pub struct CustomTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "8")] pub skip: ::core::option::Option, + /// The version of the table to read, for time travel queries. + #[prost(message, optional, tag = "9")] + pub version: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProjectionNode { @@ -1392,6 +1401,24 @@ pub mod table_reference { Full(super::FullTableReference), } } +/// A version of a table, for time travel queries +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableVersion { + #[prost(oneof = "table_version::TableVersionEnum", tags = "1, 2")] + pub table_version_enum: ::core::option::Option, +} +/// Nested message and enum types in `TableVersion`. +pub mod table_version { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum TableVersionEnum { + /// A catalog specific version identifier, such as a snapshot id or a tag + #[prost(message, tag = "1")] + Version(super::super::datafusion_common::ScalarValue), + /// A point in time, in nanoseconds since the Unix epoch (UTC) + #[prost(int64, tag = "2")] + Timestamp(i64), + } +} /// PhysicalPlanNode is a nested type #[derive(Clone, PartialEq, ::prost::Message)] pub struct PhysicalPlanNode { @@ -2678,6 +2705,9 @@ pub struct EmptyTableScanNode { pub projection: ::core::option::Option, #[prost(message, repeated, tag = "4")] pub filters: ::prost::alloc::vec::Vec, + /// The version of the table to read, for time travel queries. + #[prost(message, optional, tag = "5")] + pub version: ::core::option::Option, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GenerateSeriesArgsContainsNull { diff --git a/datafusion/proto-models/src/to_proto.rs b/datafusion/proto-models/src/to_proto.rs index 0d5f7e2209caa..11caab75ff5af 100644 --- a/datafusion/proto-models/src/to_proto.rs +++ b/datafusion/proto-models/src/to_proto.rs @@ -11,7 +11,7 @@ use datafusion_common::DataFusionError; use datafusion_common::display::{PlanType, StringifiedPlan}; use datafusion_common::{ - JoinConstraint, JoinType, NullEquality, TableReference, UnnestOptions, + JoinConstraint, JoinType, NullEquality, TableReference, TableVersion, UnnestOptions, }; use crate::generated::datafusion_common::EmptyMessage; @@ -114,6 +114,21 @@ impl From<&StringifiedPlan> for protobuf::StringifiedPlan { } } +impl TryFrom<&TableVersion> for protobuf::TableVersion { + type Error = datafusion_proto_common::ToProtoError; + + fn try_from(version: &TableVersion) -> Result { + use protobuf::table_version::TableVersionEnum; + let table_version_enum = match version { + TableVersion::Version(value) => TableVersionEnum::Version(value.try_into()?), + TableVersion::Timestamp(ns) => TableVersionEnum::Timestamp(*ns), + }; + Ok(protobuf::TableVersion { + table_version_enum: Some(table_version_enum), + }) + } +} + impl From for protobuf::TableReference { fn from(t: TableReference) -> Self { use protobuf::table_reference::TableReferenceEnum; diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index bd4f100c283f1..a4fe178b69190 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -28,7 +28,7 @@ use datafusion_common::format::{ }; use datafusion_common::utils::{usize_from_wire, usize_to_wire}; use datafusion_common::{ - NullEquality, Result, TableReference, assert_or_internal_err, context, + NullEquality, Result, TableReference, TableVersion, assert_or_internal_err, context, internal_datafusion_err, internal_err, not_impl_err, plan_err, }; use datafusion_datasource::file_format::FileFormat; @@ -360,6 +360,12 @@ fn from_table_reference( Ok(TableReference::try_from(table_ref.clone())?) } +fn from_table_version( + version: Option<&protobuf::TableVersion>, +) -> Result> { + Ok(version.map(TableVersion::try_from).transpose()?) +} + /// Converts [LogicalPlan::TableScan] to [TableSource] /// method to be used to deserialize nodes /// serialized by [from_table_source] @@ -672,9 +678,11 @@ impl AsLogicalPlan for LogicalPlanNode { .with_listing_options(options) .with_schema(Arc::new(schema)); - let provider = ListingTable::try_new(config)?.with_cache( - ctx.runtime_env().cache_manager.get_file_statistic_cache(), - ); + let provider = ListingTable::try_new(config)? + .with_cache( + ctx.runtime_env().cache_manager.get_file_statistic_cache(), + ) + .with_version(from_table_version(scan.version.as_ref())?); let table_name = from_table_reference(scan.table_name.as_ref(), "ListingTableScan")?; @@ -725,6 +733,17 @@ impl AsLogicalPlan for LogicalPlanNode { schema, ctx, )?; + // The codec restores the version as part of the provider's + // state. Fail rather than silently reading another version. + let version = from_table_version(scan.version.as_ref())?; + if provider.version() != version.as_ref() { + return internal_err!( + "Decoded table provider for '{table_name}' reads version {:?}, \ + but {:?} was encoded", + provider.version(), + version + ); + } let table_scan = TableScanBuilder::new(table_name, provider_as_source(provider)) @@ -1223,7 +1242,8 @@ impl AsLogicalPlan for LogicalPlanNode { None }; - let provider = ViewTable::new(input, definition); + let provider = ViewTable::new(input, definition) + .with_version(from_table_version(scan.version.as_ref())?); let table_name = from_table_reference(scan.table_name.as_ref(), "ViewScan")?; @@ -1371,7 +1391,10 @@ impl AsLogicalPlan for LogicalPlanNode { let table_name = from_table_reference(scan.table_name.as_ref(), "EmptyTableScan")?; - let provider = Arc::new(EmptyTable::new(Arc::clone(&schema))); + let provider = Arc::new( + EmptyTable::new(Arc::clone(&schema)) + .with_version(from_table_version(scan.version.as_ref())?), + ); let table_scan = TableScanBuilder::new(table_name, provider_as_source(provider)) @@ -1434,6 +1457,10 @@ impl AsLogicalPlan for LogicalPlanNode { }) => { let provider = source_as_provider(source)?; let schema = provider.schema(); + let version = source + .version() + .map(protobuf::TableVersion::try_from) + .transpose()?; let projection = match projection { None => None, @@ -1565,6 +1592,7 @@ impl AsLogicalPlan for LogicalPlanNode { file_sort_order: exprs_vec, fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), + version, }, )), }) @@ -1590,6 +1618,7 @@ impl AsLogicalPlan for LogicalPlanNode { .unwrap_or_default(), fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), + version, }, ))), }) @@ -1620,6 +1649,7 @@ impl AsLogicalPlan for LogicalPlanNode { schema: Some(schema), projection, filters, + version, }, )), }) @@ -1639,6 +1669,7 @@ impl AsLogicalPlan for LogicalPlanNode { custom_table_data: bytes, fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), + version, }); let node = LogicalPlanNode { logical_plan_type: Some(scan), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index f1712c7e818d1..e8796658f10d1 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -64,8 +64,8 @@ use datafusion_common::format::{ use datafusion_common::scalar::ScalarStructBuilder; use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, - SplitPoint, TableReference, internal_datafusion_err, internal_err, not_impl_err, - plan_err, + SplitPoint, TableReference, TableVersion, internal_datafusion_err, internal_err, + not_impl_err, plan_err, }; use datafusion_execution::TaskContext; use datafusion_expr::dml::CopyTo; @@ -3842,6 +3842,40 @@ async fn roundtrip_empty_table_scan() -> Result<()> { Ok(()) } +#[tokio::test] +async fn roundtrip_table_scan_with_version() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + let ctx = SessionContext::new(); + + for version in [ + TableVersion::Version(ScalarValue::Int64(Some(3))), + TableVersion::Version(ScalarValue::from("v1")), + TableVersion::Timestamp(1_704_067_200_000_000_000), + ] { + let table = Arc::new( + EmptyTable::new(Arc::clone(&schema)).with_version(Some(version.clone())), + ); + let plan = LogicalPlanBuilder::scan( + "empty", + Arc::new(DefaultTableSource::new(table)), + None, + )? + .build()?; + let bytes = logical_plan_to_bytes(&plan)?; + let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?; + // `TableScan` equality includes the version of its source + assert_eq!(plan, restored); + let LogicalPlan::TableScan(scan) = restored else { + panic!("expected a TableScan"); + }; + assert_eq!(scan.source.version(), Some(&version)); + } + Ok(()) +} + #[tokio::test] async fn roundtrip_empty_table_scan_with_projection() -> Result<()> { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/session/src/schema.rs b/datafusion/session/src/schema.rs index da6989c0b9356..dd51dcbf30404 100644 --- a/datafusion/session/src/schema.rs +++ b/datafusion/session/src/schema.rs @@ -6,7 +6,7 @@ //! representing collections of named tables. use async_trait::async_trait; -use datafusion_common::{DataFusionError, exec_err}; +use datafusion_common::{DataFusionError, TableVersion, exec_err, not_impl_err}; use std::any::Any; use std::fmt::Debug; use std::sync::Arc; @@ -38,6 +38,30 @@ pub trait SchemaProvider: Any + Debug + Sync + Send { name: &str, ) -> Result>, DataFusionError>; + /// Retrieves a specific version of a table from the schema by name, if it + /// exists, otherwise returns `None`. + /// + /// This is used to plan time travel queries such as + /// `SELECT * FROM t VERSION AS OF 3` or + /// `SELECT * FROM t TIMESTAMP AS OF '2024-01-01'`. See [`TableVersion`] + /// for the syntaxes that are supported. + /// + /// The returned provider must report the version it reads from + /// [`TableProvider::version`], otherwise planning fails. This may differ + /// from the requested `version`, for example when a timestamp resolves to + /// the latest snapshot committed before it. + /// + /// The default implementation returns a "not implemented" error, as + /// returning the current version of the table would silently produce + /// wrong results. + async fn table_at( + &self, + name: &str, + version: &TableVersion, + ) -> Result>> { + not_impl_err!("Time travel is not supported for table '{name}@{version}'") + } + /// Retrieves the type of a specific table from the schema by name, if it exists, otherwise /// returns `None`. Implementations for which this operation is cheap but [Self::table] is /// expensive can override this to improve operations that only need the type, e.g. diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs index a8720bfaeb321..9f0b4eebfca4a 100644 --- a/datafusion/session/src/table.rs +++ b/datafusion/session/src/table.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use crate::session::Session; use arrow_schema::SchemaRef; use async_trait::async_trait; -use datafusion_common::{Constraints, Statistics, not_impl_err}; +use datafusion_common::{Constraints, Statistics, TableVersion, not_impl_err}; use datafusion_common::{DFSchemaRef, Result, internal_err}; use datafusion_expr::Expr; use datafusion_expr::statistics::StatisticsRequest; @@ -71,6 +71,24 @@ pub trait TableProvider: Any + Debug + Sync + Send { None } + /// The version of the table this provider reads, for time travel + /// queries such as `SELECT * FROM t VERSION AS OF 3`. + /// + /// Providers returned by [`SchemaProvider::table_at`] must report the + /// version they read. This may differ from the requested version: for + /// example, `TIMESTAMP AS OF '2024-01-15'` may resolve to the snapshot + /// with id `3`, committed before that time. Reporting the resolved + /// version shows it in `EXPLAIN`, and lets scans of the same snapshot + /// compare equal regardless of how it was requested. + /// + /// Defaults to `None`, meaning the version is unknown or the table is not + /// versioned. + /// + /// [`SchemaProvider::table_at`]: crate::SchemaProvider::table_at + fn version(&self) -> Option<&TableVersion> { + None + } + /// Create an [`ExecutionPlan`] for scanning the table with optional /// `projection`, `filter`, and `limit`, described below. /// diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rs index 73a4614741881..f02cb6e82c07c 100644 --- a/datafusion/sql/src/relation/mod.rs +++ b/datafusion/sql/src/relation/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use std::ops::ControlFlow; use std::sync::Arc; use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; @@ -12,11 +13,14 @@ use datafusion_common::{ }; use datafusion_expr::builder::subquery_alias; use datafusion_expr::planner::{ - PlannedRelation, RelationPlannerContext, RelationPlanning, + PlannedRelation, RelationPlannerContext, RelationPlanning, TableVersionExpr, }; -use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, expr::Unnest}; +use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, Volatility, expr::Unnest}; use datafusion_expr::{Subquery, SubqueryAlias}; -use sqlparser::ast::{FunctionArg, FunctionArgExpr, Spanned, TableFactor}; +use sqlparser::ast::{ + Expr as SQLExpr, FunctionArg, FunctionArgExpr, FunctionArguments, Spanned, Statement, + TableFactor, TableVersion as SQLTableVersion, Visit, Visitor, +}; mod join; @@ -34,6 +38,48 @@ fn get_cte_for_table_reference<'a>( planner_context.get_cte(&name.to_string()) } +/// Returns the expression of a time travel clause +fn version_expr(version: &TableVersionExpr) -> &Expr { + match version { + TableVersionExpr::Version(expr) | TableVersionExpr::Timestamp(expr) => expr, + } +} + +/// Returns the argument name and value of an `AT ( => )` time +/// travel clause, or `None` if `func` has any other form. +fn at_clause_argument(func: &SQLExpr) -> Option<(&str, &SQLExpr)> { + let SQLExpr::Function(func) = func else { + return None; + }; + let is_at = matches!( + func.name.0.as_slice(), + [part] if part.as_ident().is_some_and(|i| i.value.eq_ignore_ascii_case("AT")) + ); + let FunctionArguments::List(list) = &func.args else { + return None; + }; + if !is_at || list.duplicate_treatment.is_some() || !list.clauses.is_empty() { + return None; + } + match list.args.as_slice() { + [ + FunctionArg::Named { + name, + arg: FunctionArgExpr::Expr(expr), + .. + }, + ] + | [ + FunctionArg::ExprNamed { + name: SQLExpr::Identifier(name), + arg: FunctionArgExpr::Expr(expr), + .. + }, + ] => Some((name.value.as_str(), expr)), + _ => None, + } +} + // Implement RelationPlannerContext impl RelationPlannerContext for SqlToRelRelationContext<'_, '_, S> { fn context_provider(&self) -> &dyn ContextProvider { @@ -48,17 +94,13 @@ impl RelationPlannerContext for SqlToRelRelationContext<'_, get_cte_for_table_reference(self.planner_context, name) } - fn sql_to_expr( - &mut self, - expr: sqlparser::ast::Expr, - schema: &DFSchema, - ) -> Result { + fn sql_to_expr(&mut self, expr: SQLExpr, schema: &DFSchema) -> Result { self.planner.sql_to_expr(expr, schema, self.planner_context) } fn sql_expr_to_logical_expr( &mut self, - expr: sqlparser::ast::Expr, + expr: SQLExpr, schema: &DFSchema, ) -> Result { self.planner @@ -148,9 +190,18 @@ impl SqlToRel<'_, S> { let relation_span = relation.span(); let (plan, alias) = match relation { TableFactor::Table { - name, alias, args, .. + name, + alias, + args, + version, + .. } => { if let Some(func_args) = args { + if version.is_some() { + return not_impl_err!( + "Time travel is not supported for table functions" + ); + } let tbl_func_name = name.0.first().unwrap().as_ident().unwrap().to_string(); let args = func_args @@ -185,6 +236,20 @@ impl SqlToRel<'_, S> { // Normalize name and alias let table_ref = self.object_name_to_table_reference(name)?; let cte = get_cte_for_table_reference(planner_context, &table_ref); + if let Some(version) = version { + if cte.is_some() { + return plan_err!( + "Time travel is not supported for CTE '{table_ref}'" + ); + } + let plan = self.versioned_table_scan( + table_ref, + version, + relation_span, + planner_context, + )?; + return Ok(PlannedRelation::new(plan, alias)); + } ( match ( cte, @@ -301,6 +366,143 @@ impl SqlToRel<'_, S> { Ok(PlannedRelation::new(plan, alias)) } + /// Plans a scan of a specific version of a table, as requested by a time + /// travel clause such as `VERSION AS OF 3`. + fn versioned_table_scan( + &self, + table_ref: TableReference, + version: SQLTableVersion, + relation_span: sqlparser::tokenizer::Span, + planner_context: &mut PlannerContext, + ) -> Result { + let version = self.sql_table_version_to_expr(version, planner_context)?; + if version_expr(&version).exists(|e| Ok(matches!(e, Expr::Placeholder(_))))? { + return not_impl_err!( + "Parameters are not supported in time travel clauses: {table_ref}" + ); + } + let version = self.context_provider.evaluate_table_version(version)?; + let provider = self + .context_provider + .get_table_source_at(table_ref.clone(), &version) + .map_err(|e| { + e.with_diagnostic(Diagnostic::new_error( + format!("cannot read table '{table_ref}@{version}'"), + Span::try_from_sqlparser_span(relation_span), + )) + })?; + // The source reports the version it reads, which may differ from the + // requested version, e.g. a snapshot id for a timestamp + if provider.version().is_none() { + return plan_err!( + "The source of table '{table_ref}@{version}' must report the version \ + it reads" + ); + } + LogicalPlanBuilder::scan(table_ref, provider, None)?.build() + } + + /// Converts a time travel clause to a [`TableVersionExpr`]. + /// + /// The different syntaxes used by SQL dialects are normalized to either a + /// version or a timestamp: + /// + /// * `VERSION AS OF ` and `AT (VERSION => )` become + /// [`TableVersionExpr::Version`] + /// * `TIMESTAMP AS OF `, `FOR SYSTEM_TIME AS OF ` and + /// `AT (TIMESTAMP => )` become [`TableVersionExpr::Timestamp`] + /// + /// Other clauses, such as Snowflake's `BEFORE (...)`, `AT (OFFSET => ...)` + /// or `CHANGES (...)`, are not supported. + pub fn sql_table_version_to_expr( + &self, + version: SQLTableVersion, + planner_context: &mut PlannerContext, + ) -> Result { + let (is_timestamp, expr) = match version { + SQLTableVersion::VersionAsOf(expr) => (false, expr), + SQLTableVersion::TimestampAsOf(expr) + | SQLTableVersion::ForSystemTimeAsOf(expr) => (true, expr), + SQLTableVersion::Function(func) => match at_clause_argument(&func) { + Some((name, expr)) if name.eq_ignore_ascii_case("VERSION") => { + (false, expr.clone()) + } + Some((name, expr)) if name.eq_ignore_ascii_case("TIMESTAMP") => { + (true, expr.clone()) + } + _ => return not_impl_err!("Unsupported time travel clause: {func}"), + }, + SQLTableVersion::Changes { .. } => { + return not_impl_err!("CHANGES clauses are not supported"); + } + }; + let expr = self.sql_to_expr(expr, &DFSchema::empty(), planner_context)?; + Ok(if is_timestamp { + TableVersionExpr::Timestamp(expr) + } else { + TableVersionExpr::Version(expr) + }) + } + + /// Rejects time travel clauses in a prepared statement that depend on the + /// time of execution, such as `AT (TIMESTAMP => now())`. + /// + /// Time travel clauses are evaluated, and the table version is resolved, + /// when the statement is prepared. Functions such as `now()` would + /// therefore return the time of preparation rather than of execution. + pub(crate) fn check_prepared_table_versions( + &self, + statement: &Statement, + ) -> Result<()> { + #[derive(Default)] + struct TableVersions(Vec); + + impl Visitor for TableVersions { + type Break = (); + + fn pre_visit_table_factor( + &mut self, + table_factor: &TableFactor, + ) -> ControlFlow { + if let TableFactor::Table { + version: Some(version), + .. + } = table_factor + { + self.0.push(version.clone()); + } + ControlFlow::Continue(()) + } + } + + let mut versions = TableVersions::default(); + let _ = statement.visit(&mut versions); + for version in versions.0 { + let display = version.to_string(); + // Errors are reported when the statement is planned + let Ok(expr) = + self.sql_table_version_to_expr(version, &mut PlannerContext::new()) + else { + continue; + }; + let depends_on_execution = version_expr(&expr).exists(|e| { + Ok(matches!( + e, + Expr::ScalarFunction(f) + if f.func.signature().volatility != Volatility::Immutable + )) + })?; + if depends_on_execution { + return not_impl_err!( + "Time travel clauses in prepared statements must not depend on \ + the time of execution, such as with now(): {}", + display.trim() + ); + } + } + Ok(()) + } + pub(crate) fn create_relation_subquery( &self, subquery: TableFactor, diff --git a/datafusion/sql/src/resolve.rs b/datafusion/sql/src/resolve.rs index 619fb583f1404..c7422359e2d27 100644 --- a/datafusion/sql/src/resolve.rs +++ b/datafusion/sql/src/resolve.rs @@ -16,9 +16,15 @@ use sqlparser::ast::*; // This avoids a second normalization/conversion pass after visiting the AST. struct RelationVisitor { relations: BTreeSet, + versioned_relations: BTreeSet<(TableReference, TableVersion)>, all_ctes: BTreeSet, ctes_in_scope: Vec, enable_ident_normalization: bool, + /// The names of the table factors with a time travel clause that are + /// being visited, innermost last. These names are recorded in + /// `versioned_relations`, so they are skipped when visited as relations. + /// Only used for pointer comparison. + versioned_names: Vec<*const ObjectName>, } impl RelationVisitor { @@ -39,15 +45,83 @@ impl RelationVisitor { Err(e) => ControlFlow::Break(e), } } + + /// Record the reference to `relation` with a time travel clause, such as + /// `VERSION AS OF 3`, if it's not a CTE reference. + fn insert_versioned_relation( + &mut self, + relation: &ObjectName, + version: &TableVersion, + ) -> ControlFlow { + match object_name_to_table_reference( + relation.clone(), + self.enable_ident_normalization, + ) { + Ok(relation) => { + if !self.ctes_in_scope.contains(&relation) { + self.versioned_relations.insert((relation, version.clone())); + } + ControlFlow::Continue(()) + } + Err(e) => ControlFlow::Break(e), + } + } +} + +/// Returns the name and time travel clause of `table_factor`, if it is a +/// table with a time travel clause, such as `t VERSION AS OF 3` +fn versioned_table(table_factor: &TableFactor) -> Option<(&ObjectName, &TableVersion)> { + match table_factor { + TableFactor::Table { + name, + args: None, + version: Some(version), + .. + } => Some((name, version)), + _ => None, + } } impl Visitor for RelationVisitor { type Break = DataFusionError; fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow { + if self + .versioned_names + .iter() + .any(|name| std::ptr::eq(*name, relation)) + { + return ControlFlow::Continue(()); + } + self.insert_relation(relation) } + fn pre_visit_table_factor( + &mut self, + table_factor: &TableFactor, + ) -> ControlFlow { + if let Some((name, version)) = versioned_table(table_factor) { + // The table name is also visited as a relation. Skip exactly that + // node, so it is not recorded without its version as well. Other + // relations within this table factor, such as those in a subquery + // in the time travel clause, are still recorded. + self.versioned_names.push(name); + self.insert_versioned_relation(name, version)?; + } + ControlFlow::Continue(()) + } + + fn post_visit_table_factor( + &mut self, + table_factor: &TableFactor, + ) -> ControlFlow { + if versioned_table(table_factor).is_some() { + self.versioned_names.pop(); + } + ControlFlow::Continue(()) + } + fn pre_visit_query(&mut self, q: &Query) -> ControlFlow { if let Some(with) = &q.with { for cte in &with.cte_tables { @@ -129,9 +203,26 @@ fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) -> Re Ok(()) } +/// Tables, views and CTEs referenced in a SQL statement, as collected by +/// [`resolve_table_references_with_versions`]. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TableReferences { + /// Tables and views referenced without a time travel clause + pub tables: Vec, + /// Tables and views referenced with a time travel clause, such as + /// `VERSION AS OF 3`, together with that clause + pub versioned_tables: Vec<(TableReference, TableVersion)>, + /// CTE aliases that were defined and possibly referenced + pub ctes: Vec, +} + /// Collects all tables and views referenced in the SQL statement. CTEs are collected separately. /// This can be used to determine which tables need to be in the catalog for a query to be planned. /// +/// Tables referenced with a time travel clause, such as `VERSION AS OF 3`, +/// are included without their version. Use +/// [`resolve_table_references_with_versions`] to also obtain those versions. +/// /// # Returns /// /// A `(table_refs, ctes)` tuple, the first element contains table and view references and the second @@ -167,25 +258,198 @@ pub fn resolve_table_references( statement: &crate::parser::Statement, enable_ident_normalization: bool, ) -> Result<(Vec, Vec)> { + let visitor = visit_relations(statement, enable_ident_normalization)?; + let mut relations = visitor.relations; + relations.extend(visitor.versioned_relations.into_iter().map(|(r, _)| r)); + Ok(( + relations.into_iter().collect(), + visitor.all_ctes.into_iter().collect(), + )) +} + +/// Collects all tables, views and CTEs referenced in the SQL statement, like +/// [`resolve_table_references`], and also the time travel clauses of tables, +/// such as `VERSION AS OF 3`. +/// +/// A table referenced with a time travel clause is only included in +/// [`TableReferences::tables`] if it is also referenced without one. +/// +/// ## Example +/// +/// ``` +/// # use datafusion_sql::parser::DFParserBuilder; +/// # use datafusion_sql::resolve::resolve_table_references_with_versions; +/// # use datafusion_sql::sqlparser::dialect::DatabricksDialect; +/// let query = "SELECT * FROM foo JOIN bar VERSION AS OF 3 ON foo.id = bar.id"; +/// let statement = DFParserBuilder::new(query) +/// .with_dialect(&DatabricksDialect {}) +/// .build() +/// .unwrap() +/// .parse_statements() +/// .unwrap() +/// .pop_back() +/// .unwrap(); +/// let references = resolve_table_references_with_versions(&statement, true).unwrap(); +/// assert_eq!(references.tables.len(), 1); +/// assert_eq!(references.tables[0].to_string(), "foo"); +/// assert_eq!(references.versioned_tables.len(), 1); +/// assert_eq!(references.versioned_tables[0].0.to_string(), "bar"); +/// assert_eq!(references.versioned_tables[0].1.to_string(), "VERSION AS OF 3"); +/// ``` +pub fn resolve_table_references_with_versions( + statement: &crate::parser::Statement, + enable_ident_normalization: bool, +) -> Result { + let visitor = visit_relations(statement, enable_ident_normalization)?; + Ok(TableReferences { + tables: visitor.relations.into_iter().collect(), + versioned_tables: visitor.versioned_relations.into_iter().collect(), + ctes: visitor.all_ctes.into_iter().collect(), + }) +} + +fn visit_relations( + statement: &crate::parser::Statement, + enable_ident_normalization: bool, +) -> Result { let mut visitor = RelationVisitor { relations: BTreeSet::new(), + versioned_relations: BTreeSet::new(), all_ctes: BTreeSet::new(), ctes_in_scope: vec![], enable_ident_normalization, + versioned_names: vec![], }; - visit_statement(statement, &mut visitor)?; - - Ok(( - visitor.relations.into_iter().collect(), - visitor.all_ctes.into_iter().collect(), - )) + Ok(visitor) } #[cfg(test)] mod tests { use super::*; + fn parse_snowflake(query: &str) -> crate::parser::Statement { + use crate::parser::DFParserBuilder; + use sqlparser::dialect::SnowflakeDialect; + + DFParserBuilder::new(query) + .with_dialect(&SnowflakeDialect {}) + .build() + .unwrap() + .parse_statements() + .unwrap() + .pop_back() + .unwrap() + } + + #[test] + fn resolve_table_references_with_versions_collects_versions() { + let statement = parse_snowflake( + "SELECT * FROM a \ + JOIN b AT (VERSION => 1) ON true \ + JOIN B AT (VERSION => 1) ON true \ + JOIN b TIMESTAMP AS OF now() ON true \ + WHERE a.x IN (SELECT x FROM c VERSION AS OF 2)", + ); + let references = + resolve_table_references_with_versions(&statement, true).unwrap(); + let tables: Vec<_> = references.tables.iter().map(|t| t.to_string()).collect(); + assert_eq!(tables, ["a"]); + let versioned: Vec<_> = references + .versioned_tables + .iter() + .map(|(t, v)| format!("{t} {v}")) + .collect(); + assert_eq!( + versioned, + [ + "b TIMESTAMP AS OF now()", + "b AT(VERSION => 1)", + "c VERSION AS OF 2" + ] + ); + assert!(references.ctes.is_empty()); + + // `resolve_table_references` returns all tables, without versions + let (tables, ctes) = resolve_table_references(&statement, true).unwrap(); + let tables: Vec<_> = tables.iter().map(|t| t.to_string()).collect(); + assert_eq!(tables, ["a", "b", "c"]); + assert_eq!(ctes, references.ctes); + } + + #[test] + fn resolve_table_references_with_versions_mixed() { + // A table referenced both with and without a time travel clause + let statement = parse_snowflake( + "SELECT * FROM t AS cur JOIN t VERSION AS OF 1 AS old ON true", + ); + let references = + resolve_table_references_with_versions(&statement, true).unwrap(); + let tables: Vec<_> = references.tables.iter().map(|t| t.to_string()).collect(); + assert_eq!(tables, ["t"]); + let versioned: Vec<_> = references + .versioned_tables + .iter() + .map(|(t, v)| format!("{t} {v}")) + .collect(); + assert_eq!(versioned, ["t VERSION AS OF 1"]); + } + + #[test] + fn resolve_table_references_with_versions_subquery_in_version() { + // Tables in a subquery in the time travel clause are still recorded. + // Planning reports an error for the non-constant version. + let tables_of = |query: &str| { + let references = + resolve_table_references_with_versions(&parse_snowflake(query), true) + .unwrap(); + let tables: Vec<_> = + references.tables.iter().map(|t| t.to_string()).collect(); + let versioned: Vec<_> = references + .versioned_tables + .iter() + .map(|(t, _)| t.to_string()) + .collect(); + (tables, versioned) + }; + + let (tables, versioned) = + tables_of("SELECT * FROM t TIMESTAMP AS OF (SELECT max(ts) FROM log)"); + assert_eq!(tables, ["log"]); + assert_eq!(versioned, ["t"]); + + let (tables, versioned) = tables_of( + "SELECT * FROM t AT (VERSION => (SELECT max(v) FROM a JOIN b ON true))", + ); + assert_eq!(tables, ["a", "b"]); + assert_eq!(versioned, ["t"]); + + let (tables, versioned) = tables_of( + "SELECT * FROM t AT (VERSION => \ + (SELECT max(v) FROM log VERSION AS OF 1 JOIN a ON true))", + ); + assert_eq!(tables, ["a"]); + assert_eq!(versioned, ["log", "t"]); + } + + #[test] + fn resolve_table_references_with_versions_skips_ctes() { + // A versioned reference to a CTE is not a table reference. Planning + // reports an error for it. + let statement = parse_snowflake( + "WITH t AS (SELECT * FROM t VERSION AS OF 1) SELECT * FROM t VERSION AS OF 2", + ); + let references = + resolve_table_references_with_versions(&statement, true).unwrap(); + let versioned: Vec<_> = references + .versioned_tables + .iter() + .map(|(t, v)| format!("{t} {v}")) + .collect(); + assert_eq!(versioned, ["t VERSION AS OF 1"]); + assert_eq!(references.ctes.len(), 1); + } + #[test] fn resolve_table_references_shadowed_cte() { use crate::parser::DFParser; diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 967d42a4282dc..6aadcdf317f63 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -853,6 +853,8 @@ impl SqlToRel<'_, S> { .map(|t| self.convert_data_type_to_field(&t)) .collect::>()?; + self.check_prepared_table_versions(&statement)?; + // Create planner context with parameters let mut planner_context = PlannerContext::new() .with_prepare_param_data_types( @@ -1648,11 +1650,14 @@ impl SqlToRel<'_, S> { if !table_factor.joins.is_empty() { return not_impl_err!("DELETE FROM only supports single table, got: joins"); } - let TableFactor::Table { name, .. } = table_factor.relation else { + let TableFactor::Table { name, version, .. } = table_factor.relation else { return not_impl_err!( "DELETE FROM only supports single table, got: {table_factor:?}" ); }; + if version.is_some() { + return not_impl_err!("Time travel is not supported for DELETE targets"); + } Ok(name) } @@ -2322,6 +2327,11 @@ impl SqlToRel<'_, S> { predicate_expr: Option, ) -> Result { let (table_name, table_alias) = match &table.relation { + TableFactor::Table { + version: Some(_), .. + } => { + return not_impl_err!("Time travel is not supported for UPDATE targets"); + } TableFactor::Table { name, alias, .. } => (name.clone(), alias.clone()), _ => plan_err!("Cannot update non-table relation!")?, }; diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rs index 2a5d32409132a..7d9dd9ef85c47 100644 --- a/datafusion/sql/src/unparser/dialect.rs +++ b/datafusion/sql/src/unparser/dialect.rs @@ -92,6 +92,12 @@ pub trait Dialect: Send + Sync { CharacterLengthStyle::CharacterLength } + /// The syntax to use when unparsing the time travel clause of a table + /// scan, such as `VERSION AS OF 3` + fn table_version_style(&self) -> TableVersionStyle { + TableVersionStyle::At + } + /// The SQL type to use for Arrow Int64 unparsing /// Most dialects use BigInt, but some, like MySQL, require SIGNED fn int64_cast_dtype(&self) -> ast::DataType { @@ -356,6 +362,19 @@ pub enum CharacterLengthStyle { CharacterLength, } +/// `TableVersionStyle` to use for unparsing the time travel clause of a table +/// scan, such as `VERSION AS OF 3` +#[derive(Clone, Copy, PartialEq)] +pub enum TableVersionStyle { + /// `AT (VERSION => v)` and `AT (TIMESTAMP => t)`, as used by DuckDB + At, + /// `VERSION AS OF v` and `TIMESTAMP AS OF t`, as used by Databricks + AsOf, + /// `FOR SYSTEM_TIME AS OF t`, as used by BigQuery and SQL Server. Only + /// timestamps are supported. + ForSystemTime, +} + /// `DistinctFromStyle` to use for unparsing `IsDistinctFrom` and `IsNotDistinctFrom` operators #[derive(Clone, Copy, PartialEq)] pub enum DistinctFromStyle { @@ -742,6 +761,10 @@ impl Dialect for BigQueryDialect { Some('`') } + fn table_version_style(&self) -> TableVersionStyle { + TableVersionStyle::ForSystemTime + } + fn col_alias_overrides(&self, alias: &str) -> Result> { // Check if alias contains any special characters not supported by BigQuery col names // https://cloud.google.com/bigquery/docs/schemas#flexible-column-names @@ -890,6 +913,7 @@ pub struct CustomDialect { large_utf8_cast_dtype: ast::DataType, date_field_extract_style: DateFieldExtractStyle, character_length_style: CharacterLengthStyle, + table_version_style: TableVersionStyle, int8_cast_dtype: ast::DataType, int64_cast_dtype: ast::DataType, int32_cast_dtype: ast::DataType, @@ -922,6 +946,7 @@ impl Default for CustomDialect { large_utf8_cast_dtype: ast::DataType::Text, date_field_extract_style: DateFieldExtractStyle::DatePart, character_length_style: CharacterLengthStyle::CharacterLength, + table_version_style: TableVersionStyle::At, int8_cast_dtype: ast::DataType::TinyInt(None), int64_cast_dtype: ast::DataType::BigInt(None), int32_cast_dtype: ast::DataType::Integer(None), @@ -984,6 +1009,10 @@ impl Dialect for CustomDialect { self.character_length_style } + fn table_version_style(&self) -> TableVersionStyle { + self.table_version_style + } + fn int64_cast_dtype(&self) -> ast::DataType { self.int64_cast_dtype.clone() } @@ -1107,6 +1136,7 @@ pub struct CustomDialectBuilder { large_utf8_cast_dtype: ast::DataType, date_field_extract_style: DateFieldExtractStyle, character_length_style: CharacterLengthStyle, + table_version_style: TableVersionStyle, int8_cast_dtype: ast::DataType, int64_cast_dtype: ast::DataType, int32_cast_dtype: ast::DataType, @@ -1145,6 +1175,7 @@ impl CustomDialectBuilder { large_utf8_cast_dtype: ast::DataType::Text, date_field_extract_style: DateFieldExtractStyle::DatePart, character_length_style: CharacterLengthStyle::CharacterLength, + table_version_style: TableVersionStyle::At, int8_cast_dtype: ast::DataType::TinyInt(None), int64_cast_dtype: ast::DataType::BigInt(None), int32_cast_dtype: ast::DataType::Integer(None), @@ -1180,6 +1211,7 @@ impl CustomDialectBuilder { large_utf8_cast_dtype: self.large_utf8_cast_dtype, date_field_extract_style: self.date_field_extract_style, character_length_style: self.character_length_style, + table_version_style: self.table_version_style, int8_cast_dtype: self.int8_cast_dtype, int64_cast_dtype: self.int64_cast_dtype, int32_cast_dtype: self.int32_cast_dtype, @@ -1241,6 +1273,15 @@ impl CustomDialectBuilder { self } + /// Customize the dialect with a specific time travel syntax listed in `TableVersionStyle` + pub fn with_table_version_style( + mut self, + table_version_style: TableVersionStyle, + ) -> Self { + self.table_version_style = table_version_style; + self + } + /// Customize the dialect with a specific SQL type for Int8 casting: TinyInt, SmallInt, etc. pub fn with_int8_cast_dtype(mut self, int8_cast_dtype: ast::DataType) -> Self { self.int8_cast_dtype = int8_cast_dtype; diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 564de3a2816f6..037ab4c11242d 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -8,6 +8,7 @@ use super::{ BuilderError, DerivedRelationBuilder, QueryBuilder, RelationBuilder, SelectBuilder, TableRelationBuilder, TableWithJoinsBuilder, }, + dialect::TableVersionStyle, rewrite::{ TableAliasRewriter, inject_column_aliases_into_subquery, normalize_union_schema, rewrite_plan_for_sort_on_non_projected_fields, @@ -30,7 +31,7 @@ use crate::unparser::{ }; use crate::utils::UNNEST_PLACEHOLDER; use datafusion_common::{ - Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference, + Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference, TableVersion, assert_or_internal_err, internal_datafusion_err, internal_err, not_impl_err, tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion}, utils::combine_limit, @@ -869,6 +870,12 @@ impl Unparser<'_> { self.new_ident_quoted_if_needs(scan.table_name.table().to_string()), ); builder.name(ast::ObjectName::from(table_parts)); + builder.version( + scan.source + .version() + .map(|v| self.table_version_to_sql(v)) + .transpose()?, + ); relation.table(builder); Ok(()) @@ -2337,6 +2344,52 @@ impl Unparser<'_> { Ok(Some(flatten)) } + /// Converts the time travel clause of a table scan to SQL, using the + /// syntax of the dialect + fn table_version_to_sql(&self, version: &TableVersion) -> Result { + let value = self.expr_to_sql(&Expr::Literal(version.value(), None))?; + Ok(match (self.dialect.table_version_style(), version) { + (TableVersionStyle::At, _) => { + let name = match version { + TableVersion::Version(_) => "VERSION", + TableVersion::Timestamp(_) => "TIMESTAMP", + }; + ast::TableVersion::Function(ast::Expr::Function(ast::Function { + name: ast::ObjectName::from(vec![Ident::new("AT")]), + uses_odbc_syntax: false, + parameters: ast::FunctionArguments::None, + args: ast::FunctionArguments::List(ast::FunctionArgumentList { + duplicate_treatment: None, + args: vec![ast::FunctionArg::Named { + name: Ident::new(name), + arg: ast::FunctionArgExpr::Expr(value), + operator: ast::FunctionArgOperator::RightArrow, + }], + clauses: vec![], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + })) + } + (TableVersionStyle::AsOf, TableVersion::Version(_)) => { + ast::TableVersion::VersionAsOf(value) + } + (TableVersionStyle::AsOf, TableVersion::Timestamp(_)) => { + ast::TableVersion::TimestampAsOf(value) + } + (TableVersionStyle::ForSystemTime, TableVersion::Timestamp(_)) => { + ast::TableVersion::ForSystemTimeAsOf(value) + } + (TableVersionStyle::ForSystemTime, TableVersion::Version(_)) => { + return not_impl_err!( + "This dialect does not support time travel to version {version}" + ); + } + }) + } + fn is_scan_with_pushdown(scan: &TableScan) -> bool { scan.projection.is_some() || !scan.filters.is_empty() diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs index 2c08b5595ab74..73d69df2b4f08 100644 --- a/datafusion/sql/tests/cases/plan_to_sql.rs +++ b/datafusion/sql/tests/cases/plan_to_sql.rs @@ -2,12 +2,12 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::tree_node::{Transformed, TransformedResult}; use datafusion_common::{ - Column, DFSchema, DFSchemaRef, DataFusionError, Result, TableReference, - assert_contains, + Column, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, TableReference, + TableVersion, assert_contains, }; use datafusion_expr::expr::{WindowFunction, WindowFunctionParams}; use datafusion_expr::test::function_stub::{ @@ -15,10 +15,10 @@ use datafusion_expr::test::function_stub::{ }; use datafusion_expr::{ ColumnarValue, EmptyRelation, Expr, Extension, LogicalPlan, LogicalPlanBuilder, - ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TableScanBuilder, Union, - UserDefinedLogicalNode, UserDefinedLogicalNodeCore, Volatility, WindowFrame, - WindowFunctionDefinition, cast, col, exists, in_subquery, lit, scalar_subquery, - table_scan, wildcard, + ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TableScanBuilder, + TableSource, Union, UserDefinedLogicalNode, UserDefinedLogicalNodeCore, Volatility, + WindowFrame, WindowFunctionDefinition, cast, col, exists, in_subquery, lit, + scalar_subquery, table_scan, wildcard, }; use datafusion_functions::unicode; use datafusion_functions_aggregate::grouping::grouping_udaf; @@ -31,6 +31,7 @@ use datafusion_sql::unparser::dialect::{ BigQueryDialect, CustomDialectBuilder, DefaultDialect as UnparserDefaultDialect, DefaultDialect, Dialect as UnparserDialect, MySqlDialect as UnparserMySqlDialect, PostgreSqlDialect as UnparserPostgreSqlDialect, SnowflakeDialect, SqliteDialect, + TableVersionStyle, }; use datafusion_sql::unparser::{Unparser, expr_to_sql, plan_to_sql}; use insta::assert_snapshot; @@ -1690,6 +1691,77 @@ fn test_table_scan_alias() -> Result<()> { Ok(()) } +#[test] +fn test_table_scan_version_to_sql() -> Result<()> { + let schema = Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("age", DataType::Utf8, false), + ]); + /// A source that reads a specific version of a table + struct VersionedSource { + schema: SchemaRef, + version: TableVersion, + } + + impl TableSource for VersionedSource { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } + + fn version(&self) -> Option<&TableVersion> { + Some(&self.version) + } + } + + let scan = |version: TableVersion, filters: Vec| -> Result { + let source = Arc::new(VersionedSource { + schema: Arc::new(schema.clone()), + version, + }); + Ok(LogicalPlan::TableScan( + TableScanBuilder::new("t1", source) + .with_filters(filters) + .build()?, + )) + }; + let version = TableVersion::Version(ScalarValue::Int64(Some(3))); + let tag = TableVersion::Version(ScalarValue::from("v1")); + // 2024-01-01T00:00:00Z + let timestamp = TableVersion::Timestamp(1_704_067_200_000_000_000); + + let sql = plan_to_sql(&scan(version.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 AT(VERSION => 3)"); + let sql = plan_to_sql(&scan(tag.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 AT(VERSION => 'v1')"); + let sql = plan_to_sql(&scan(timestamp.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 AT(TIMESTAMP => CAST('2024-01-01T00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE))"); + + // The version is kept when filters are pushed down into the scan + let sql = plan_to_sql(&scan(version.clone(), vec![col("id").eq(lit("a"))])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 AT(VERSION => 3) WHERE (t1.id = 'a')"); + + let as_of = CustomDialectBuilder::new() + .with_table_version_style(TableVersionStyle::AsOf) + .build(); + let unparser = Unparser::new(&as_of); + let sql = unparser.plan_to_sql(&scan(version.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 VERSION AS OF 3"); + let sql = unparser.plan_to_sql(&scan(timestamp.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 TIMESTAMP AS OF CAST('2024-01-01T00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE)"); + + let big_query = BigQueryDialect {}; + let unparser = Unparser::new(&big_query); + let sql = unparser.plan_to_sql(&scan(timestamp, vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM `t1` FOR SYSTEM_TIME AS OF CAST('2024-01-01T00:00:00+00:00' AS TIMESTAMP)"); + let err = unparser + .plan_to_sql(&scan(version, vec![])?) + .unwrap_err() + .strip_backtrace(); + assert_snapshot!(err, @"This feature is not implemented: This dialect does not support time travel to version 3"); + + Ok(()) +} + #[test] fn test_table_scan_pushdown() -> Result<()> { let schema = Schema::new(vec![ diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index 1046ae9d8e548..58af0fa63ab24 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -12,8 +12,12 @@ use arrow::datatypes::*; use datafusion_common::config::ConfigOptions; use datafusion_common::datatype::DataTypeExt; use datafusion_common::file_options::file_type::FileType; -use datafusion_common::{DFSchema, GetExt, Result, TableReference, plan_err}; -use datafusion_expr::planner::{ExprPlanner, PlannerResult, TypePlanner}; +use datafusion_common::{ + DFSchema, GetExt, Result, TableReference, TableVersion, not_impl_err, plan_err, +}; +use datafusion_expr::planner::{ + ExprPlanner, PlannerResult, TableVersionExpr, TypePlanner, +}; use datafusion_expr::{ AggregateUDF, Expr, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF, }; @@ -333,6 +337,34 @@ impl ContextProvider for MockContextProvider { Ok(Arc::new(EmptyTable::new(schema))) } + /// Only literal versions are supported, as evaluating expressions + /// requires an `ExprSimplifier` + fn evaluate_table_version(&self, version: TableVersionExpr) -> Result { + match version { + TableVersionExpr::Version(Expr::Literal(value, _)) => { + Ok(TableVersion::Version(value)) + } + version => not_impl_err!("Unsupported table version: {version:?}"), + } + } + + /// Only `person` supports time travel. For `j1`, the current version is + /// returned without reporting a version, which is invalid. + fn get_table_source_at( + &self, + name: TableReference, + version: &TableVersion, + ) -> Result> { + match name.table() { + "person" => Ok(Arc::new(EmptyTable::new_with_version( + self.get_table_source(name)?.schema(), + version.clone(), + ))), + "j1" => self.get_table_source(name), + _ => not_impl_err!("Time travel is not supported for table '{name}'"), + } + } + fn udf_names(&self) -> Vec { self.state.scalar_functions.keys().cloned().collect() } @@ -364,11 +396,22 @@ impl ContextProvider for MockContextProvider { struct EmptyTable { table_schema: SchemaRef, + version: Option, } impl EmptyTable { fn new(table_schema: SchemaRef) -> Self { - Self { table_schema } + Self { + table_schema, + version: None, + } + } + + fn new_with_version(table_schema: SchemaRef, version: TableVersion) -> Self { + Self { + table_schema, + version: Some(version), + } } } @@ -376,6 +419,10 @@ impl TableSource for EmptyTable { fn schema(&self) -> SchemaRef { Arc::clone(&self.table_schema) } + + fn version(&self) -> Option<&TableVersion> { + self.version.as_ref() + } } #[derive(Debug)] diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index d89984da3ce2a..798aa97c60be1 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -6174,3 +6174,26 @@ fn test_reserved_column_name_explain_copy() { "Error during planning: __common_expr_1 is a reserved DataFusion column name, please use another name" ); } + +#[test] +fn test_time_travel() { + let dialect = &DatabricksDialect {}; + + // The version is reported by the source and shown with the scan. It does + // not change how columns are qualified. + let plan = logical_plan_with_dialect( + "SELECT person.id FROM person VERSION AS OF 3 WHERE person.age > 21", + dialect, + ) + .unwrap(); + assert_snapshot!(plan, @r" + Projection: person.id + Filter: person.age > Int64(21) + TableScan: person@3 + "); + + // A source that does not report the version it reads is rejected + let err = logical_plan_with_dialect("SELECT * FROM j1 VERSION AS OF 3", dialect) + .unwrap_err(); + assert_snapshot!(err.strip_backtrace(), @"Error during planning: The source of table 'j1@3' must report the version it reads"); +} diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index ffcff02cdd525..251a3d9751cfc 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -61,6 +61,7 @@ mod custom_sort_pushdown; mod evaluation_counter; mod exact_sort_pushdown; mod range_partitioning; +mod time_travel; /// Context for running tests pub struct TestContext { @@ -240,6 +241,10 @@ impl TestContext { info!("Registering table with dictionary-encoded struct column"); register_dictionary_struct_table(test_ctx.session_ctx()); } + "time_travel.slt" => { + info!("Registering versioned schema"); + time_travel::register_versioned_schema(test_ctx.session_ctx()); + } "async_udf.slt" => { info!("Registering dummy async udf"); register_async_abs_udf(test_ctx.session_ctx()) diff --git a/datafusion/sqllogictest/src/test_context/time_travel.rs b/datafusion/sqllogictest/src/test_context/time_travel.rs new file mode 100644 index 0000000000000..a7cb4448962d5 --- /dev/null +++ b/datafusion/sqllogictest/src/test_context/time_travel.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Int64Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use async_trait::async_trait; +use datafusion::catalog::SchemaProvider; +use datafusion::common::{Result, ScalarValue, TableVersion, plan_err}; +use datafusion::datasource::{MemTable, TableProvider}; +use datafusion::prelude::SessionContext; + +// ============================================================================== +// Versioned Schema Provider (sqllogictest-only) +// ============================================================================== + +/// A version of table `t` +struct Version { + version: i64, + /// Commit time, in nanoseconds since the epoch + committed: i64, + rows: &'static [(i64, &'static str)], +} + +/// The versions of table `t` +const VERSIONS: [Version; 3] = [ + Version { + version: 1, + committed: 1_704_067_200_000_000_000, // 2024-01-01T00:00:00Z + rows: &[(1, "a")], + }, + Version { + version: 2, + committed: 1_706_745_600_000_000_000, // 2024-02-01T00:00:00Z + rows: &[(1, "a"), (2, "b")], + }, + Version { + version: 3, + committed: 1_709_251_200_000_000_000, // 2024-03-01T00:00:00Z + rows: &[(1, "a"), (2, "b"), (3, "c")], + }, +]; + +/// Named versions (tags) of table `t` +const TAGS: [(&str, i64); 2] = [("first", 1), ("latest", 3)]; + +/// A schema with a single table `t` that supports time travel queries. +/// +/// The table has three versions, which can be selected by version number, by +/// tag or by timestamp. Time travel on table `no_version` returns a table that +/// does not report its version, which is invalid. +#[derive(Debug)] +struct VersionedSchema {} + +impl VersionedSchema { + fn table_version(version: i64) -> Result> { + let Some(Version { rows, .. }) = VERSIONS.iter().find(|v| v.version == version) + else { + return plan_err!("Table t has no version {version}"); + }; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from_iter_values(rows.iter().map(|(id, _)| *id))), + Arc::new(StringArray::from_iter_values( + rows.iter().map(|(_, name)| *name), + )), + ], + )?; + // Report the version that is read, which may differ from the requested + // version, e.g. when it was requested by tag or timestamp + let table = MemTable::try_new(schema, vec![vec![batch]])?.with_version(Some( + TableVersion::Version(ScalarValue::Int64(Some(version))), + )); + Ok(Arc::new(table)) + } +} + +#[async_trait] +impl SchemaProvider for VersionedSchema { + fn table_names(&self) -> Vec { + vec!["t".to_string()] + } + + async fn table(&self, name: &str) -> Result>> { + if name != "t" { + return Ok(None); + } + Self::table_version(VERSIONS[VERSIONS.len() - 1].version).map(Some) + } + + async fn table_at( + &self, + name: &str, + version: &TableVersion, + ) -> Result>> { + if name == "no_version" { + // Invalid: the provider does not report the version it reads + let schema = + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + return Ok(Some(Arc::new(MemTable::try_new(schema, vec![vec![]])?))); + } + if name != "t" { + return Ok(None); + } + let version = match version { + TableVersion::Version(value) => { + if let Some(Some(tag)) = value.try_as_str() { + match TAGS.iter().find(|(t, _)| *t == tag) { + Some((_, version)) => *version, + None => return plan_err!("Table t has no tag '{tag}'"), + } + } else { + match value.cast_to(&DataType::Int64)? { + ScalarValue::Int64(Some(version)) => version, + _ => return plan_err!("Invalid version {value}"), + } + } + } + TableVersion::Timestamp(ts) => { + match VERSIONS.iter().rev().find(|v| v.committed <= *ts) { + Some(v) => v.version, + None => return plan_err!("Table t did not exist at {version}"), + } + } + }; + Self::table_version(version).map(Some) + } + + fn table_exist(&self, name: &str) -> bool { + name == "t" + } +} + +/// Registers the schema `versioned`, which contains a table `t` that supports +/// time travel queries +pub(super) fn register_versioned_schema(ctx: &SessionContext) { + ctx.catalog("datafusion") + .expect("default catalog should exist") + .register_schema("versioned", Arc::new(VersionedSchema {})) + .expect("versioned schema registration should succeed"); +} diff --git a/datafusion/sqllogictest/test_files/time_travel.slt b/datafusion/sqllogictest/test_files/time_travel.slt new file mode 100644 index 0000000000000..085b76875fe75 --- /dev/null +++ b/datafusion/sqllogictest/test_files/time_travel.slt @@ -0,0 +1,368 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +########## +## Time travel queries +## +## The test context registers a schema `versioned` containing a table `t` with +## three versions: +## +## | version | committed at | rows | +## |---------|----------------------|------------------------| +## | 1 | 2024-01-01T00:00:00Z | (1, a) | +## | 2 | 2024-02-01T00:00:00Z | (1, a), (2, b) | +## | 3 | 2024-03-01T00:00:00Z | (1, a), (2, b), (3, c) | +## +## Version 1 is tagged 'first' and version 3 is tagged 'latest'. +########## + +# The generic dialect does not parse time travel clauses (yet) +statement error DataFusion error: SQL error: ParserError +SELECT * FROM versioned.t VERSION AS OF 1 + +# The Snowflake dialect parses all supported time travel syntaxes +statement ok +set datafusion.sql_parser.dialect = 'Snowflake'; + +# Without a time travel clause, the current version is read +query IT rowsort +SELECT * FROM versioned.t +---- +1 a +2 b +3 c + +########## +## Versions +########## + +query IT rowsort +SELECT * FROM versioned.t VERSION AS OF 1 +---- +1 a + +query IT rowsort +SELECT * FROM versioned.t AT (VERSION => 2) +---- +1 a +2 b + +# Version names are case insensitive +query IT rowsort +SELECT * FROM versioned.t at (version => 2) +---- +1 a +2 b + +# Versions can be any scalar value, such as a tag +query IT rowsort +SELECT * FROM versioned.t AT (VERSION => 'first') +---- +1 a + +# Versions are constant expressions that are evaluated during planning +query IT rowsort +SELECT * FROM versioned.t AT (VERSION => 1 + 1) +---- +1 a +2 b + +query IT rowsort +SELECT * FROM versioned.t AT (VERSION => concat('lat', 'est')) +---- +1 a +2 b +3 c + +########## +## Timestamps +########## + +query IT rowsort +SELECT * FROM versioned.t TIMESTAMP AS OF '2024-01-15' +---- +1 a + +query IT rowsort +SELECT * FROM versioned.t FOR SYSTEM_TIME AS OF '2024-02-15T12:00:00Z' +---- +1 a +2 b + +# A timestamp in another time zone is converted to UTC +query IT rowsort +SELECT * FROM versioned.t AT (TIMESTAMP => '2024-02-29T23:30:00-01:00') +---- +1 a +2 b +3 c + +query IT rowsort +SELECT * FROM versioned.t AT (TIMESTAMP => '2024-03-01'::timestamp) +---- +1 a +2 b +3 c + +query IT rowsort +SELECT * FROM versioned.t AT (TIMESTAMP => now() - INTERVAL '1 hour') +---- +1 a +2 b +3 c + +# The syntax produced by the unparser +query IT rowsort +SELECT * FROM versioned.t AT(TIMESTAMP => CAST('2024-01-01T00:00:00+00:00' AS TIMESTAMP WITH TIME ZONE)) +---- +1 a + +query IT rowsort +SELECT * FROM versioned.t AT (TIMESTAMP => DATE '2024-02-01') +---- +1 a +2 b + +########## +## Combining versions +########## + +# The time travel clause does not change how columns are qualified +query IT rowsort +SELECT t.id, t.name FROM versioned.t VERSION AS OF 2 WHERE t.id > 1 +---- +2 b + +query IT rowsort +SELECT * FROM versioned.t AT (VERSION => 1) AS old WHERE old.id = 1 +---- +1 a + +# Different versions of the same table can be read in one query +query ITT rowsort +SELECT cur.id, cur.name, old.name +FROM versioned.t AS cur +LEFT JOIN versioned.t AT (VERSION => 2) AS old ON cur.id = old.id +---- +1 a a +2 b b +3 c NULL + +query IT rowsort +SELECT * FROM versioned.t EXCEPT SELECT * FROM versioned.t AT (VERSION => 1) +---- +2 b +3 c + +query I +SELECT count(*) FROM versioned.t +WHERE id IN (SELECT id FROM versioned.t TIMESTAMP AS OF '2024-01-01') +---- +1 + +# The same version requested with different syntaxes +query I +SELECT count(*) FROM versioned.t VERSION AS OF 2 a +JOIN versioned.t AT (VERSION => 2) b ON a.id = b.id +JOIN versioned.t TIMESTAMP AS OF '2024-02-01' c ON a.id = c.id +---- +2 + +# Scans of different versions of a table must not be treated as equal, for +# example when the optimizer deduplicates identical subqueries +query II +SELECT + (SELECT count(*) FROM versioned.t AT (VERSION => 1)) AS v1, + (SELECT count(*) FROM versioned.t AT (VERSION => 2)) AS v2 +---- +1 2 + +query II +SELECT + (SELECT max(id) FROM versioned.t AT (VERSION => 1)) + 0 AS v1, + (SELECT max(id) FROM versioned.t AT (VERSION => 2)) + 0 AS v2 +---- +1 2 + +query I +SELECT count(*) FROM versioned.t +WHERE id IN (SELECT id FROM versioned.t AT (VERSION => 1)) + OR id IN (SELECT id FROM versioned.t AT (VERSION => 2)) +---- +2 + +# Views capture the version they were defined with +statement ok +CREATE VIEW first_version AS SELECT * FROM versioned.t AT (VERSION => 'first'); + +query IT rowsort +SELECT * FROM first_version +---- +1 a + +statement ok +DROP VIEW first_version; + +########## +## EXPLAIN shows the resolved version +########## + +query TT +EXPLAIN SELECT * FROM versioned.t AT (VERSION => 1 + 1) +---- +logical_plan TableScan: versioned.t@2 projection=[id, name] +physical_plan DataSourceExec: partitions=1, partition_sizes=[1] + +# The tag 'first' resolves to version 1 +query TT +EXPLAIN SELECT name FROM versioned.t AT (VERSION => 'first') WHERE id = 1 +---- +logical_plan +01)Projection: versioned.t.name +02)--Filter: versioned.t.id = Int64(1) +03)----TableScan: versioned.t@1 projection=[id, name] +physical_plan +01)FilterExec: id@0 = 1, projection=[name@1] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# The timestamp resolves to version 1, the latest version committed before it +query TT +EXPLAIN SELECT * FROM versioned.t TIMESTAMP AS OF '2024-01-15T12:30:00+01:00' +---- +logical_plan TableScan: versioned.t@1 projection=[id, name] +physical_plan DataSourceExec: partitions=1, partition_sizes=[1] + +########## +## Errors +########## + +statement error DataFusion error: Error during planning: Table t has no version 7 +SELECT * FROM versioned.t VERSION AS OF 7 + +statement error DataFusion error: Error during planning: Table t has no tag 'nope' +SELECT * FROM versioned.t AT (VERSION => 'nope') + +statement error DataFusion error: Error during planning: Table t did not exist at 2023-01-01T00:00:00Z +SELECT * FROM versioned.t TIMESTAMP AS OF '2023-01-01' + +statement error DataFusion error: Error during planning: table 'datafusion.versioned.unknown@1' not found +SELECT * FROM versioned.unknown VERSION AS OF 1 + +# Catalogs must return a table that reports the version it reads +statement error DataFusion error: Error during planning: The source of table 'versioned.no_version@1' must report the version it reads +SELECT * FROM versioned.no_version VERSION AS OF 1 + +# Tables that do not support time travel +statement ok +CREATE TABLE plain AS VALUES (1); + +statement error DataFusion error: This feature is not implemented: Time travel is not supported for table 'plain@1' +SELECT * FROM plain VERSION AS OF 1 + +statement ok +DROP TABLE plain; + +# The version must be a non-null constant +statement error DataFusion error: Schema error: No field named id\. +SELECT * FROM versioned.t AT (VERSION => id) + +statement error DataFusion error: Error during planning: Time travel version must evaluate to a non-null constant, got: NULL +SELECT * FROM versioned.t AT (VERSION => NULL) + +statement error DataFusion error: Error during planning: Time travel version must evaluate to a non-null constant +SELECT * FROM versioned.t AT (VERSION => random()) + +statement error Error parsing timestamp from 'yesterday' +SELECT * FROM versioned.t AT (TIMESTAMP => 'yesterday') + +# Unsupported time travel clauses +statement error DataFusion error: This feature is not implemented: Unsupported time travel clause: AT\(OFFSET => -60\) +SELECT * FROM versioned.t AT (OFFSET => -60) + +statement error DataFusion error: This feature is not implemented: Unsupported time travel clause: BEFORE\(STATEMENT => 'abc'\) +SELECT * FROM versioned.t BEFORE (STATEMENT => 'abc') + +statement error DataFusion error: This feature is not implemented: Unsupported time travel clause: AT\(VERSION => 1, TIMESTAMP => '2024-01-01'\) +SELECT * FROM versioned.t AT (VERSION => 1, TIMESTAMP => '2024-01-01') + +# Time travel is not supported for CTEs +statement error DataFusion error: Error during planning: Time travel is not supported for CTE 'c' +WITH c AS (SELECT 1) SELECT * FROM c VERSION AS OF 1 + +# Time travel is not supported for DML targets +statement error DataFusion error: This feature is not implemented: Time travel is not supported for DELETE targets +DELETE FROM versioned.t VERSION AS OF 1 + +########## +## Prepared statements +########## + +# Constant time travel clauses are resolved when the statement is prepared +statement ok +PREPARE versioned_lookup AS +SELECT name FROM versioned.t AT (VERSION => concat('fir', 'st')) WHERE id = $1; + +query T +EXECUTE versioned_lookup(1) +---- +a + +query T +EXECUTE versioned_lookup(2) +---- + +statement ok +DEALLOCATE versioned_lookup; + +# Parameters are not supported in time travel clauses +statement error DataFusion error: This feature is not implemented: Parameters are not supported in time travel clauses: versioned\.t +PREPARE p AS SELECT * FROM versioned.t AT (VERSION => $1) + +# Clauses that depend on the time of execution would be evaluated when the +# statement is prepared instead +statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution, such as with now\(\): AT\(TIMESTAMP => now\(\) \- INTERVAL '1 hour'\) +PREPARE p AS SELECT * FROM versioned.t AT (TIMESTAMP => now() - INTERVAL '1 hour') + +# Also within subqueries and DML statements +statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution +PREPARE p AS SELECT * FROM (SELECT * FROM versioned.t AT (TIMESTAMP => now())) + +statement ok +CREATE TABLE copied (id BIGINT, name VARCHAR); + +statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution +PREPARE p AS INSERT INTO copied SELECT * FROM versioned.t AT (TIMESTAMP => now()) + +statement ok +DROP TABLE copied; + +########## +## Other dialects +########## + +statement ok +set datafusion.sql_parser.dialect = 'Databricks'; + +query IT rowsort +SELECT * FROM versioned.t VERSION AS OF 2 +---- +1 a +2 b + +query IT rowsort +SELECT * FROM versioned.t TIMESTAMP AS OF '2024-01-15' +---- +1 a + +statement ok +set datafusion.sql_parser.dialect = 'BigQuery'; + +query IT rowsort +SELECT * FROM versioned.t FOR SYSTEM_TIME AS OF '2024-02-15' +---- +1 a +2 b + +statement ok +set datafusion.sql_parser.dialect = 'Generic'; diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs index e6840ba4491b0..2308194ffac6d 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs @@ -4,7 +4,7 @@ use crate::logical_plan::producer::{SubstraitProducer, to_substrait_named_struct}; use datafusion::common::{ - DFSchema, ToDFSchema, not_impl_datafusion_err, substrait_datafusion_err, + DFSchema, ToDFSchema, not_impl_datafusion_err, not_impl_err, substrait_datafusion_err, }; use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values, lit}; @@ -50,6 +50,12 @@ pub fn from_table_scan( producer: &mut impl SubstraitProducer, scan: &TableScan, ) -> datafusion::common::Result> { + if let Some(version) = scan.source.version() { + return not_impl_err!( + "Time travel table scans are not supported in Substrait: {}@{version}", + scan.table_name + ); + } let skip = scan.skip.map(i64::try_from).transpose().map_err(|_| { not_impl_datafusion_err!("TableScan offset exceeds the supported Int64 range") })?; diff --git a/docs/source/library-user-guide/catalogs.md b/docs/source/library-user-guide/catalogs.md index 930d1e202bdaa..041e2907e66b8 100644 --- a/docs/source/library-user-guide/catalogs.md +++ b/docs/source/library-user-guide/catalogs.md @@ -203,6 +203,77 @@ impl SchemaProvider for Schema { } ``` +### Time travel + +Time travel queries, such as `SELECT * FROM t VERSION AS OF 3` or +`SELECT * FROM t TIMESTAMP AS OF '2024-01-01'`, read a table as it existed at an +earlier version or point in time. DataFusion evaluates the version expression +during planning and passes the resulting [`TableVersion`] to +`SchemaProvider::table_at`. A `TableVersion` is either a `Version`, holding any +`ScalarValue` such as a snapshot id or a tag, or a `Timestamp` in nanoseconds +since the Unix epoch (UTC). + +The default implementation of `table_at` returns a "not implemented" error. +Schema providers that support time travel return a table provider that reads the +requested version of the table. + +The returned provider must report the version it reads from +`TableProvider::version`. This may differ from the requested version: a tag or a +timestamp typically resolves to a snapshot id. The built-in providers, such as +`ListingTable` and `MemTable`, report the version set with `with_version`: + +```rust +# use async_trait::async_trait; +# use std::sync::Arc; +# use arrow::datatypes::DataType; +# use datafusion::catalog::{MemTable, TableProvider, SchemaProvider}; +# use datafusion::common::{Result, ScalarValue, TableVersion}; +# +# #[derive(Debug)] +# struct Schema {} +# +# impl Schema { +# fn snapshot(&self, _name: &str, _id: i64) -> Result> { todo!() } +# fn latest_snapshot_before(&self, _nanos: i64) -> Result { todo!() } +# } +# +#[async_trait] +impl SchemaProvider for Schema { + async fn table_at( + &self, + name: &str, + version: &TableVersion, + ) -> Result>> { + // Resolve the requested version to a snapshot id + let id = match version { + TableVersion::Version(value) => { + i64::try_from(value.cast_to(&DataType::Int64)?)? + } + TableVersion::Timestamp(nanos) => self.latest_snapshot_before(*nanos)?, + }; + let Some(table) = self.snapshot(name, id)? else { + return Ok(None); + }; + // Report the snapshot that is read + let table = table.with_version(Some(TableVersion::Version(ScalarValue::from(id)))); + Ok(Some(Arc::new(table))) + } + +# async fn table(&self, name: &str) -> Result>> { todo!() } +# fn table_names(&self) -> Vec { todo!() } +# fn table_exist(&self, _: &str) -> bool { todo!() } +} +``` + +EXPLAIN shows the version that each table scan reads. Scans of the same version +compare equal, however that version was requested: + +```text +TableScan: t@3 projection=[id, name] +``` + +[`tableversion`]: https://docs.rs/datafusion/latest/datafusion/common/enum.TableVersion.html + ## Implementing `MemoryCatalogProvider` As mentioned, the `CatalogProvider` can manage the schemas in a catalog, and the `MemoryCatalogProvider` is a simple implementation of the `CatalogProvider` trait. It stores schemas in a `DashMap`. With that the `CatalogProvider` trait can be implemented. diff --git a/docs/source/library-user-guide/upgrading/57.0.0.md b/docs/source/library-user-guide/upgrading/57.0.0.md index 4564a84c28a34..aee010e68e0a6 100644 --- a/docs/source/library-user-guide/upgrading/57.0.0.md +++ b/docs/source/library-user-guide/upgrading/57.0.0.md @@ -77,3 +77,26 @@ types (via `is::()` / `downcast_ref::()`), the The trait already required `'static`, which `Any` implies, so existing implementations need no changes. `ParquetSource::parquet_file_reader_factory` returns the trait object to downcast. + +### Time travel queries + +DataFusion now supports time travel queries such as +`SELECT * FROM t VERSION AS OF 3`. `SchemaProvider` gained a `table_at` method +that catalogs implement to return a specific version of a table, and +`TableProvider` and `TableSource` gained a `version` method that reports the +version a table reads. All have default implementations. See the +[catalog guide](../catalogs.md#time-travel) for details. + +**Who is affected:** + +- Implementations of `SchemaProvider::table_at` must return a provider whose + `TableProvider::version` reports the version it reads. `ListingTable`, + `MemTable`, `ViewTable` and `EmptyTable` support this with `with_version`. +- Custom `ContextProvider` implementations that implement + `get_table_source_at` must return a source whose `version` reports the + version it reads. +- Users serializing logical plans with `datafusion-proto`: the + `ListingTableScanNode`, `ViewTableScanNode`, `CustomTableScanNode` and + `EmptyTableScanNode` messages gained a `version` field. A + `LogicalExtensionCodec` must restore the version of the table providers it + decodes, or decoding fails. diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.md index 61d1da804ecb4..d25a6d8d0fa8b 100644 --- a/docs/source/user-guide/sql/select.md +++ b/docs/source/user-guide/sql/select.md @@ -204,7 +204,7 @@ GROUP BY customer_id; FROM from_item [, ...] from_item: - table_name [[AS] alias [(column_alias [, ...])]] + table_name [time_travel] [[AS] alias [(column_alias [, ...])]] | (query) [[AS] alias [(column_alias [, ...])]] | VALUES (expression [, ...]) [, ...] [[AS] alias [(column_alias [, ...])]] | table_function(argument [, ...]) [[AS] alias [(column_alias [, ...])]] @@ -289,6 +289,53 @@ unnest plan node carries a single null-handling mode for all its output columns, so a mix would be ambiguous. The planner returns an error in that case. +### Time travel + +A time travel clause reads a table as it existed at an earlier version or point +in time: + +```text +time_travel: + VERSION AS OF expression +| AT (VERSION => expression) +| TIMESTAMP AS OF expression +| FOR SYSTEM_TIME AS OF expression +| AT (TIMESTAMP => expression) +``` + +The expression is evaluated once during planning and must be a constant. It can +use functions such as `now()`, but cannot reference columns or parameters such as +`$1`. A version can be any value that the catalog understands, such as a snapshot +id or a tag. A timestamp is converted to a `TIMESTAMP` in the UTC time zone. + +In a prepared statement, the expression is evaluated when the statement is +prepared rather than when it is executed. It therefore cannot use functions that +depend on the time of execution, such as `now()`. + +```sql +SELECT * FROM orders VERSION AS OF 42; +SELECT * FROM orders AT (VERSION => 'release-1'); +SELECT * FROM orders TIMESTAMP AS OF '2024-01-01T00:00:00Z'; +SELECT * FROM orders AT (TIMESTAMP => now() - INTERVAL '1 day') AS yesterday; +``` + +Different versions of a table can be used in the same query: + +```sql +SELECT cur.id +FROM orders AS cur +JOIN orders AT (VERSION => 41) AS prev ON cur.id = prev.id +WHERE cur.status <> prev.status; +``` + +Time travel requires a catalog that supports it (see +[`SchemaProvider::table_at`]); the built-in in-memory and listing tables do not. +Time travel clauses are currently only parsed by the `Snowflake`, `Databricks`, +`BigQuery` and `MsSQL` dialects (see `datafusion.sql_parser.dialect`), and not +every dialect supports every syntax. + +[`schemaprovider::table_at`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html#method.table_at + ## WHERE clause ```text From 033d4b33d03210837ef0d5b846375daf7e13e2de Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 12 Nov 2026 17:33:42 +0100 Subject: [PATCH 2/3] Generalize table state selection --- datafusion/catalog-listing/src/table.rs | 26 +- .../catalog/src/default_table_source.rs | 6 +- .../catalog/src/dynamic_file/catalog.rs | 8 +- datafusion/catalog/src/empty.rs | 26 +- datafusion/catalog/src/memory/table.rs | 26 +- datafusion/catalog/src/view.rs | 26 +- datafusion/common/src/lib.rs | 4 +- datafusion/common/src/table_read_selector.rs | 110 ++++ datafusion/common/src/table_version.rs | 101 ---- datafusion/core/src/execution/context/mod.rs | 17 +- .../core/src/execution/session_state.rs | 508 ++++++++++++----- datafusion/core/tests/user_defined/mod.rs | 3 + .../tests/user_defined/relation_planner.rs | 90 +-- .../table_read_selector_planner.rs | 254 +++++++++ datafusion/expr/src/logical_plan/display.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 22 +- datafusion/expr/src/planner.rs | 187 +++++- datafusion/expr/src/table_source.rs | 22 +- .../proto-models/proto/datafusion.proto | 30 +- datafusion/proto-models/src/from_proto.rs | 22 +- .../proto-models/src/generated/pbjson.rs | 254 +++++---- .../proto-models/src/generated/prost.rs | 40 +- datafusion/proto-models/src/to_proto.rs | 23 +- datafusion/proto/src/logical_plan/mod.rs | 52 +- .../tests/cases/roundtrip_logical_plan.rs | 19 +- datafusion/session/src/schema.rs | 28 +- datafusion/session/src/table.rs | 37 +- datafusion/sql/src/relation/mod.rs | 390 ++++++++----- datafusion/sql/src/resolve.rs | 536 +++++++++++------- datafusion/sql/src/statement.rs | 2 +- datafusion/sql/src/unparser/dialect.rs | 6 +- datafusion/sql/src/unparser/plan.rs | 44 +- datafusion/sql/tests/cases/plan_to_sql.rs | 20 +- datafusion/sql/tests/common/mod.rs | 55 +- datafusion/sql/tests/sql_integration.rs | 30 +- .../src/test_context/time_travel.rs | 215 ++++++- .../sqllogictest/test_files/time_travel.slt | 213 ++++++- .../src/logical_plan/producer/rel/read_rel.rs | 4 +- docs/source/library-user-guide/catalogs.md | 74 ++- .../library-user-guide/upgrading/57.0.0.md | 38 +- docs/source/user-guide/sql/select.md | 8 +- 41 files changed, 2404 insertions(+), 1180 deletions(-) create mode 100644 datafusion/common/src/table_read_selector.rs delete mode 100644 datafusion/common/src/table_version.rs create mode 100644 datafusion/core/tests/user_defined/table_read_selector_planner.rs diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index e00e59ec494c2..3151ee803584e 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -13,7 +13,7 @@ use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::pruning::PrunableStatistics; use datafusion_common::stats::{Precision, is_known_empty}; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, TableVersion, + Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, TableReadSelector, internal_datafusion_err, plan_err, project_schema, }; use datafusion_datasource::file::FileSource; @@ -199,8 +199,7 @@ pub struct ListingTable { /// validation. Constant for the table, so computed once here instead of per /// file. file_schema_fingerprint: Arc, - /// The version of the table this provider reads, if versioned - version: Option, + read_selector: Option, } impl ListingTable { @@ -274,7 +273,7 @@ impl ListingTable { column_defaults: HashMap::new(), expr_adapter_factory: config.expr_adapter_factory, file_schema_fingerprint, - version: None, + read_selector: None, }; Ok(table) @@ -316,10 +315,17 @@ impl ListingTable { self } - /// Set the version of the table this provider reads, for time travel - /// queries. See [`TableProvider::version`]. - pub fn with_version(mut self, version: Option) -> Self { - self.version = version; + /// Sets the [`TableReadSelector`] reported by + /// [`TableProvider::read_selector`], which identifies which read of the + /// table this provider performs, such as a version of the table. + /// + /// This only labels the provider, it does not change the data it reads: + /// the provider must already be set up for that read. + pub fn with_read_selector( + mut self, + read_selector: Option, + ) -> Self { + self.read_selector = read_selector; self } @@ -544,8 +550,8 @@ impl TableProvider for ListingTable { Some(&self.constraints) } - fn version(&self) -> Option<&TableVersion> { - self.version.as_ref() + fn read_selector(&self) -> Option<&TableReadSelector> { + self.read_selector.as_ref() } fn table_type(&self) -> TableType { diff --git a/datafusion/catalog/src/default_table_source.rs b/datafusion/catalog/src/default_table_source.rs index f9117648f600d..50cf04dabb13f 100644 --- a/datafusion/catalog/src/default_table_source.rs +++ b/datafusion/catalog/src/default_table_source.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use crate::TableProvider; use arrow::datatypes::SchemaRef; -use datafusion_common::{Constraints, TableVersion, internal_err}; +use datafusion_common::{Constraints, TableReadSelector, internal_err}; use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource, TableType}; /// Implements [`TableSource`] for a [`TableProvider`] @@ -69,8 +69,8 @@ impl TableSource for DefaultTableSource { self.table_provider.get_column_default(column) } - fn version(&self) -> Option<&TableVersion> { - self.table_provider.version() + fn read_selector(&self) -> Option<&TableReadSelector> { + self.table_provider.read_selector() } } diff --git a/datafusion/catalog/src/dynamic_file/catalog.rs b/datafusion/catalog/src/dynamic_file/catalog.rs index 69f253017e916..eb8245f1b2744 100644 --- a/datafusion/catalog/src/dynamic_file/catalog.rs +++ b/datafusion/catalog/src/dynamic_file/catalog.rs @@ -6,7 +6,7 @@ use crate::{CatalogProvider, CatalogProviderList, SchemaProvider, TableProvider}; use async_trait::async_trait; -use datafusion_common::TableVersion; +use datafusion_common::TableReadSelector; use std::fmt::Debug; use std::sync::Arc; @@ -139,12 +139,12 @@ impl SchemaProvider for DynamicFileSchemaProvider { self.factory.try_new(name).await } - async fn table_at( + async fn table_with_read_selector( &self, name: &str, - version: &TableVersion, + selector: &TableReadSelector, ) -> datafusion_common::Result>> { - self.inner.table_at(name, version).await + self.inner.table_with_read_selector(name, selector).await } fn register_table( diff --git a/datafusion/catalog/src/empty.rs b/datafusion/catalog/src/empty.rs index 48a4bb2cb4562..ae00432f03e5c 100644 --- a/datafusion/catalog/src/empty.rs +++ b/datafusion/catalog/src/empty.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use arrow::datatypes::*; use async_trait::async_trait; -use datafusion_common::{Result, TableVersion, project_schema}; +use datafusion_common::{Result, TableReadSelector, project_schema}; use datafusion_expr::{Expr, TableType}; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::empty::EmptyExec; @@ -22,8 +22,7 @@ use crate::TableProvider; pub struct EmptyTable { schema: SchemaRef, partitions: usize, - /// The version of the table this provider reads, if versioned - version: Option, + read_selector: Option, } impl EmptyTable { @@ -32,14 +31,21 @@ impl EmptyTable { Self { schema, partitions: 1, - version: None, + read_selector: None, } } - /// Set the version of the table this provider reads, for time travel - /// queries. See [`TableProvider::version`]. - pub fn with_version(mut self, version: Option) -> Self { - self.version = version; + /// Sets the [`TableReadSelector`] reported by + /// [`TableProvider::read_selector`], which identifies which read of the + /// table this provider performs, such as a version of the table. + /// + /// This only labels the provider, it does not change the data it reads: + /// the provider must already be set up for that read. + pub fn with_read_selector( + mut self, + read_selector: Option, + ) -> Self { + self.read_selector = read_selector; self } @@ -60,8 +66,8 @@ impl TableProvider for EmptyTable { TableType::Base } - fn version(&self) -> Option<&TableVersion> { - self.version.as_ref() + fn read_selector(&self) -> Option<&TableReadSelector> { + self.read_selector.as_ref() } async fn scan( diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 3c9ae696693bb..bd96c6f803ff6 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -19,7 +19,7 @@ use arrow::record_batch::RecordBatch; use datafusion_common::error::Result; use datafusion_common::tree_node::TreeNodeRecursion; use datafusion_common::{ - Constraints, DFSchema, SchemaExt, TableVersion, internal_datafusion_err, + Constraints, DFSchema, SchemaExt, TableReadSelector, internal_datafusion_err, internal_err, not_impl_err, plan_err, }; use datafusion_datasource::memory::{MemSink, MemorySourceConfig}; @@ -70,8 +70,7 @@ pub struct MemTable { /// Optional pre-known sort order(s). Must be `SortExpr`s. /// inserting data into this table removes the order pub sort_order: Arc>>>, - /// The version of the table this provider reads, if versioned - version: Option, + read_selector: Option, } impl MemTable { @@ -105,14 +104,21 @@ impl MemTable { constraints: Constraints::default(), column_defaults: HashMap::new(), sort_order: Arc::new(Mutex::new(vec![])), - version: None, + read_selector: None, }) } - /// Set the version of the table this provider reads, for time travel - /// queries. See [`TableProvider::version`]. - pub fn with_version(mut self, version: Option) -> Self { - self.version = version; + /// Sets the [`TableReadSelector`] reported by + /// [`TableProvider::read_selector`], which identifies which read of the + /// table this provider performs, such as a version of the table. + /// + /// This only labels the provider, it does not change the data it reads: + /// the provider must already be set up for that read. + pub fn with_read_selector( + mut self, + read_selector: Option, + ) -> Self { + self.read_selector = read_selector; self } @@ -188,8 +194,8 @@ impl TableProvider for MemTable { Some(&self.constraints) } - fn version(&self) -> Option<&TableVersion> { - self.version.as_ref() + fn read_selector(&self) -> Option<&TableReadSelector> { + self.read_selector.as_ref() } fn table_type(&self) -> TableType { diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs index fa56dc3f4d098..54d1fd5599197 100644 --- a/datafusion/catalog/src/view.rs +++ b/datafusion/catalog/src/view.rs @@ -12,7 +12,7 @@ use crate::TableProvider; use arrow::datatypes::SchemaRef; use async_trait::async_trait; use datafusion_common::error::Result; -use datafusion_common::{Column, TableVersion}; +use datafusion_common::{Column, TableReadSelector}; use datafusion_expr::TableType; use datafusion_expr::{Expr, LogicalPlan}; use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown}; @@ -28,8 +28,7 @@ pub struct ViewTable { table_schema: SchemaRef, /// SQL used to create the view, if available definition: Option, - /// The version of the table this provider reads, if versioned - version: Option, + read_selector: Option, } impl ViewTable { @@ -46,14 +45,21 @@ impl ViewTable { logical_plan, table_schema, definition, - version: None, + read_selector: None, } } - /// Set the version of the table this provider reads, for time travel - /// queries. See [`TableProvider::version`]. - pub fn with_version(mut self, version: Option) -> Self { - self.version = version; + /// Sets the [`TableReadSelector`] reported by + /// [`TableProvider::read_selector`], which identifies which read of the + /// table this provider performs, such as a version of the table. + /// + /// This only labels the provider, it does not change the data it reads: + /// the provider must already be set up for that read. + pub fn with_read_selector( + mut self, + read_selector: Option, + ) -> Self { + self.read_selector = read_selector; self } @@ -82,8 +88,8 @@ impl TableProvider for ViewTable { TableType::View } - fn version(&self) -> Option<&TableVersion> { - self.version.as_ref() + fn read_selector(&self) -> Option<&TableReadSelector> { + self.read_selector.as_ref() } fn get_table_definition(&self) -> Option<&str> { diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 27d40fb47f0ab..0500e131913c6 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -19,8 +19,8 @@ mod join_type; mod param_value; mod partitioning; mod schema_reference; +mod table_read_selector; mod table_reference; -mod table_version; mod unnest; pub mod alias; @@ -90,8 +90,8 @@ pub use scalar::{ScalarType, ScalarValue}; pub use schema_reference::SchemaReference; pub use spans::{Location, Span, Spans}; pub use stats::{ColumnStatistics, Statistics}; +pub use table_read_selector::TableReadSelector; pub use table_reference::{ResolvedTableReference, TableReference}; -pub use table_version::TableVersion; pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; diff --git a/datafusion/common/src/table_read_selector.rs b/datafusion/common/src/table_read_selector.rs new file mode 100644 index 0000000000000..2d3af97e72e03 --- /dev/null +++ b/datafusion/common/src/table_read_selector.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::fmt::{Display, Formatter}; + +use arrow::temporal_conversions::timestamp_ns_to_datetime; +use chrono::SecondsFormat; + +use crate::ScalarValue; + +/// Selects which read of a table to perform, such as reading the table as of +/// a version or point in time for a time travel query. +/// +/// A selector typically picks a point in the history of a table, such as a +/// snapshot. It determines *which* data is read, not *how* it is read: hints +/// about how to scan a table do not belong in a selector. +/// +/// A selector plays two roles: +/// +/// * As a request, it is what a query asks for. It is passed to +/// [`SchemaProvider::table_with_read_selector`] to look up the table. +/// * As an identity, it is what a table provider actually reads, as reported +/// by [`TableProvider::read_selector`]. This may differ from the request: for +/// example, a [`TableReadSelector::AsOfTimestamp`] may resolve to the +/// [`TableReadSelector::AsOfVersion`] of the latest snapshot committed at +/// or before it. +/// +/// Scans of the same table with equal selectors are assumed to read the same +/// data. The schema of that data is determined by the table provider, so it +/// may differ between selectors, for example after the schema of the table +/// evolved. +/// +/// See [`SqlToRel::read_selector`] for how SQL time travel clauses are +/// mapped to selectors. +/// +/// [`SchemaProvider::table_with_read_selector`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html#method.table_with_read_selector +/// [`TableProvider::read_selector`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProvider.html#method.read_selector +/// [`SqlToRel::read_selector`]: https://docs.rs/datafusion/latest/datafusion/sql/planner/struct.SqlToRel.html#method.read_selector +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] +pub enum TableReadSelector { + /// The state of the table as of a version: a catalog specific version + /// identifier, such as a snapshot id, a version number, a tag or a branch + /// name. It may identify the state of the whole catalog, such as a commit + /// hash. + /// + /// As a request, a name such as a tag or a branch selects the version it + /// points to when the query is planned. As an identity, it is exactly the + /// version that is read. + AsOfVersion(ScalarValue), + /// The state of the table as of a point in time, in nanoseconds since the + /// Unix epoch (UTC). + /// + /// As a request, it selects the latest state committed at or before this + /// time, so a commit made exactly at this time is included. As an + /// identity, it is the state of the table at this time. + AsOfTimestamp(i64), +} + +impl Display for TableReadSelector { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + // Strings are quoted, to tell them apart from numbers and + // timestamps + TableReadSelector::AsOfVersion(v) => match v.try_as_str() { + Some(Some(s)) => write!(f, "{s:?}"), + _ => write!(f, "{v}"), + }, + TableReadSelector::AsOfTimestamp(ns) => match timestamp_ns_to_datetime(*ns) { + Some(dt) => write!( + f, + "{}", + dt.and_utc().to_rfc3339_opts(SecondsFormat::AutoSi, true) + ), + None => write!(f, "{ns}"), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + assert_eq!( + TableReadSelector::AsOfVersion(ScalarValue::Int64(Some(3))).to_string(), + "3" + ); + assert_eq!( + TableReadSelector::AsOfVersion(ScalarValue::from("v1")).to_string(), + "\"v1\"" + ); + assert_eq!( + TableReadSelector::AsOfTimestamp(1_704_067_200_000_000_000).to_string(), + "2024-01-01T00:00:00Z" + ); + assert_eq!( + TableReadSelector::AsOfVersion(ScalarValue::from("a \"quoted\" tag")) + .to_string(), + r#""a \"quoted\" tag""# + ); + assert_eq!( + TableReadSelector::AsOfVersion(ScalarValue::Utf8View(Some("3".into()))) + .to_string(), + "\"3\"" + ); + } +} diff --git a/datafusion/common/src/table_version.rs b/datafusion/common/src/table_version.rs deleted file mode 100644 index 9d1cc9e7ff8b4..0000000000000 --- a/datafusion/common/src/table_version.rs +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership -// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements - -use std::fmt::{Display, Formatter}; -use std::sync::Arc; - -use arrow::temporal_conversions::timestamp_ns_to_datetime; -use chrono::SecondsFormat; - -use crate::ScalarValue; - -/// A specific version of a table, as requested by a time travel query. -/// -/// Time travel queries read a table as it existed at some point in the past. -/// SQL dialects use a variety of syntaxes for this, all of which are -/// normalized to one of the two variants below. For example: -/// -/// | SQL | `TableVersion` | -/// |----------------------------------------------|----------------| -/// | `t VERSION AS OF 3` | `Version` | -/// | `t AT (VERSION => 'v1')` | `Version` | -/// | `t TIMESTAMP AS OF '2024-01-01'` | `Timestamp` | -/// | `t FOR SYSTEM_TIME AS OF now()` | `Timestamp` | -/// | `t AT (TIMESTAMP => now() - INTERVAL '1 h')` | `Timestamp` | -/// -/// The version expression is evaluated during planning, so catalogs only -/// ever see the resulting constant value. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] -pub enum TableVersion { - /// A catalog specific version identifier, such as a snapshot id, a - /// version number, a tag or a branch name. - Version(ScalarValue), - /// A point in time, in nanoseconds since the Unix epoch (UTC). - Timestamp(i64), -} - -impl TableVersion { - /// Returns the value of this version as a [`ScalarValue`]. - /// - /// For [`TableVersion::Timestamp`] this is a - /// [`ScalarValue::TimestampNanosecond`] in the UTC time zone. - pub fn value(&self) -> ScalarValue { - match self { - TableVersion::Version(v) => v.clone(), - TableVersion::Timestamp(ns) => { - ScalarValue::TimestampNanosecond(Some(*ns), Some(Arc::from("UTC"))) - } - } - } -} - -impl Display for TableVersion { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - // Strings are quoted, to tell them apart from numbers and - // timestamps - TableVersion::Version(v) => match v.try_as_str() { - Some(Some(s)) => write!(f, "{s:?}"), - _ => write!(f, "{v}"), - }, - TableVersion::Timestamp(ns) => match timestamp_ns_to_datetime(*ns) { - Some(dt) => write!( - f, - "{}", - dt.and_utc().to_rfc3339_opts(SecondsFormat::AutoSi, true) - ), - None => write!(f, "{ns}"), - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn display() { - assert_eq!( - TableVersion::Version(ScalarValue::Int64(Some(3))).to_string(), - "3" - ); - assert_eq!( - TableVersion::Version(ScalarValue::from("v1")).to_string(), - "\"v1\"" - ); - assert_eq!( - TableVersion::Timestamp(1_704_067_200_000_000_000).to_string(), - "2024-01-01T00:00:00Z" - ); - assert_eq!( - TableVersion::Version(ScalarValue::from("a \"quoted\" tag")).to_string(), - r#""a \"quoted\" tag""# - ); - assert_eq!( - TableVersion::Version(ScalarValue::Utf8View(Some("3".into()))).to_string(), - "\"3\"" - ); - } -} diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index b008542b47329..bcc8da04efa3b 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -75,7 +75,7 @@ use datafusion_execution::registry::SerializerRegistry; use datafusion_expr::HigherOrderUDF; pub use datafusion_expr::execution_props::ExecutionProps; #[cfg(feature = "sql")] -use datafusion_expr::planner::RelationPlanner; +use datafusion_expr::planner::{RelationPlanner, TableReadSelectorPlanner}; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::{ Expr, UserDefinedLogicalNode, WindowUDF, @@ -1749,6 +1749,21 @@ impl SessionContext { self.state.write().register_relation_planner(planner) } + #[cfg(feature = "sql")] + /// Registers a [`TableReadSelectorPlanner`] to customize which read of a + /// table a SQL table factor performs. + /// + /// Planners are invoked in reverse registration order, allowing newer + /// planners to take precedence over existing ones. + pub fn register_table_read_selector_planner( + &self, + planner: Arc, + ) -> Result<()> { + self.state + .write() + .register_table_read_selector_planner(planner) + } + /// Deregisters a UDF within this context. pub fn deregister_udf(&self, name: &str) { self.state.write().deregister_udf(name).ok(); diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bcaf99458a2a9..3ab7aa3b2153d 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -19,7 +19,11 @@ use crate::datasource::provider_as_source; use crate::execution::SessionStateDefaults; use crate::execution::context::{EmptySerializerRegistry, FunctionFactory, QueryPlanner}; use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}; -use arrow_schema::{DataType, FieldRef, TimeUnit}; +#[cfg(feature = "sql")] +use arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use arrow_schema::FieldRef; +#[cfg(feature = "sql")] +use arrow_schema::{DataType, Schema}; use datafusion_catalog::MemoryCatalogProviderList; use datafusion_catalog::information_schema::{ INFORMATION_SCHEMA, InformationSchemaProvider, @@ -32,9 +36,11 @@ use datafusion_common::config::{ConfigExtension, ConfigOptions, TableOptions}; use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan}; use datafusion_common::tree_node::TreeNode; use datafusion_common::{ - DFSchema, DataFusionError, ResolvedTableReference, ScalarValue, TableReference, - TableVersion, config_err, exec_err, internal_err, plan_datafusion_err, plan_err, + DFSchema, DataFusionError, ResolvedTableReference, TableReference, config_err, + exec_err, plan_datafusion_err, plan_err, }; +#[cfg(feature = "sql")] +use datafusion_common::{ScalarValue, TableReadSelector}; use datafusion_execution::TaskContext; use datafusion_execution::config::SessionConfig; use datafusion_execution::runtime_env::RuntimeEnv; @@ -43,16 +49,16 @@ use datafusion_expr::TableSource; use datafusion_expr::execution_props::ExecutionProps; use datafusion_expr::expr_rewriter::FunctionRewrite; use datafusion_expr::physical_planning_context::PhysicalPlanningContext; -use datafusion_expr::planner::{ExprPlanner, TableVersionExpr}; +use datafusion_expr::planner::ExprPlanner; #[cfg(feature = "sql")] -use datafusion_expr::planner::{RelationPlanner, TypePlanner}; +use datafusion_expr::planner::{RelationPlanner, TableReadSelectorPlanner, TypePlanner}; use datafusion_expr::registry::{ ExtensionTypeRegistryRef, FunctionRegistry, MemoryExtensionTypeRegistry, SerializerRegistry, }; use datafusion_expr::simplify::SimplifyContext; use datafusion_expr::{ - AggregateUDF, Explain, Expr, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, cast, + AggregateUDF, Explain, Expr, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, }; use datafusion_optimizer::simplify_expressions::ExprSimplifier; use datafusion_optimizer::{ @@ -70,6 +76,7 @@ use datafusion_session::{ use datafusion_sql::{ parser::{DFParserBuilder, Statement}, planner::{ContextProvider, ParserOptions, PlannerContext, SqlToRel}, + resolve::ResolvedReference, }; use async_trait::async_trait; @@ -150,6 +157,9 @@ struct SessionStateInner { expr_planners: Vec>, #[cfg(feature = "sql")] relation_planners: Vec>, + /// Determine which read of a table a SQL table factor performs + #[cfg(feature = "sql")] + table_read_selector_planners: Vec>, /// Provides support for customizing the SQL type planning #[cfg(feature = "sql")] type_planner: Option>, @@ -252,6 +262,11 @@ impl Debug for SessionState { #[cfg(feature = "sql")] let ret = ret.field("relation_planners", &self.inner.relation_planners); + #[cfg(feature = "sql")] + let ret = ret.field( + "table_read_selector_planners", + &self.inner.table_read_selector_planners, + ); #[cfg(feature = "sql")] let ret = ret.field("type_planner", &self.inner.type_planner); @@ -633,138 +648,210 @@ impl SessionState { } /// Convert an AST Statement into a LogicalPlan + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] + // TableReadSelector holds a ScalarValue, which clippy cannot prove immutable + // (nested values sit behind `Arc`). Arrow arrays are immutable, so + // the hash is stable. #[cfg(feature = "sql")] pub async fn statement_to_plan( &self, statement: Statement, ) -> datafusion_common::Result { - let references = datafusion_sql::resolve::resolve_table_references_with_versions( - &statement, + let table_references = self.extract_table_references(&statement)?; + + let tables = self.prefetch_tables(table_references).await?; + + let provider = SessionContextProvider { + state: self, + tables, + }; + let sql_to_rel = SqlToRel::new_with_options(&provider, self.get_parser_options()); + sql_to_rel.statement_to_plan(statement) + } + + #[cfg(feature = "sql")] + fn extract_table_references( + &self, + statement: &Statement, + ) -> Result)>, DataFusionError> { + let provider = SessionContextProvider { + state: self, + tables: HashMap::new(), + }; + let sql_to_rel = SqlToRel::new_with_options(&provider, self.get_parser_options()); + + let mut table_references = vec![]; + let handle_reference = |reference: ResolvedReference<'_>| { + match reference { + ResolvedReference::TableFactor(table, table_factor) => self + .add_read_selector( + &sql_to_rel, + table, + table_factor, + &mut table_references, + ), + ResolvedReference::Table(table) => table_references.push((table, None)), + // CTEs are not retrieved from the catalog. The tables their + // bodies reference are reported separately. + ResolvedReference::Cte(_) => {} + } + Ok(()) + }; + datafusion_sql::resolve::resolve_references( + statement, self.config() .options() .sql_parser .enable_ident_normalization, + handle_reference, )?; + Ok(table_references) + } - let mut provider = SessionContextProvider { - state: self, - tables: HashMap::with_capacity(references.tables.len()), - versioned_tables: HashMap::with_capacity(references.versioned_tables.len()), - }; - - // A `RelationPlanner` may handle a time travel clause itself, starting - // from the current version of the table, so also fetch that version. - // Without relation planners, the current version would go unused. - let mut tables = references.tables; - if !self.relation_planners().is_empty() { - tables.extend(references.versioned_tables.iter().map(|(t, _)| t.clone())); - } - - for reference in tables { - let resolved = self.resolve_table_ref(reference); - if let Entry::Vacant(v) = provider.tables.entry(resolved) { - let resolved = v.key(); - if let Ok(schema) = self.schema_for_ref(resolved.clone()) - && let Some(table) = schema.table(&resolved.table).await? - { - v.insert(provider_as_source(table)); + /// Determines the [`TableReadSelector`] of `table_factor`, which references + /// the table `reference`, ahead of planning, and adds the table to + /// `tables`, so it can be fetched from the catalog. + /// + /// Table factors whose selector cannot be determined are skipped: the SQL + /// planner reports the same error, unless a [`RelationPlanner`] handles + /// the table factor itself. + #[cfg(feature = "sql")] + fn add_read_selector( + &self, + sql_to_rel: &SqlToRel<'_, SessionContextProvider<'_>>, + reference: TableReference, + table_factor: &sqlparser::ast::TableFactor, + tables: &mut Vec<(TableReference, Option)>, + ) { + // A `RelationPlanner` may handle a table factor with a selector itself, + // starting from the table without a selector, so also fetch that + // table. Without relation planners, it would go unused. + let has_relation_planners = !self.relation_planners().is_empty(); + match sql_to_rel.read_selector(&reference, table_factor) { + Ok(None) => tables.push((reference, None)), + Ok(Some(selector)) => { + if has_relation_planners { + tables.push((reference.clone(), None)); } + tables.push((reference, Some(selector))); } + Err(_) if has_relation_planners => tables.push((reference, None)), + Err(_) => {} } + } - let versions = - self.evaluate_table_versions(&provider, references.versioned_tables); - for key in versions { - if let Entry::Vacant(v) = provider.versioned_tables.entry(key) { - let (resolved, version) = v.key(); - if let Ok(schema) = self.schema_for_ref(resolved.clone()) { - match schema.table_at(&resolved.table, version).await { - Ok(Some(table)) => { - v.insert(Ok(provider_as_source(table))); - } - Ok(None) => {} - Err(e) => { - v.insert(Err(Arc::new(e))); - } + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] + // TableReadSelector holds a ScalarValue, which clippy cannot prove immutable + // (nested values sit behind `Arc`). Arrow arrays are immutable, so + // the hash is stable. + #[cfg(feature = "sql")] + async fn prefetch_tables( + &self, + table_references: Vec<(TableReference, Option)>, + ) -> Result< + HashMap<(ResolvedTableReference, Option), TableSourceResult>, + DataFusionError, + > { + let mut tables = HashMap::with_capacity(table_references.len()); + for (reference, selector) in table_references { + let resolved = self.resolve_table_ref(reference); + if let Entry::Vacant(v) = tables.entry((resolved, selector)) { + let (resolved, selector) = v.key(); + let Ok(schema) = self.schema_for_ref(resolved.clone()) else { + continue; + }; + let Some(selector) = selector else { + if let Some(table) = schema.table(&resolved.table).await? { + v.insert(Ok(provider_as_source(table))); + } + continue; + }; + match table_with_read_selector(schema.as_ref(), resolved, selector).await + { + Ok(Some(table)) => { + v.insert(Ok(provider_as_source(table))); + } + Ok(None) => {} + Err(e) => { + v.insert(Err(Arc::new(e))); } } } } - - let query = SqlToRel::new_with_options(&provider, self.get_parser_options()); - query.statement_to_plan(statement) + Ok(tables) } - /// Evaluates the time travel clauses of `versioned_tables` ahead of - /// planning, so the referenced table versions can be fetched from the - /// catalog. + /// Evaluates a constant expression to a [`ScalarValue`]. /// - /// Clauses that fail to evaluate are skipped: a [`RelationPlanner`] may - /// handle them, and otherwise the SQL planner reports the same error. + /// An expression is constant if it has the same value for every row of a + /// query: it does not reference any columns or parameters, and does not + /// call volatile functions such as `random()`. Deterministic functions + /// and functions whose value is fixed for the duration of a query, such + /// as `now()`, are allowed. + /// + /// The result may be a null value. + /// + /// # Errors + /// + /// Returns an error if the expression is not constant, such as when it + /// references a column, or if evaluating it fails, such as for `1 / 0`. #[cfg(feature = "sql")] - fn evaluate_table_versions( - &self, - provider: &SessionContextProvider<'_>, - versioned_tables: Vec<(TableReference, sqlparser::ast::TableVersion)>, - ) -> Vec<(ResolvedTableReference, TableVersion)> { - let query = SqlToRel::new_with_options(provider, self.get_parser_options()); - versioned_tables - .into_iter() - .filter_map(|(reference, version)| { - let version = query - .sql_table_version_to_expr(version, &mut PlannerContext::new()) - .and_then(|version| self.evaluate_table_version(version)) - .ok()?; - Some((self.resolve_table_ref(reference), version)) - }) - .collect() + fn evaluate_const_expr(&self, expr: Expr) -> datafusion_common::Result { + let schema = DFSchema::empty(); + let simplifier = self.expr_simplifier(&schema); + let expr = self.coerce_and_rewrite(&simplifier, expr, &schema)?; + match simplifier.simplify(expr)? { + Expr::Literal(value, _) => Ok(value), + expr => { + // The simplifier leaves expressions that fail to evaluate, such + // as `1 / 0`, unchanged. Evaluate it to report why. + let batch = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(1)), + )?; + create_physical_expr( + &expr, + &schema, + self.execution_props(), + &PhysicalPlanningContext::default(), + )? + .evaluate(&batch)?; + plan_err!("Expression must evaluate to a constant, got: {expr}") + } + } } - /// Evaluates the expression of a time travel clause, such as - /// `VERSION AS OF `, to a constant [`TableVersion`]. - /// - /// The expression must not reference any columns. Functions such as - /// `now()` are evaluated using this session's query execution start time. - /// Timestamps are converted to nanoseconds in the UTC time zone. - pub fn evaluate_table_version( - &self, - version: TableVersionExpr, - ) -> datafusion_common::Result { + /// Returns an [`ExprSimplifier`] for expressions over `df_schema`. + fn expr_simplifier(&self, df_schema: &DFSchema) -> ExprSimplifier { let simplify_context = SimplifyContext::builder() + .with_schema(Arc::new(df_schema.clone())) .with_config_options(Arc::clone(self.config_options())) .with_query_execution_start_time( self.execution_props().query_execution_start_time, ) .build(); - let simplifier = ExprSimplifier::new(simplify_context); - let (expr, is_timestamp) = match version { - TableVersionExpr::Version(expr) => (expr, false), - TableVersionExpr::Timestamp(expr) => ( - cast( - expr, - DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), - ), - true, - ), - }; - let expr = simplifier.coerce(expr, &DFSchema::empty())?; - let value = match simplifier.simplify(expr)? { - Expr::Literal(value, _) if !value.is_null() => value, - expr => { - return plan_err!( - "Time travel version must evaluate to a non-null constant, got: {expr}" - ); - } - }; - match value { - ScalarValue::TimestampNanosecond(Some(ns), _) if is_timestamp => { - Ok(TableVersion::Timestamp(ns)) - } - value if !is_timestamp => Ok(TableVersion::Version(value)), - value => { - internal_err!("Expected a timestamp time travel version, got: {value:?}") - } + ExprSimplifier::new(simplify_context) + } + + /// Applies type coercion and this session's function rewrites to `expr`. + fn coerce_and_rewrite( + &self, + simplifier: &ExprSimplifier, + expr: Expr, + df_schema: &DFSchema, + ) -> datafusion_common::Result { + let config_options = self.config_options(); + // apply type coercion here to ensure types match + let mut expr = simplifier.coerce(expr, df_schema)?; + + // rewrite Exprs to functions if necessary + for rewrite in self.inner.analyzer.function_rewrites() { + expr = expr + .transform_up(|expr| rewrite.rewrite(expr, df_schema, config_options))? + .data; } + Ok(expr) } #[cfg(feature = "sql")] @@ -835,7 +922,6 @@ impl SessionState { let provider = SessionContextProvider { state: self, tables: HashMap::new(), - versioned_tables: HashMap::new(), }; let query = SqlToRel::new_with_options(&provider, self.get_parser_options()); @@ -877,6 +963,27 @@ impl SessionState { Ok(()) } + #[cfg(feature = "sql")] + /// Returns the registered table read selector planners in priority order. + pub fn table_read_selector_planners(&self) -> &[Arc] { + &self.inner.table_read_selector_planners + } + + #[cfg(feature = "sql")] + /// Registers a [`TableReadSelectorPlanner`] to customize which read of a + /// table a SQL table factor performs. + /// + /// Newly registered planners are given higher priority than existing ones. + pub fn register_table_read_selector_planner( + &mut self, + planner: Arc, + ) -> datafusion_common::Result<()> { + Arc::make_mut(&mut self.inner) + .table_read_selector_planners + .insert(0, planner); + Ok(()) + } + /// Returns the [`QueryPlanner`] for this session pub fn query_planner(&self) -> &Arc { &self.inner.query_planner @@ -1003,24 +1110,8 @@ impl SessionState { expr: Expr, df_schema: &DFSchema, ) -> datafusion_common::Result> { - let config_options = self.config_options(); - let simplify_context = SimplifyContext::builder() - .with_schema(Arc::new(df_schema.clone())) - .with_config_options(Arc::clone(config_options)) - .with_query_execution_start_time( - self.execution_props().query_execution_start_time, - ) - .build(); - let simplifier = ExprSimplifier::new(simplify_context); - // apply type coercion here to ensure types match - let mut expr = simplifier.coerce(expr, df_schema)?; - - // rewrite Exprs to functions if necessary - for rewrite in self.inner.analyzer.function_rewrites() { - expr = expr - .transform_up(|expr| rewrite.rewrite(expr, df_schema, config_options))? - .data; - } + let simplifier = self.expr_simplifier(df_schema); + let expr = self.coerce_and_rewrite(&simplifier, expr, df_schema)?; create_physical_expr( &expr, df_schema, @@ -1265,6 +1356,8 @@ pub struct SessionStateBuilder { #[cfg(feature = "sql")] relation_planners: Option>>, #[cfg(feature = "sql")] + table_read_selector_planners: Option>>, + #[cfg(feature = "sql")] type_planner: Option>, optimizer: Option, physical_optimizers: Option, @@ -1309,6 +1402,8 @@ impl SessionStateBuilder { #[cfg(feature = "sql")] relation_planners: None, #[cfg(feature = "sql")] + table_read_selector_planners: None, + #[cfg(feature = "sql")] type_planner: None, optimizer: None, physical_optimizers: None, @@ -1371,6 +1466,8 @@ impl SessionStateBuilder { #[cfg(feature = "sql")] relation_planners: Some(existing.relation_planners), #[cfg(feature = "sql")] + table_read_selector_planners: Some(existing.table_read_selector_planners), + #[cfg(feature = "sql")] type_planner: existing.type_planner, optimizer: Some(existing.optimizer), physical_optimizers: Some(existing.physical_optimizers), @@ -1536,6 +1633,17 @@ impl SessionStateBuilder { self } + #[cfg(feature = "sql")] + /// Sets the [`TableReadSelectorPlanner`]s used to determine which read of + /// a table a SQL table factor performs. + pub fn with_table_read_selector_planners( + mut self, + table_read_selector_planners: Vec>, + ) -> Self { + self.table_read_selector_planners = Some(table_read_selector_planners); + self + } + /// Set the [`TypePlanner`] used to customize the behavior of the SQL planner. #[cfg(feature = "sql")] pub fn with_type_planner(mut self, type_planner: Arc) -> Self { @@ -1810,6 +1918,8 @@ impl SessionStateBuilder { #[cfg(feature = "sql")] relation_planners, #[cfg(feature = "sql")] + table_read_selector_planners, + #[cfg(feature = "sql")] type_planner, optimizer, physical_optimizers, @@ -1847,6 +1957,9 @@ impl SessionStateBuilder { #[cfg(feature = "sql")] relation_planners: relation_planners.unwrap_or_default(), #[cfg(feature = "sql")] + table_read_selector_planners: table_read_selector_planners + .unwrap_or_default(), + #[cfg(feature = "sql")] type_planner, optimizer: optimizer.unwrap_or_default(), physical_optimizers: physical_optimizers.unwrap_or_default(), @@ -2036,6 +2149,15 @@ impl SessionStateBuilder { &mut self.relation_planners } + #[cfg(feature = "sql")] + /// Returns a mutable reference to the current [`TableReadSelectorPlanner`] + /// list. + pub fn table_read_selector_planners( + &mut self, + ) -> &mut Option>> { + &mut self.table_read_selector_planners + } + /// Returns the current type_planner value #[cfg(feature = "sql")] pub fn type_planner(&mut self) -> &mut Option> { @@ -2221,17 +2343,40 @@ impl From for SessionStateBuilder { #[cfg(feature = "sql")] struct SessionContextProvider<'a> { state: &'a SessionState, - tables: HashMap>, - /// Tables referenced with a time travel clause. Lookup errors are kept - /// and only reported if the planner actually needs the table, as a - /// [`RelationPlanner`] may handle the time travel clause itself. - versioned_tables: - HashMap<(ResolvedTableReference, TableVersion), VersionedTableSource>, + /// The tables referenced by the statement, with their selector. Errors + /// retrieving a table with a selector are kept and only reported if the + /// planner actually needs the table, as a [`RelationPlanner`] may handle + /// the selector itself. + tables: + HashMap<(ResolvedTableReference, Option), TableSourceResult>, } -/// The result of looking up a specific version of a table +/// The result of retrieving a table from the catalog #[cfg(feature = "sql")] -type VersionedTableSource = Result, Arc>; +type TableSourceResult = Result, Arc>; + +/// Retrieves the table `name` with `selector` from `schema`, see +/// [`SchemaProvider::table_with_read_selector`]. +/// +/// Returns an error if the provider does not report the selector it reads. +#[cfg(feature = "sql")] +async fn table_with_read_selector( + schema: &dyn SchemaProvider, + name: &ResolvedTableReference, + selector: &TableReadSelector, +) -> datafusion_common::Result>> { + let table = schema + .table_with_read_selector(&name.table, selector) + .await?; + if let Some(table) = &table + && table.read_selector().is_none() + { + return plan_err!( + "The table provider of '{name}@{selector}' must report the selector it reads" + ); + } + Ok(table) +} #[cfg(feature = "sql")] impl ContextProvider for SessionContextProvider<'_> { @@ -2243,6 +2388,10 @@ impl ContextProvider for SessionContextProvider<'_> { self.state.relation_planners() } + fn get_table_read_selector_planners(&self) -> &[Arc] { + self.state.table_read_selector_planners() + } + fn get_type_planner(&self) -> Option> { if let Some(type_planner) = &self.state.inner.type_planner { Some(Arc::clone(type_planner)) @@ -2255,30 +2404,28 @@ impl ContextProvider for SessionContextProvider<'_> { &self, name: TableReference, ) -> datafusion_common::Result> { - let name = self.state.resolve_table_ref(name); - self.tables - .get(&name) - .cloned() - .ok_or_else(|| plan_datafusion_err!("table '{name}' not found")) + let key = (self.state.resolve_table_ref(name), None); + match self.tables.get(&key) { + Some(Ok(table)) => Ok(Arc::clone(table)), + Some(Err(e)) => Err(DataFusionError::Shared(Arc::clone(e))), + None => plan_err!("table '{}' not found", key.0), + } } - fn evaluate_table_version( - &self, - version: TableVersionExpr, - ) -> datafusion_common::Result { - self.state.evaluate_table_version(version) + fn evaluate_const_expr(&self, expr: Expr) -> datafusion_common::Result { + self.state.evaluate_const_expr(expr) } - fn get_table_source_at( + fn get_table_source_with_read_selector( &self, name: TableReference, - version: &TableVersion, + selector: &TableReadSelector, ) -> datafusion_common::Result> { - let name = self.state.resolve_table_ref(name); - match self.versioned_tables.get(&(name.clone(), version.clone())) { + let key = (self.state.resolve_table_ref(name), Some(selector.clone())); + match self.tables.get(&key) { Some(Ok(table)) => Ok(Arc::clone(table)), Some(Err(e)) => Err(DataFusionError::Shared(Arc::clone(e))), - None => plan_err!("table '{name}@{version}' not found"), + None => plan_err!("table '{}@{selector}' not found", key.0), } } @@ -2747,7 +2894,6 @@ mod tests { let provider = SessionContextProvider { state, tables: HashMap::new(), - versioned_tables: HashMap::new(), }; let sql = "[1,2,3]"; @@ -2778,7 +2924,6 @@ mod tests { let provider = SessionContextProvider { state: &state, tables: HashMap::new(), - versioned_tables: HashMap::new(), }; let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); @@ -2933,6 +3078,69 @@ mod tests { Ok(()) } + // Expressions may evaluate to null; whether that is valid is up to the + // caller + #[test] + fn evaluate_const_expr_allows_null() -> Result<()> { + use datafusion_common::ScalarValue; + use datafusion_expr::{cast, lit}; + + let state = SessionStateBuilder::new().with_default_features().build(); + assert_eq!( + state.evaluate_const_expr(cast(lit(ScalarValue::Null), DataType::Int64))?, + ScalarValue::Int64(None) + ); + Ok(()) + } + + // Function rewrites apply when evaluating expressions to constants, as + // they do when planning queries + #[test] + fn evaluate_const_expr_applies_function_rewrites() -> Result<()> { + use datafusion_common::ScalarValue; + use datafusion_common::config::ConfigOptions; + use datafusion_common::tree_node::Transformed; + use datafusion_expr::expr_rewriter::FunctionRewrite; + use datafusion_expr::lit; + use datafusion_expr::registry::FunctionRegistry; + + /// Rewrites the string `'old'` to `'new'` + #[derive(Debug)] + struct RenameOld; + + impl FunctionRewrite for RenameOld { + fn name(&self) -> &str { + "rename_old" + } + + fn rewrite( + &self, + expr: Expr, + _schema: &DFSchema, + _config: &ConfigOptions, + ) -> Result> { + Ok(match expr { + Expr::Literal(ScalarValue::Utf8(Some(s)), _) if s == "old" => { + Transformed::yes(lit("new")) + } + expr => Transformed::no(expr), + }) + } + } + + let mut state = SessionStateBuilder::new().with_default_features().build(); + assert_eq!( + state.evaluate_const_expr(lit("old"))?, + ScalarValue::from("old") + ); + state.register_function_rewrite(Arc::new(RenameOld))?; + assert_eq!( + state.evaluate_const_expr(lit("old"))?, + ScalarValue::from("new") + ); + Ok(()) + } + /// This test demonstrates why it's more convenient and somewhat necessary to provide /// an `expr_planners` method for `SessionState`. #[tokio::test] diff --git a/datafusion/core/tests/user_defined/mod.rs b/datafusion/core/tests/user_defined/mod.rs index a39b36010cce8..997268b3f8849 100644 --- a/datafusion/core/tests/user_defined/mod.rs +++ b/datafusion/core/tests/user_defined/mod.rs @@ -26,6 +26,9 @@ mod expr_planner; /// Tests for Relation Planner extensions mod relation_planner; +/// Tests for Table Read Selector Planner extensions +mod table_read_selector_planner; + /// Tests for insert operations mod insert_operation; diff --git a/datafusion/core/tests/user_defined/relation_planner.rs b/datafusion/core/tests/user_defined/relation_planner.rs index fee216192c5e2..bf9a9a17db26c 100644 --- a/datafusion/core/tests/user_defined/relation_planner.rs +++ b/datafusion/core/tests/user_defined/relation_planner.rs @@ -5,26 +5,24 @@ //! Tests for the RelationPlanner extension point use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::Ordering; use arrow::array::{Int64Array, RecordBatch, StringArray}; use arrow::datatypes::{DataType, Field, Schema}; -use async_trait::async_trait; use datafusion::catalog::memory::MemTable; -use datafusion::catalog::{SchemaProvider, TableProvider}; use datafusion::common::test_util::batches_to_string; use datafusion::prelude::*; -use datafusion_common::{DFSchema, Result, ScalarValue, TableVersion, not_impl_err}; +use datafusion_common::{DFSchema, Result, ScalarValue}; use datafusion_expr::Expr; use datafusion_expr::logical_plan::builder::LogicalPlanBuilder; use datafusion_expr::planner::{ PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning, }; -use datafusion_sql::sqlparser::ast::{ - TableAlias, TableFactor, TableVersion as SQLTableVersion, -}; +use datafusion_sql::sqlparser::ast::{TableAlias, TableFactor, TableVersion}; use insta::assert_snapshot; +use super::table_read_selector_planner::register_counting_schema; + // ============================================================================ // Test Planners - Example Implementations // ============================================================================ @@ -342,51 +340,6 @@ fn create_context_with_catalog_table( ctx } -/// A schema with a single table `t`, which counts how often the current and -/// a specific version of `t` are looked up. -#[derive(Debug)] -struct CountingSchema { - table: Arc, - /// Whether `table_at` is implemented, or returns "not implemented" like - /// catalogs that predate it - supports_time_travel: bool, - table_calls: AtomicUsize, - table_at_calls: AtomicUsize, -} - -#[async_trait] -impl SchemaProvider for CountingSchema { - fn table_names(&self) -> Vec { - vec!["t".to_string()] - } - - async fn table(&self, name: &str) -> Result>> { - self.table_calls.fetch_add(1, Ordering::SeqCst); - Ok((name == "t").then(|| Arc::clone(&self.table))) - } - - async fn table_at( - &self, - name: &str, - version: &TableVersion, - ) -> Result>> { - self.table_at_calls.fetch_add(1, Ordering::SeqCst); - if !self.supports_time_travel { - return not_impl_err!("Time travel is not supported"); - } - if name != "t" { - return Ok(None); - } - let table = MemTable::try_new(self.table.schema(), vec![vec![]])? - .with_version(Some(version.clone())); - Ok(Some(Arc::new(table))) - } - - fn table_exist(&self, name: &str) -> bool { - name == "t" - } -} - /// Create a SessionContext with a simple single-column Int64 table. /// /// Useful for basic tests that need a real catalog table. @@ -417,10 +370,11 @@ fn create_context_with_simple_table( // ============================================================================ /// Example planner that implements time travel itself, for tables whose -/// catalog does not support [`SchemaProvider::table_at`]. It emulates version -/// `n` of a table by only returning the rows with an `id` of at most `n`. +/// catalog does not support [`SchemaProvider::table_with_read_selector`]. It +/// emulates version `n` of a table by only returning the rows with an `id` of +/// at most `n`. /// -/// [`SchemaProvider::table_at`]: datafusion::catalog::SchemaProvider::table_at +/// [`SchemaProvider::table_with_read_selector`]: datafusion::catalog::SchemaProvider::table_with_read_selector #[derive(Debug)] struct EmulatedVersionPlanner; @@ -434,7 +388,7 @@ impl RelationPlanner for EmulatedVersionPlanner { name, alias, args: None, - version: Some(SQLTableVersion::VersionAsOf(version)), + version: Some(TableVersion::VersionAsOf(version)), .. } = relation else { @@ -825,14 +779,14 @@ mod tests { let error = execute_sql(&ctx, "SELECT * FROM t VERSION AS OF 2") .await .expect_err("time travel should not be supported"); - assert_snapshot!(error.strip_backtrace(), @"This feature is not implemented: Time travel is not supported for table 't@2'"); + assert_snapshot!(error.strip_backtrace(), @"This feature is not implemented: Table read selectors are not supported for table 't@2'"); } // The current version of a table referenced with a time travel clause is // only fetched when a RelationPlanner may handle the time travel clause - // itself, whether or not the catalog implements `table_at`. + // itself, whether or not the catalog implements `table_with_read_selector`. #[tokio::test] - async fn time_travel_fetches_current_version_only_for_planners() { + async fn time_travel_fetches_current_version_only_for_relation_planners() { for (supports_time_travel, with_planner) in [(true, false), (true, true), (false, false), (false, true)] { @@ -847,17 +801,7 @@ mod tests { ctx.sql("SET datafusion.sql_parser.dialect = 'Databricks'") .await .unwrap(); - let table = ctx.table_provider("unused").await.unwrap(); - let schema = Arc::new(CountingSchema { - table, - supports_time_travel, - table_calls: AtomicUsize::new(0), - table_at_calls: AtomicUsize::new(0), - }); - ctx.catalog("datafusion") - .unwrap() - .register_schema("counting", Arc::clone(&schema) as _) - .unwrap(); + let schema = register_counting_schema(&ctx, supports_time_travel).await; let result = execute_sql(&ctx, "SELECT * FROM counting.t VERSION AS OF 1").await; @@ -873,7 +817,11 @@ mod tests { usize::from(with_planner), "{case}" ); - assert_eq!(schema.table_at_calls.load(Ordering::SeqCst), 1, "{case}"); + assert_eq!( + schema.table_with_read_selector_calls.load(Ordering::SeqCst), + 1, + "{case}" + ); } } diff --git a/datafusion/core/tests/user_defined/table_read_selector_planner.rs b/datafusion/core/tests/user_defined/table_read_selector_planner.rs new file mode 100644 index 0000000000000..93d15641e26aa --- /dev/null +++ b/datafusion/core/tests/user_defined/table_read_selector_planner.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Tests for the TableReadSelectorPlanner extension point + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arrow::array::{Int64Array, RecordBatch, StringArray}; +use arrow::datatypes::{DataType, Field, Schema}; +use async_trait::async_trait; +use datafusion::catalog::memory::MemTable; +use datafusion::catalog::{SchemaProvider, TableProvider}; +use datafusion::prelude::*; +use datafusion_common::{Result, TableReadSelector, TableReference, not_impl_err}; +use datafusion_expr::planner::{ + TableReadSelectorContext, TableReadSelectorPlanner, TableReadSelectorPlanning, +}; +use datafusion_sql::sqlparser::ast::TableFactor; +use insta::assert_snapshot; + +/// A schema with a single table `t`, which counts how often the current and +/// a specific version of `t` are looked up. +#[derive(Debug)] +pub(super) struct CountingSchema { + table: Arc, + /// Whether `table_with_read_selector` is implemented, or returns "not + /// implemented" like catalogs that predate it + supports_time_travel: bool, + pub(super) table_calls: AtomicUsize, + pub(super) table_with_read_selector_calls: AtomicUsize, +} + +#[async_trait] +impl SchemaProvider for CountingSchema { + fn table_names(&self) -> Vec { + vec!["t".to_string()] + } + + async fn table(&self, name: &str) -> Result>> { + self.table_calls.fetch_add(1, Ordering::SeqCst); + Ok((name == "t").then(|| Arc::clone(&self.table))) + } + + async fn table_with_read_selector( + &self, + name: &str, + selector: &TableReadSelector, + ) -> Result>> { + self.table_with_read_selector_calls + .fetch_add(1, Ordering::SeqCst); + if !self.supports_time_travel { + return not_impl_err!("Time travel is not supported"); + } + if name != "t" { + return Ok(None); + } + let table = MemTable::try_new(self.table.schema(), vec![vec![]])? + .with_read_selector(Some(selector.clone())); + Ok(Some(Arc::new(table))) + } + + fn table_exist(&self, name: &str) -> bool { + name == "t" + } +} + +/// Registers a [`CountingSchema`] named `counting` in `ctx`, with a table +/// `t` that has the schema of table `unused` +pub(super) async fn register_counting_schema( + ctx: &SessionContext, + supports_time_travel: bool, +) -> Arc { + let table = ctx.table_provider("unused").await.unwrap(); + let schema = Arc::new(CountingSchema { + table, + supports_time_travel, + table_calls: AtomicUsize::new(0), + table_with_read_selector_calls: AtomicUsize::new(0), + }); + ctx.catalog("datafusion") + .unwrap() + .register_schema("counting", Arc::clone(&schema) as _) + .unwrap(); + schema +} + +/// Creates a SessionContext with a table `unused`, with an Int64 column `id` +/// and a Utf8 column `name`, and the given table read selector planners +fn create_context(planners: Vec>) -> SessionContext { + let ctx = SessionContext::new(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["a", "b"])), + ], + ) + .unwrap(); + let table = MemTable::try_new(schema, vec![vec![batch]]).unwrap(); + ctx.register_table("unused", Arc::new(table)).unwrap(); + for planner in planners { + ctx.register_table_read_selector_planner(planner).unwrap(); + } + ctx +} + +/// Example planner that reads a specific version of a table in the `counting` +/// schema, given by a table hint such as `t WITH (1 + 1)`, rather than a time +/// travel clause. +#[derive(Debug)] +struct VersionHintPlanner; + +impl TableReadSelectorPlanner for VersionHintPlanner { + fn plan_read_selector( + &self, + table: &TableReference, + relation: &TableFactor, + context: &dyn TableReadSelectorContext, + ) -> Result { + if table.schema() != Some("counting") { + return Ok(TableReadSelectorPlanning::Original); + } + let TableFactor::Table { with_hints, .. } = relation else { + return Ok(TableReadSelectorPlanning::Original); + }; + let [hint] = with_hints.as_slice() else { + return Ok(TableReadSelectorPlanning::Original); + }; + let version = context.evaluate_const_expr(hint.clone())?; + Ok(TableReadSelectorPlanning::Planned(Some( + TableReadSelector::AsOfVersion(version), + ))) + } +} + +/// Example planner that ignores time travel clauses, reading the table as +/// returned by the catalog instead +#[derive(Debug)] +struct IgnoreTimeTravelPlanner; + +impl TableReadSelectorPlanner for IgnoreTimeTravelPlanner { + fn plan_read_selector( + &self, + _table: &TableReference, + _relation: &TableFactor, + _context: &dyn TableReadSelectorContext, + ) -> Result { + Ok(TableReadSelectorPlanning::Planned(None)) + } +} + +// A TableReadSelectorPlanner can determine the selector of a table from +// syntax other than a time travel clause +#[tokio::test] +async fn planner_determines_read_selector() { + let ctx = create_context(vec![Arc::new(VersionHintPlanner)]); + ctx.sql("SET datafusion.sql_parser.dialect = 'MsSQL'") + .await + .unwrap(); + let schema = register_counting_schema(&ctx, true).await; + + let plan = ctx + .sql("SELECT * FROM counting.t WITH (1 + 1)") + .await + .unwrap() + .into_unoptimized_plan(); + assert_snapshot!(plan, @r" + Projection: counting.t.id, counting.t.name + TableScan: counting.t@2 + "); + assert_eq!( + schema.table_with_read_selector_calls.load(Ordering::SeqCst), + 1 + ); + // Unlike for a `RelationPlanner`, the current version of the table is + // not fetched as well + assert_eq!(schema.table_calls.load(Ordering::SeqCst), 0); + + // The planner only handles tables in the `counting` schema, so the + // hint of other tables is ignored by DataFusion's default logic + let plan = ctx + .sql("SELECT * FROM unused WITH (1 + 1)") + .await + .unwrap() + .into_unoptimized_plan(); + assert_snapshot!(plan, @r" + Projection: unused.id, unused.name + TableScan: unused + "); + + // References to CTEs are read without a selector, so the planner is + // not asked for one + let plan = ctx + .sql("WITH c AS (SELECT 1 AS id) SELECT * FROM c WITH (1 + 1)") + .await + .unwrap() + .into_unoptimized_plan(); + assert_snapshot!(plan, @r" + Projection: c.id + SubqueryAlias: c + Projection: Int64(1) AS id + EmptyRelation: rows=1 + "); + + // Also not in prepared statements, where evaluating `now()` for a + // selector would otherwise be rejected + ctx.sql("PREPARE p AS WITH c AS (SELECT 1 AS id) SELECT * FROM c WITH (now())") + .await + .unwrap(); +} + +// A TableReadSelectorPlanner can read a table as returned by the catalog, +// regardless of its time travel clause. The first planner that determines +// the selector of a table takes precedence, and the planner registered +// last is asked first. +#[tokio::test] +async fn planner_ignores_time_travel() { + let ctx = create_context(vec![ + Arc::new(IgnoreTimeTravelPlanner), + Arc::new(VersionHintPlanner), + ]); + ctx.sql("SET datafusion.sql_parser.dialect = 'MsSQL'") + .await + .unwrap(); + let schema = register_counting_schema(&ctx, false).await; + + let plan = ctx + .sql("SELECT * FROM counting.t FOR SYSTEM_TIME AS OF '2024-01-01'") + .await + .unwrap() + .into_unoptimized_plan(); + assert_snapshot!(plan, @r" + Projection: counting.t.id, counting.t.name + TableScan: counting.t + "); + assert_eq!( + schema.table_with_read_selector_calls.load(Ordering::SeqCst), + 0 + ); + + // `VersionHintPlanner` takes precedence over `IgnoreTimeTravelPlanner`, + // so the catalog's lack of support for table read selectors is reported + let error = ctx + .sql("SELECT * FROM counting.t WITH (2)") + .await + .expect_err("table read selectors should not be supported"); + assert_snapshot!(error.strip_backtrace(), @"This feature is not implemented: Time travel is not supported"); +} diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs index 79cf0ba0ea4ec..eba3f088df83d 100644 --- a/datafusion/expr/src/logical_plan/display.rs +++ b/datafusion/expr/src/logical_plan/display.rs @@ -387,9 +387,11 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> { "Relation Name": table_name.table(), ); - if let Some(version) = source.version() { - object - .push("Version", serde_json::Value::String(version.to_string())); + if let Some(selector) = source.read_selector() { + object.push( + "Read Selector", + serde_json::Value::String(selector.to_string()), + ); } if let Some(s) = table_name.schema() { diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 2d356891f0bcc..48ba9c7cdeb7a 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -58,7 +58,7 @@ use datafusion_common::tree_node::{ use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Dependency, FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, - ScalarValue, Spans, SplitPoint, TableReference, TableVersion, UnnestOptions, + ScalarValue, Spans, SplitPoint, TableReadSelector, TableReference, UnnestOptions, aggregate_functional_dependencies_with_computed_inputs, assert_eq_or_internal_err, assert_or_internal_err, internal_err, plan_datafusion_err, plan_err, validate_range_split_points, @@ -2230,8 +2230,8 @@ impl LogicalPlan { }; write!(f, "TableScan: {table_name}")?; - if let Some(version) = source.version() { - write!(f, "@{version}")?; + if let Some(selector) = source.read_selector() { + write!(f, "@{selector}")?; } write!(f, "{projected_fields}")?; @@ -3295,7 +3295,7 @@ impl Debug for TableScan { fn fmt(&self, f: &mut Formatter) -> fmt::Result { f.debug_struct("TableScan") .field("table_name", &self.table_name) - .field("version", &self.source.version()) + .field("read_selector", &self.source.read_selector()) .field("source", &"...") .field("projection", &self.projection) .field("projected_schema", &self.projected_schema) @@ -3313,14 +3313,14 @@ impl PartialEq for TableScan { && self.projected_schema == other.projected_schema && self.filters == other.filters && self.fetch == other.fetch - && self.source.version() == other.source.version() + && self.source.read_selector() == other.source.read_selector() } } impl Eq for TableScan {} // Manual implementation needed because of `source` and `projected_schema` fields. -// Comparison excludes these fields, except for the version of the `source`. +// Comparison excludes these fields, except for the selector of the `source`. impl PartialOrd for TableScan { fn partial_cmp(&self, other: &Self) -> Option { #[derive(PartialEq, PartialOrd)] @@ -3333,22 +3333,22 @@ impl PartialOrd for TableScan { pub filters: &'a Vec, /// Optional number of rows to read pub fetch: &'a Option, - /// The version of the table to read - pub version: Option<&'a TableVersion>, + /// The selector of the table to read + pub read_selector: Option<&'a TableReadSelector>, } let comparable_self = ComparableTableScan { table_name: &self.table_name, projection: &self.projection, filters: &self.filters, fetch: &self.fetch, - version: self.source.version(), + read_selector: self.source.read_selector(), }; let comparable_other = ComparableTableScan { table_name: &other.table_name, projection: &other.projection, filters: &other.filters, fetch: &other.fetch, - version: other.source.version(), + read_selector: other.source.read_selector(), }; comparable_self .partial_cmp(&comparable_other) @@ -3364,7 +3364,7 @@ impl Hash for TableScan { self.projected_schema.hash(state); self.filters.hash(state); self.fetch.hash(state); - self.source.version().hash(state); + self.source.read_selector().hash(state); } } diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rs index fb793ccdcafd1..df407fe6ba8d2 100644 --- a/datafusion/expr/src/planner.rs +++ b/datafusion/expr/src/planner.rs @@ -17,8 +17,8 @@ use crate::{ use arrow::datatypes::{DataType, Field, FieldRef, SchemaRef}; use datafusion_common::datatype::DataTypeExt; use datafusion_common::{ - DFSchema, Result, TableReference, TableVersion, config::ConfigOptions, - file_options::file_type::FileType, not_impl_err, + DFSchema, Result, ScalarValue, TableReadSelector, TableReference, + config::ConfigOptions, file_options::file_type::FileType, not_impl_err, }; #[cfg(feature = "sql")] use sqlparser::ast::{Expr as SQLExpr, Ident, ObjectName, TableAlias, TableFactor}; @@ -32,27 +32,19 @@ pub trait ContextProvider { /// Returns a table by reference, if it exists fn get_table_source(&self, name: TableReference) -> Result>; - /// Evaluates the expression of a time travel clause, such as - /// `VERSION AS OF `, to a constant [`TableVersion`]. + /// Returns a table by reference and [`TableReadSelector`], if it exists. /// - /// The expression must not reference any columns, but may call functions - /// such as `now()`. - fn evaluate_table_version(&self, _version: TableVersionExpr) -> Result { - not_impl_err!("Time travel queries are not supported") - } - - /// Returns a specific version of a table by reference, if it exists. - /// - /// The returned source must report the version it reads from - /// [`TableSource::version`], otherwise planning fails. This may differ - /// from the requested `version`. See [`TableVersion`] for the time travel - /// syntaxes that resolve to this. - fn get_table_source_at( + /// The returned source must report the selector it reads from + /// [`TableSource::read_selector`], otherwise planning fails. This may differ + /// from the requested `selector`. + fn get_table_source_with_read_selector( &self, name: TableReference, - version: &TableVersion, + selector: &TableReadSelector, ) -> Result> { - not_impl_err!("Time travel is not supported for table '{name}@{version}'") + not_impl_err!( + "Table read selectors are not supported for table '{name}@{selector}'" + ) } /// Return the type of a file based on its extension (e.g. `.parquet`) @@ -97,6 +89,24 @@ pub trait ContextProvider { not_impl_err!("Recursive CTE is not implemented") } + /// Evaluates a constant expression to a [`ScalarValue`]. + /// + /// An expression is constant if it has the same value for every row of a + /// query: it does not reference any columns or parameters, and does not + /// call volatile functions such as `random()`. Deterministic functions + /// and functions whose value is fixed for the duration of a query, such + /// as `now()`, are allowed. + /// + /// The result may be a null value. + /// + /// # Errors + /// + /// Returns an error if the expression is not constant, such as when it + /// references a column, or if evaluating it fails, such as for `1 / 0`. + fn evaluate_const_expr(&self, expr: Expr) -> Result { + not_impl_err!("Evaluating constant expressions is not supported: {expr}") + } + /// Return [`ExprPlanner`] extensions for planning expressions fn get_expr_planners(&self) -> &[Arc] { &[] @@ -108,6 +118,13 @@ pub trait ContextProvider { &[] } + /// Return [`TableReadSelectorPlanner`] extensions for determining which + /// read of a table a table factor performs + #[cfg(feature = "sql")] + fn get_table_read_selector_planners(&self) -> &[Arc] { + &[] + } + /// Return [`TypePlanner`] extensions for planning data types #[cfg(feature = "sql")] fn get_type_planner(&self) -> Option> { @@ -158,18 +175,6 @@ pub trait ContextProvider { fn udwf_names(&self) -> Vec; } -/// The unevaluated form of a [`TableVersion`], as written in a time travel -/// clause such as `VERSION AS OF ` or `TIMESTAMP AS OF `. -/// -/// See [`ContextProvider::evaluate_table_version`]. -#[derive(Debug, Clone, PartialEq)] -pub enum TableVersionExpr { - /// Evaluates to a [`TableVersion::Version`] - Version(Expr), - /// Evaluates to a [`TableVersion::Timestamp`] - Timestamp(Expr), -} - /// Customize planning of SQL AST expressions to [`Expr`]s /// /// For more background, please also see the [Extending SQL in DataFusion: from ->> to TABLESAMPLE blog] @@ -454,6 +459,126 @@ pub trait RelationPlanner: Debug + Send + Sync { ) -> Result; } +/// Customize which read of a table a SQL table factor performs, for example to +/// support a table hint or a time travel clause that DataFusion does not map +/// to a [`TableReadSelector`] itself. +/// +/// Table read selectors are determined before the tables of a query are +/// retrieved from the catalog, so the catalog can return the table for the +/// selector. A [`RelationPlanner`] that plans a table factor itself, during +/// planning, can only use the table as it was retrieved. +/// +/// Planners are registered with [`SessionContext::register_table_read_selector_planner`]. +/// +/// # Example +/// +/// A planner that reads the version given by a table hint, such as +/// `SELECT * FROM t WITH (3)`: +/// +/// ``` +/// # use datafusion_common::{Result, TableReadSelector, TableReference}; +/// # use datafusion_expr::planner::{ +/// # TableReadSelectorContext, TableReadSelectorPlanner, TableReadSelectorPlanning, +/// # }; +/// # use sqlparser::ast::TableFactor; +/// #[derive(Debug)] +/// struct VersionHintPlanner; +/// +/// impl TableReadSelectorPlanner for VersionHintPlanner { +/// fn plan_read_selector( +/// &self, +/// _table: &TableReference, +/// relation: &TableFactor, +/// context: &dyn TableReadSelectorContext, +/// ) -> Result { +/// let TableFactor::Table { with_hints, .. } = relation else { +/// return Ok(TableReadSelectorPlanning::Original); +/// }; +/// let [hint] = with_hints.as_slice() else { +/// return Ok(TableReadSelectorPlanning::Original); +/// }; +/// let version = context.evaluate_const_expr(hint.clone())?; +/// Ok(TableReadSelectorPlanning::Planned(Some( +/// TableReadSelector::AsOfVersion(version), +/// ))) +/// } +/// } +/// ``` +/// +/// [`SessionContext::register_table_read_selector_planner`]: https://docs.rs/datafusion/latest/datafusion/execution/context/struct.SessionContext.html#method.register_table_read_selector_planner +#[cfg(feature = "sql")] +pub trait TableReadSelectorPlanner: Debug + Send + Sync { + /// Determine the [`TableReadSelector`] of a table factor, which specifies + /// which read of the table to perform, such as reading a specific + /// version. + /// + /// `table` is the table or view the table factor references, normalized + /// but not resolved against the default catalog and schema, so `t` and + /// `public.t` are different references. + /// + /// This is called for table factors that name a table or view, such as + /// `t VERSION AS OF 3`: before planning, to fetch the table from the + /// catalog, when the table factor is planned by DataFusion's default + /// logic, and when a statement is prepared, to check its selectors. As it + /// may be called more than once for the same table factor, it must + /// return the same result each time. It is not called for references to + /// CTEs, which are read without a selector, nor during planning for table + /// factors that a [`RelationPlanner`] plans itself. + /// + /// Returning [`TableReadSelectorPlanning::Planned`] short-circuits further + /// planning and uses the provided selector. `Planned(None)` reads the table + /// as returned by the catalog, without a selector. Returning + /// [`TableReadSelectorPlanning::Original`] allows the next registered planner, + /// or DataFusion's default logic, to determine the selector. The default + /// logic maps time travel clauses, such as `VERSION AS OF ` and + /// `TIMESTAMP AS OF `, to [`TableReadSelector::AsOfVersion`] and + /// [`TableReadSelector::AsOfTimestamp`]. + /// + /// The table is then retrieved with + /// [`ContextProvider::get_table_source_with_read_selector`]. + fn plan_read_selector( + &self, + table: &TableReference, + relation: &TableFactor, + context: &dyn TableReadSelectorContext, + ) -> Result; +} + +/// Result of determining the [`TableReadSelector`] of a table factor with +/// [`TableReadSelectorPlanner::plan_read_selector`] +#[cfg(feature = "sql")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TableReadSelectorPlanning { + /// The selector was determined by an extension planner. `None` reads the + /// table as returned by the catalog, without a selector. + Planned(Option), + /// The extension planner did not handle the table factor, so the next + /// registered planner, or DataFusion's default logic, determines the + /// selector. + Original, +} + +/// Provides utilities for [`TableReadSelectorPlanner::plan_read_selector`] +/// to interact with DataFusion's SQL planner. +/// +/// Selectors are determined before the tables of a query are retrieved from +/// the catalog, so unlike [`RelationPlannerContext`], this context cannot +/// plan relations or look up tables. +#[cfg(feature = "sql")] +pub trait TableReadSelectorContext { + /// Evaluates a constant SQL expression, such as the `1 + 1` of + /// `VERSION AS OF 1 + 1`, to a [`ScalarValue`]. See + /// [`ContextProvider::evaluate_const_expr`] for which expressions are + /// constant. The result may be null. + /// + /// Returns an error if the expression is not constant, or if evaluating + /// it fails. Parameters, such as `$1`, are not supported. In a prepared + /// statement, the expression must also not depend on the time of + /// execution, such as with `now()`: the selector is determined when the + /// statement is prepared, not when it is executed. + fn evaluate_const_expr(&self, expr: SQLExpr) -> Result; +} + /// Provides utilities for relation planners to interact with DataFusion's SQL /// planner. /// diff --git a/datafusion/expr/src/table_source.rs b/datafusion/expr/src/table_source.rs index f565acaa7924f..46c9343d2bfd2 100644 --- a/datafusion/expr/src/table_source.rs +++ b/datafusion/expr/src/table_source.rs @@ -7,7 +7,7 @@ use crate::{Expr, LogicalPlan}; use arrow::datatypes::SchemaRef; -use datafusion_common::{Constraints, Result, TableVersion}; +use datafusion_common::{Constraints, Result, TableReadSelector}; use std::{any::Any, borrow::Cow}; @@ -122,17 +122,19 @@ pub trait TableSource: Any + Sync + Send { None } - /// The version of the table this source reads, for time travel queries - /// such as `SELECT * FROM t VERSION AS OF 3`. `None` means the version is - /// unknown or the table is not versioned. + /// The [`TableReadSelector`] this source reads, such as a specific version of + /// the table. `None` means the source does not identify which read of + /// the table it performs. /// - /// Sources returned by [`ContextProvider::get_table_source_at`] must - /// report the version they read, which may differ from the requested - /// version. See [`TableProvider::version`]. + /// Sources returned by + /// [`ContextProvider::get_table_source_with_read_selector`] must report + /// the selector they read, which may differ from the requested selector. + /// Scans of the same table with equal selectors are assumed to read the + /// same data. See [`TableProvider::read_selector`]. /// - /// [`TableProvider::version`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.version - /// [`ContextProvider::get_table_source_at`]: crate::planner::ContextProvider::get_table_source_at - fn version(&self) -> Option<&TableVersion> { + /// [`TableProvider::read_selector`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.read_selector + /// [`ContextProvider::get_table_source_with_read_selector`]: crate::planner::ContextProvider::get_table_source_with_read_selector + fn read_selector(&self) -> Option<&TableReadSelector> { None } } diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index e87f2a5e9650d..b406daccc1130 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -93,8 +93,8 @@ message ListingTableScanNode { optional uint64 fetch = 17; // Optional number of rows to skip. optional uint64 skip = 18; - // The version of the table to read, for time travel queries. - TableVersion version = 19; + // The selector of the table to read, such as a specific version. + TableReadSelector read_selector = 19; } message ViewTableScanNode { @@ -108,8 +108,8 @@ message ViewTableScanNode { optional uint64 fetch = 7; // Optional number of rows to skip. optional uint64 skip = 8; - // The version of the table to read, for time travel queries. - TableVersion version = 9; + // The selector of the table to read, such as a specific version. + TableReadSelector read_selector = 9; } // Logical Plan to Scan a CustomTableProvider registered at runtime @@ -124,8 +124,8 @@ message CustomTableScanNode { optional uint64 fetch = 7; // Optional number of rows to skip. optional uint64 skip = 8; - // The version of the table to read, for time travel queries. - TableVersion version = 9; + // The selector of the table to read, such as a specific version. + TableReadSelector read_selector = 9; } message ProjectionNode { @@ -889,13 +889,15 @@ message TableReference { } } -// A version of a table, for time travel queries -message TableVersion { - oneof table_version_enum { - // A catalog specific version identifier, such as a snapshot id or a tag - datafusion_common.ScalarValue version = 1; +// Selects which read of a table to perform, such as reading the table as of a +// version or point in time for time travel queries +message TableReadSelector { + oneof table_read_selector_enum { + // A catalog specific version identifier, such as a snapshot id, a tag or + // a commit hash + datafusion_common.ScalarValue as_of_version = 1; // A point in time, in nanoseconds since the Unix epoch (UTC) - int64 timestamp = 2; + int64 as_of_timestamp = 2; } } @@ -1788,8 +1790,8 @@ message EmptyTableScanNode { datafusion_common.Schema schema = 2; ProjectionColumns projection = 3; repeated LogicalExprNode filters = 4; - // The version of the table to read, for time travel queries. - TableVersion version = 5; + // The selector of the table to read, such as a specific version. + TableReadSelector read_selector = 5; } enum GenerateSeriesName { diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs index 6f2d8dc752818..72aa69d685a4d 100644 --- a/datafusion/proto-models/src/from_proto.rs +++ b/datafusion/proto-models/src/from_proto.rs @@ -21,8 +21,8 @@ use datafusion_common::parquet_config::RowGroupRangeAssignment; use datafusion_common::parsers::{CompressionTypeVariant, CsvQuoteStyle}; use datafusion_common::utils::usize_from_wire; use datafusion_common::{ - JoinConstraint, JoinType, NullEquality, RecursionUnnestOption, TableReference, - TableVersion, UnnestOptions, + JoinConstraint, JoinType, NullEquality, RecursionUnnestOption, TableReadSelector, + TableReference, UnnestOptions, }; use datafusion_proto_common::FromProtoError as Error; @@ -99,20 +99,22 @@ impl TryFrom for TableReference { } } -impl TryFrom<&protobuf::TableVersion> for TableVersion { +impl TryFrom<&protobuf::TableReadSelector> for TableReadSelector { type Error = Error; - fn try_from(value: &protobuf::TableVersion) -> Result { - use protobuf::table_version::TableVersionEnum; + fn try_from(value: &protobuf::TableReadSelector) -> Result { + use protobuf::table_read_selector::TableReadSelectorEnum; match value - .table_version_enum + .table_read_selector_enum .as_ref() - .ok_or_else(|| Error::required("table_version_enum"))? + .ok_or_else(|| Error::required("table_read_selector_enum"))? { - TableVersionEnum::Version(value) => { - Ok(TableVersion::Version(value.try_into()?)) + TableReadSelectorEnum::AsOfVersion(value) => { + Ok(TableReadSelector::AsOfVersion(value.try_into()?)) + } + TableReadSelectorEnum::AsOfTimestamp(ns) => { + Ok(TableReadSelector::AsOfTimestamp(*ns)) } - TableVersionEnum::Timestamp(ns) => Ok(TableVersion::Timestamp(*ns)), } } } diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 4516ebd02598c..2382040e71102 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -5728,7 +5728,7 @@ impl serde::Serialize for CustomTableScanNode { if self.skip.is_some() { len += 1; } - if self.version.is_some() { + if self.read_selector.is_some() { len += 1; } let mut struct_ser = serializer.serialize_struct("datafusion.CustomTableScanNode", len)?; @@ -5759,8 +5759,8 @@ impl serde::Serialize for CustomTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } - if let Some(v) = self.version.as_ref() { - struct_ser.serialize_field("version", v)?; + if let Some(v) = self.read_selector.as_ref() { + struct_ser.serialize_field("readSelector", v)?; } struct_ser.end() } @@ -5781,7 +5781,8 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { "customTableData", "fetch", "skip", - "version", + "read_selector", + "readSelector", ]; #[allow(clippy::enum_variant_names)] @@ -5793,7 +5794,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { CustomTableData, Fetch, Skip, - Version, + ReadSelector, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -5822,7 +5823,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { "customTableData" | "custom_table_data" => Ok(GeneratedField::CustomTableData), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), - "version" => Ok(GeneratedField::Version), + "readSelector" | "read_selector" => Ok(GeneratedField::ReadSelector), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -5849,7 +5850,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { let mut custom_table_data__ = None; let mut fetch__ = None; let mut skip__ = None; - let mut version__ = None; + let mut read_selector__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -5900,11 +5901,11 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } - GeneratedField::Version => { - if version__.is_some() { - return Err(serde::de::Error::duplicate_field("version")); + GeneratedField::ReadSelector => { + if read_selector__.is_some() { + return Err(serde::de::Error::duplicate_field("readSelector")); } - version__ = map_.next_value()?; + read_selector__ = map_.next_value()?; } } } @@ -5916,7 +5917,7 @@ impl<'de> serde::Deserialize<'de> for CustomTableScanNode { custom_table_data: custom_table_data__.unwrap_or_default(), fetch: fetch__, skip: skip__, - version: version__, + read_selector: read_selector__, }) } } @@ -6848,7 +6849,7 @@ impl serde::Serialize for EmptyTableScanNode { if !self.filters.is_empty() { len += 1; } - if self.version.is_some() { + if self.read_selector.is_some() { len += 1; } let mut struct_ser = serializer.serialize_struct("datafusion.EmptyTableScanNode", len)?; @@ -6864,8 +6865,8 @@ impl serde::Serialize for EmptyTableScanNode { if !self.filters.is_empty() { struct_ser.serialize_field("filters", &self.filters)?; } - if let Some(v) = self.version.as_ref() { - struct_ser.serialize_field("version", v)?; + if let Some(v) = self.read_selector.as_ref() { + struct_ser.serialize_field("readSelector", v)?; } struct_ser.end() } @@ -6882,7 +6883,8 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { "schema", "projection", "filters", - "version", + "read_selector", + "readSelector", ]; #[allow(clippy::enum_variant_names)] @@ -6891,7 +6893,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { Schema, Projection, Filters, - Version, + ReadSelector, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -6917,7 +6919,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { "schema" => Ok(GeneratedField::Schema), "projection" => Ok(GeneratedField::Projection), "filters" => Ok(GeneratedField::Filters), - "version" => Ok(GeneratedField::Version), + "readSelector" | "read_selector" => Ok(GeneratedField::ReadSelector), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -6941,7 +6943,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { let mut schema__ = None; let mut projection__ = None; let mut filters__ = None; - let mut version__ = None; + let mut read_selector__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -6968,11 +6970,11 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { } filters__ = Some(map_.next_value()?); } - GeneratedField::Version => { - if version__.is_some() { - return Err(serde::de::Error::duplicate_field("version")); + GeneratedField::ReadSelector => { + if read_selector__.is_some() { + return Err(serde::de::Error::duplicate_field("readSelector")); } - version__ = map_.next_value()?; + read_selector__ = map_.next_value()?; } } } @@ -6981,7 +6983,7 @@ impl<'de> serde::Deserialize<'de> for EmptyTableScanNode { schema: schema__, projection: projection__, filters: filters__.unwrap_or_default(), - version: version__, + read_selector: read_selector__, }) } } @@ -13262,7 +13264,7 @@ impl serde::Serialize for ListingTableScanNode { if self.skip.is_some() { len += 1; } - if self.version.is_some() { + if self.read_selector.is_some() { len += 1; } if self.file_format_type.is_some() { @@ -13303,8 +13305,8 @@ impl serde::Serialize for ListingTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } - if let Some(v) = self.version.as_ref() { - struct_ser.serialize_field("version", v)?; + if let Some(v) = self.read_selector.as_ref() { + struct_ser.serialize_field("readSelector", v)?; } if let Some(v) = self.file_format_type.as_ref() { match v { @@ -13349,7 +13351,8 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "fileSortOrder", "fetch", "skip", - "version", + "read_selector", + "readSelector", "csv", "parquet", "avro", @@ -13369,7 +13372,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { FileSortOrder, Fetch, Skip, - Version, + ReadSelector, Csv, Parquet, Avro, @@ -13406,7 +13409,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { "fileSortOrder" | "file_sort_order" => Ok(GeneratedField::FileSortOrder), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), - "version" => Ok(GeneratedField::Version), + "readSelector" | "read_selector" => Ok(GeneratedField::ReadSelector), "csv" => Ok(GeneratedField::Csv), "parquet" => Ok(GeneratedField::Parquet), "avro" => Ok(GeneratedField::Avro), @@ -13441,7 +13444,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { let mut file_sort_order__ = None; let mut fetch__ = None; let mut skip__ = None; - let mut version__ = None; + let mut read_selector__ = None; let mut file_format_type__ = None; while let Some(k) = map_.next_key()? { match k { @@ -13509,11 +13512,11 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } - GeneratedField::Version => { - if version__.is_some() { - return Err(serde::de::Error::duplicate_field("version")); + GeneratedField::ReadSelector => { + if read_selector__.is_some() { + return Err(serde::de::Error::duplicate_field("readSelector")); } - version__ = map_.next_value()?; + read_selector__ = map_.next_value()?; } GeneratedField::Csv => { if file_format_type__.is_some() { @@ -13563,7 +13566,7 @@ impl<'de> serde::Deserialize<'de> for ListingTableScanNode { file_sort_order: file_sort_order__.unwrap_or_default(), fetch: fetch__, skip: skip__, - version: version__, + read_selector: read_selector__, file_format_type: file_format_type__, }) } @@ -28870,7 +28873,7 @@ impl<'de> serde::Deserialize<'de> for SymmetricHashJoinExecNode { deserializer.deserialize_struct("datafusion.SymmetricHashJoinExecNode", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for TableReference { +impl serde::Serialize for TableReadSelector { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -28878,43 +28881,42 @@ impl serde::Serialize for TableReference { { use serde::ser::SerializeStruct; let mut len = 0; - if self.table_reference_enum.is_some() { + if self.table_read_selector_enum.is_some() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.TableReference", len)?; - if let Some(v) = self.table_reference_enum.as_ref() { + let mut struct_ser = serializer.serialize_struct("datafusion.TableReadSelector", len)?; + if let Some(v) = self.table_read_selector_enum.as_ref() { match v { - table_reference::TableReferenceEnum::Bare(v) => { - struct_ser.serialize_field("bare", v)?; + table_read_selector::TableReadSelectorEnum::AsOfVersion(v) => { + struct_ser.serialize_field("asOfVersion", v)?; } - table_reference::TableReferenceEnum::Partial(v) => { - struct_ser.serialize_field("partial", v)?; - } - table_reference::TableReferenceEnum::Full(v) => { - struct_ser.serialize_field("full", v)?; + table_read_selector::TableReadSelectorEnum::AsOfTimestamp(v) => { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("asOfTimestamp", ToString::to_string(&v).as_str())?; } } } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for TableReference { +impl<'de> serde::Deserialize<'de> for TableReadSelector { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "bare", - "partial", - "full", + "as_of_version", + "asOfVersion", + "as_of_timestamp", + "asOfTimestamp", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Bare, - Partial, - Full, + AsOfVersion, + AsOfTimestamp, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -28936,9 +28938,8 @@ impl<'de> serde::Deserialize<'de> for TableReference { E: serde::de::Error, { match value { - "bare" => Ok(GeneratedField::Bare), - "partial" => Ok(GeneratedField::Partial), - "full" => Ok(GeneratedField::Full), + "asOfVersion" | "as_of_version" => Ok(GeneratedField::AsOfVersion), + "asOfTimestamp" | "as_of_timestamp" => Ok(GeneratedField::AsOfTimestamp), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -28948,51 +28949,43 @@ impl<'de> serde::Deserialize<'de> for TableReference { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = TableReference; + type Value = TableReadSelector; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.TableReference") + formatter.write_str("struct datafusion.TableReadSelector") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut table_reference_enum__ = None; + let mut table_read_selector_enum__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Bare => { - if table_reference_enum__.is_some() { - return Err(serde::de::Error::duplicate_field("bare")); + GeneratedField::AsOfVersion => { + if table_read_selector_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("asOfVersion")); } - table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Bare) + table_read_selector_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_read_selector::TableReadSelectorEnum::AsOfVersion) ; } - GeneratedField::Partial => { - if table_reference_enum__.is_some() { - return Err(serde::de::Error::duplicate_field("partial")); + GeneratedField::AsOfTimestamp => { + if table_read_selector_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("asOfTimestamp")); } - table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Partial) -; - } - GeneratedField::Full => { - if table_reference_enum__.is_some() { - return Err(serde::de::Error::duplicate_field("full")); - } - table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Full) -; + table_read_selector_enum__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| table_read_selector::TableReadSelectorEnum::AsOfTimestamp(x.0)); } } } - Ok(TableReference { - table_reference_enum: table_reference_enum__, + Ok(TableReadSelector { + table_read_selector_enum: table_read_selector_enum__, }) } } - deserializer.deserialize_struct("datafusion.TableReference", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.TableReadSelector", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for TableVersion { +impl serde::Serialize for TableReference { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -29000,40 +28993,43 @@ impl serde::Serialize for TableVersion { { use serde::ser::SerializeStruct; let mut len = 0; - if self.table_version_enum.is_some() { + if self.table_reference_enum.is_some() { len += 1; } - let mut struct_ser = serializer.serialize_struct("datafusion.TableVersion", len)?; - if let Some(v) = self.table_version_enum.as_ref() { + let mut struct_ser = serializer.serialize_struct("datafusion.TableReference", len)?; + if let Some(v) = self.table_reference_enum.as_ref() { match v { - table_version::TableVersionEnum::Version(v) => { - struct_ser.serialize_field("version", v)?; + table_reference::TableReferenceEnum::Bare(v) => { + struct_ser.serialize_field("bare", v)?; } - table_version::TableVersionEnum::Timestamp(v) => { - #[allow(clippy::needless_borrow)] - #[allow(clippy::needless_borrows_for_generic_args)] - struct_ser.serialize_field("timestamp", ToString::to_string(&v).as_str())?; + table_reference::TableReferenceEnum::Partial(v) => { + struct_ser.serialize_field("partial", v)?; + } + table_reference::TableReferenceEnum::Full(v) => { + struct_ser.serialize_field("full", v)?; } } } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for TableVersion { +impl<'de> serde::Deserialize<'de> for TableReference { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "version", - "timestamp", + "bare", + "partial", + "full", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Version, - Timestamp, + Bare, + Partial, + Full, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -29055,8 +29051,9 @@ impl<'de> serde::Deserialize<'de> for TableVersion { E: serde::de::Error, { match value { - "version" => Ok(GeneratedField::Version), - "timestamp" => Ok(GeneratedField::Timestamp), + "bare" => Ok(GeneratedField::Bare), + "partial" => Ok(GeneratedField::Partial), + "full" => Ok(GeneratedField::Full), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -29066,40 +29063,48 @@ impl<'de> serde::Deserialize<'de> for TableVersion { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = TableVersion; + type Value = TableReference; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct datafusion.TableVersion") + formatter.write_str("struct datafusion.TableReference") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut table_version_enum__ = None; + let mut table_reference_enum__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Version => { - if table_version_enum__.is_some() { - return Err(serde::de::Error::duplicate_field("version")); + GeneratedField::Bare => { + if table_reference_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("bare")); } - table_version_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_version::TableVersionEnum::Version) + table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Bare) ; } - GeneratedField::Timestamp => { - if table_version_enum__.is_some() { - return Err(serde::de::Error::duplicate_field("timestamp")); + GeneratedField::Partial => { + if table_reference_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("partial")); } - table_version_enum__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| table_version::TableVersionEnum::Timestamp(x.0)); + table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Partial) +; + } + GeneratedField::Full => { + if table_reference_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("full")); + } + table_reference_enum__ = map_.next_value::<::std::option::Option<_>>()?.map(table_reference::TableReferenceEnum::Full) +; } } } - Ok(TableVersion { - table_version_enum: table_version_enum__, + Ok(TableReference { + table_reference_enum: table_reference_enum__, }) } } - deserializer.deserialize_struct("datafusion.TableVersion", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("datafusion.TableReference", FIELDS, GeneratedVisitor) } } impl serde::Serialize for TryCastNode { @@ -30323,7 +30328,7 @@ impl serde::Serialize for ViewTableScanNode { if self.skip.is_some() { len += 1; } - if self.version.is_some() { + if self.read_selector.is_some() { len += 1; } let mut struct_ser = serializer.serialize_struct("datafusion.ViewTableScanNode", len)?; @@ -30352,8 +30357,8 @@ impl serde::Serialize for ViewTableScanNode { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("skip", ToString::to_string(&v).as_str())?; } - if let Some(v) = self.version.as_ref() { - struct_ser.serialize_field("version", v)?; + if let Some(v) = self.read_selector.as_ref() { + struct_ser.serialize_field("readSelector", v)?; } struct_ser.end() } @@ -30373,7 +30378,8 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { "definition", "fetch", "skip", - "version", + "read_selector", + "readSelector", ]; #[allow(clippy::enum_variant_names)] @@ -30385,7 +30391,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { Definition, Fetch, Skip, - Version, + ReadSelector, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -30414,7 +30420,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { "definition" => Ok(GeneratedField::Definition), "fetch" => Ok(GeneratedField::Fetch), "skip" => Ok(GeneratedField::Skip), - "version" => Ok(GeneratedField::Version), + "readSelector" | "read_selector" => Ok(GeneratedField::ReadSelector), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -30441,7 +30447,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { let mut definition__ = None; let mut fetch__ = None; let mut skip__ = None; - let mut version__ = None; + let mut read_selector__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::TableName => { @@ -30490,11 +30496,11 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0) ; } - GeneratedField::Version => { - if version__.is_some() { - return Err(serde::de::Error::duplicate_field("version")); + GeneratedField::ReadSelector => { + if read_selector__.is_some() { + return Err(serde::de::Error::duplicate_field("readSelector")); } - version__ = map_.next_value()?; + read_selector__ = map_.next_value()?; } } } @@ -30506,7 +30512,7 @@ impl<'de> serde::Deserialize<'de> for ViewTableScanNode { definition: definition__.unwrap_or_default(), fetch: fetch__, skip: skip__, - version: version__, + read_selector: read_selector__, }) } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 20ceaa431b5be..cf2cbbd4aaea6 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -129,9 +129,9 @@ pub struct ListingTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "18")] pub skip: ::core::option::Option, - /// The version of the table to read, for time travel queries. + /// The selector of the table to read, such as a specific version. #[prost(message, optional, tag = "19")] - pub version: ::core::option::Option, + pub read_selector: ::core::option::Option, #[prost( oneof = "listing_table_scan_node::FileFormatType", tags = "10, 11, 12, 15, 16" @@ -174,9 +174,9 @@ pub struct ViewTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "8")] pub skip: ::core::option::Option, - /// The version of the table to read, for time travel queries. + /// The selector of the table to read, such as a specific version. #[prost(message, optional, tag = "9")] - pub version: ::core::option::Option, + pub read_selector: ::core::option::Option, } /// Logical Plan to Scan a CustomTableProvider registered at runtime #[derive(Clone, PartialEq, ::prost::Message)] @@ -197,9 +197,9 @@ pub struct CustomTableScanNode { /// Optional number of rows to skip. #[prost(uint64, optional, tag = "8")] pub skip: ::core::option::Option, - /// The version of the table to read, for time travel queries. + /// The selector of the table to read, such as a specific version. #[prost(message, optional, tag = "9")] - pub version: ::core::option::Option, + pub read_selector: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProjectionNode { @@ -1401,22 +1401,26 @@ pub mod table_reference { Full(super::FullTableReference), } } -/// A version of a table, for time travel queries +/// Selects which read of a table to perform, such as reading the table as of a +/// version or point in time for time travel queries #[derive(Clone, PartialEq, ::prost::Message)] -pub struct TableVersion { - #[prost(oneof = "table_version::TableVersionEnum", tags = "1, 2")] - pub table_version_enum: ::core::option::Option, +pub struct TableReadSelector { + #[prost(oneof = "table_read_selector::TableReadSelectorEnum", tags = "1, 2")] + pub table_read_selector_enum: ::core::option::Option< + table_read_selector::TableReadSelectorEnum, + >, } -/// Nested message and enum types in `TableVersion`. -pub mod table_version { +/// Nested message and enum types in `TableReadSelector`. +pub mod table_read_selector { #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum TableVersionEnum { - /// A catalog specific version identifier, such as a snapshot id or a tag + pub enum TableReadSelectorEnum { + /// A catalog specific version identifier, such as a snapshot id, a tag or + /// a commit hash #[prost(message, tag = "1")] - Version(super::super::datafusion_common::ScalarValue), + AsOfVersion(super::super::datafusion_common::ScalarValue), /// A point in time, in nanoseconds since the Unix epoch (UTC) #[prost(int64, tag = "2")] - Timestamp(i64), + AsOfTimestamp(i64), } } /// PhysicalPlanNode is a nested type @@ -2705,9 +2709,9 @@ pub struct EmptyTableScanNode { pub projection: ::core::option::Option, #[prost(message, repeated, tag = "4")] pub filters: ::prost::alloc::vec::Vec, - /// The version of the table to read, for time travel queries. + /// The selector of the table to read, such as a specific version. #[prost(message, optional, tag = "5")] - pub version: ::core::option::Option, + pub read_selector: ::core::option::Option, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GenerateSeriesArgsContainsNull { diff --git a/datafusion/proto-models/src/to_proto.rs b/datafusion/proto-models/src/to_proto.rs index 11caab75ff5af..8cf549900dcdb 100644 --- a/datafusion/proto-models/src/to_proto.rs +++ b/datafusion/proto-models/src/to_proto.rs @@ -11,7 +11,8 @@ use datafusion_common::DataFusionError; use datafusion_common::display::{PlanType, StringifiedPlan}; use datafusion_common::{ - JoinConstraint, JoinType, NullEquality, TableReference, TableVersion, UnnestOptions, + JoinConstraint, JoinType, NullEquality, TableReadSelector, TableReference, + UnnestOptions, }; use crate::generated::datafusion_common::EmptyMessage; @@ -114,17 +115,21 @@ impl From<&StringifiedPlan> for protobuf::StringifiedPlan { } } -impl TryFrom<&TableVersion> for protobuf::TableVersion { +impl TryFrom<&TableReadSelector> for protobuf::TableReadSelector { type Error = datafusion_proto_common::ToProtoError; - fn try_from(version: &TableVersion) -> Result { - use protobuf::table_version::TableVersionEnum; - let table_version_enum = match version { - TableVersion::Version(value) => TableVersionEnum::Version(value.try_into()?), - TableVersion::Timestamp(ns) => TableVersionEnum::Timestamp(*ns), + fn try_from(selector: &TableReadSelector) -> Result { + use protobuf::table_read_selector::TableReadSelectorEnum; + let table_read_selector_enum = match selector { + TableReadSelector::AsOfVersion(value) => { + TableReadSelectorEnum::AsOfVersion(value.try_into()?) + } + TableReadSelector::AsOfTimestamp(ns) => { + TableReadSelectorEnum::AsOfTimestamp(*ns) + } }; - Ok(protobuf::TableVersion { - table_version_enum: Some(table_version_enum), + Ok(protobuf::TableReadSelector { + table_read_selector_enum: Some(table_read_selector_enum), }) } } diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index a4fe178b69190..06496c9469943 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -28,8 +28,8 @@ use datafusion_common::format::{ }; use datafusion_common::utils::{usize_from_wire, usize_to_wire}; use datafusion_common::{ - NullEquality, Result, TableReference, TableVersion, assert_or_internal_err, context, - internal_datafusion_err, internal_err, not_impl_err, plan_err, + NullEquality, Result, TableReadSelector, TableReference, assert_or_internal_err, + context, internal_datafusion_err, internal_err, not_impl_err, plan_err, }; use datafusion_datasource::file_format::FileFormat; use datafusion_datasource::file_format::{ @@ -360,10 +360,10 @@ fn from_table_reference( Ok(TableReference::try_from(table_ref.clone())?) } -fn from_table_version( - version: Option<&protobuf::TableVersion>, -) -> Result> { - Ok(version.map(TableVersion::try_from).transpose()?) +fn from_read_selector( + selector: Option<&protobuf::TableReadSelector>, +) -> Result> { + Ok(selector.map(TableReadSelector::try_from).transpose()?) } /// Converts [LogicalPlan::TableScan] to [TableSource] @@ -682,7 +682,7 @@ impl AsLogicalPlan for LogicalPlanNode { .with_cache( ctx.runtime_env().cache_manager.get_file_statistic_cache(), ) - .with_version(from_table_version(scan.version.as_ref())?); + .with_read_selector(from_read_selector(scan.read_selector.as_ref())?); let table_name = from_table_reference(scan.table_name.as_ref(), "ListingTableScan")?; @@ -733,15 +733,15 @@ impl AsLogicalPlan for LogicalPlanNode { schema, ctx, )?; - // The codec restores the version as part of the provider's - // state. Fail rather than silently reading another version. - let version = from_table_version(scan.version.as_ref())?; - if provider.version() != version.as_ref() { + // The codec restores the selector as part of the provider's + // state. Fail rather than silently reading another state. + let selector = from_read_selector(scan.read_selector.as_ref())?; + if provider.read_selector() != selector.as_ref() { return internal_err!( - "Decoded table provider for '{table_name}' reads version {:?}, \ + "Decoded table provider for '{table_name}' reads selector {:?}, \ but {:?} was encoded", - provider.version(), - version + provider.read_selector(), + selector ); } @@ -1243,7 +1243,7 @@ impl AsLogicalPlan for LogicalPlanNode { }; let provider = ViewTable::new(input, definition) - .with_version(from_table_version(scan.version.as_ref())?); + .with_read_selector(from_read_selector(scan.read_selector.as_ref())?); let table_name = from_table_reference(scan.table_name.as_ref(), "ViewScan")?; @@ -1391,10 +1391,10 @@ impl AsLogicalPlan for LogicalPlanNode { let table_name = from_table_reference(scan.table_name.as_ref(), "EmptyTableScan")?; - let provider = Arc::new( - EmptyTable::new(Arc::clone(&schema)) - .with_version(from_table_version(scan.version.as_ref())?), - ); + let provider = + Arc::new(EmptyTable::new(Arc::clone(&schema)).with_read_selector( + from_read_selector(scan.read_selector.as_ref())?, + )); let table_scan = TableScanBuilder::new(table_name, provider_as_source(provider)) @@ -1457,9 +1457,9 @@ impl AsLogicalPlan for LogicalPlanNode { }) => { let provider = source_as_provider(source)?; let schema = provider.schema(); - let version = source - .version() - .map(protobuf::TableVersion::try_from) + let read_selector = source + .read_selector() + .map(protobuf::TableReadSelector::try_from) .transpose()?; let projection = match projection { @@ -1592,7 +1592,7 @@ impl AsLogicalPlan for LogicalPlanNode { file_sort_order: exprs_vec, fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), - version, + read_selector, }, )), }) @@ -1618,7 +1618,7 @@ impl AsLogicalPlan for LogicalPlanNode { .unwrap_or_default(), fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), - version, + read_selector, }, ))), }) @@ -1649,7 +1649,7 @@ impl AsLogicalPlan for LogicalPlanNode { schema: Some(schema), projection, filters, - version, + read_selector, }, )), }) @@ -1669,7 +1669,7 @@ impl AsLogicalPlan for LogicalPlanNode { custom_table_data: bytes, fetch: fetch.map(|f| f as u64), skip: skip.map(|o| o as u64), - version, + read_selector, }); let node = LogicalPlanNode { logical_plan_type: Some(scan), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index e8796658f10d1..58a19e232f773 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -64,7 +64,7 @@ use datafusion_common::format::{ use datafusion_common::scalar::ScalarStructBuilder; use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, - SplitPoint, TableReference, TableVersion, internal_datafusion_err, internal_err, + SplitPoint, TableReadSelector, TableReference, internal_datafusion_err, internal_err, not_impl_err, plan_err, }; use datafusion_execution::TaskContext; @@ -3843,20 +3843,21 @@ async fn roundtrip_empty_table_scan() -> Result<()> { } #[tokio::test] -async fn roundtrip_table_scan_with_version() -> Result<()> { +async fn roundtrip_table_scan_with_read_selector() -> Result<()> { let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("name", DataType::Utf8, true), ])); let ctx = SessionContext::new(); - for version in [ - TableVersion::Version(ScalarValue::Int64(Some(3))), - TableVersion::Version(ScalarValue::from("v1")), - TableVersion::Timestamp(1_704_067_200_000_000_000), + for selector in [ + TableReadSelector::AsOfVersion(ScalarValue::Int64(Some(3))), + TableReadSelector::AsOfVersion(ScalarValue::from("v1")), + TableReadSelector::AsOfTimestamp(1_704_067_200_000_000_000), ] { let table = Arc::new( - EmptyTable::new(Arc::clone(&schema)).with_version(Some(version.clone())), + EmptyTable::new(Arc::clone(&schema)) + .with_read_selector(Some(selector.clone())), ); let plan = LogicalPlanBuilder::scan( "empty", @@ -3866,12 +3867,12 @@ async fn roundtrip_table_scan_with_version() -> Result<()> { .build()?; let bytes = logical_plan_to_bytes(&plan)?; let restored = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?; - // `TableScan` equality includes the version of its source + // `TableScan` equality includes the selector of its source assert_eq!(plan, restored); let LogicalPlan::TableScan(scan) = restored else { panic!("expected a TableScan"); }; - assert_eq!(scan.source.version(), Some(&version)); + assert_eq!(scan.source.read_selector(), Some(&selector)); } Ok(()) } diff --git a/datafusion/session/src/schema.rs b/datafusion/session/src/schema.rs index dd51dcbf30404..69b2ba581a989 100644 --- a/datafusion/session/src/schema.rs +++ b/datafusion/session/src/schema.rs @@ -6,7 +6,7 @@ //! representing collections of named tables. use async_trait::async_trait; -use datafusion_common::{DataFusionError, TableVersion, exec_err, not_impl_err}; +use datafusion_common::{DataFusionError, TableReadSelector, exec_err, not_impl_err}; use std::any::Any; use std::fmt::Debug; use std::sync::Arc; @@ -38,28 +38,28 @@ pub trait SchemaProvider: Any + Debug + Sync + Send { name: &str, ) -> Result>, DataFusionError>; - /// Retrieves a specific version of a table from the schema by name, if it + /// Retrieves a table from the schema by name and [`TableReadSelector`], if it /// exists, otherwise returns `None`. /// - /// This is used to plan time travel queries such as - /// `SELECT * FROM t VERSION AS OF 3` or - /// `SELECT * FROM t TIMESTAMP AS OF '2024-01-01'`. See [`TableVersion`] - /// for the syntaxes that are supported. + /// The selector specifies which read of the table to perform, see + /// [`TableReadSelector`]. /// - /// The returned provider must report the version it reads from - /// [`TableProvider::version`], otherwise planning fails. This may differ - /// from the requested `version`, for example when a timestamp resolves to - /// the latest snapshot committed before it. + /// The returned provider must report the selector it reads from + /// [`TableProvider::read_selector`], otherwise planning fails. This may differ + /// from the requested `selector`, for example when a timestamp resolves to + /// the latest snapshot committed at or before it. /// /// The default implementation returns a "not implemented" error, as - /// returning the current version of the table would silently produce + /// returning the current state of the table would silently produce /// wrong results. - async fn table_at( + async fn table_with_read_selector( &self, name: &str, - version: &TableVersion, + selector: &TableReadSelector, ) -> Result>> { - not_impl_err!("Time travel is not supported for table '{name}@{version}'") + not_impl_err!( + "Table read selectors are not supported for table '{name}@{selector}'" + ) } /// Retrieves the type of a specific table from the schema by name, if it exists, otherwise diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs index 9f0b4eebfca4a..50f994467583a 100644 --- a/datafusion/session/src/table.rs +++ b/datafusion/session/src/table.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use crate::session::Session; use arrow_schema::SchemaRef; use async_trait::async_trait; -use datafusion_common::{Constraints, Statistics, TableVersion, not_impl_err}; +use datafusion_common::{Constraints, Statistics, TableReadSelector, not_impl_err}; use datafusion_common::{DFSchemaRef, Result, internal_err}; use datafusion_expr::Expr; use datafusion_expr::statistics::StatisticsRequest; @@ -71,21 +71,32 @@ pub trait TableProvider: Any + Debug + Sync + Send { None } - /// The version of the table this provider reads, for time travel - /// queries such as `SELECT * FROM t VERSION AS OF 3`. + /// The [`TableReadSelector`] this provider reads, such as a specific version + /// of the table. /// - /// Providers returned by [`SchemaProvider::table_at`] must report the - /// version they read. This may differ from the requested version: for - /// example, `TIMESTAMP AS OF '2024-01-15'` may resolve to the snapshot - /// with id `3`, committed before that time. Reporting the resolved - /// version shows it in `EXPLAIN`, and lets scans of the same snapshot - /// compare equal regardless of how it was requested. + /// Providers returned by [`SchemaProvider::table_with_read_selector`] must + /// report the selector they read. This may differ from the requested + /// selector: for example, a [`TableReadSelector::AsOfTimestamp`] may resolve + /// to the [`TableReadSelector::AsOfVersion`] of the latest snapshot committed + /// at or before it. Reporting the resolved selector shows it in `EXPLAIN`, + /// and lets scans of the same snapshot compare equal regardless of how it + /// was requested. /// - /// Defaults to `None`, meaning the version is unknown or the table is not - /// versioned. + /// Providers returned by [`SchemaProvider::table`] may also report a + /// selector, such as the id of the current snapshot, to identify the + /// state of the table they read. /// - /// [`SchemaProvider::table_at`]: crate::SchemaProvider::table_at - fn version(&self) -> Option<&TableVersion> { + /// Scans of the same table with equal selectors are assumed to read the + /// same data, and may be merged by the optimizer. Report a selector that + /// identifies the data exactly: two scans with equal selectors that read + /// different data can produce wrong results. + /// + /// Defaults to `None`, meaning the provider does not identify which read + /// of the table it performs. + /// + /// [`SchemaProvider::table`]: crate::SchemaProvider::table + /// [`SchemaProvider::table_with_read_selector`]: crate::SchemaProvider::table_with_read_selector + fn read_selector(&self) -> Option<&TableReadSelector> { None } diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rs index f02cb6e82c07c..dbc37a24ccaa9 100644 --- a/datafusion/sql/src/relation/mod.rs +++ b/datafusion/sql/src/relation/mod.rs @@ -2,24 +2,28 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::ops::ControlFlow; +use std::cell::Cell; use std::sync::Arc; use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; +use crate::resolve::ResolvedReference; +use arrow::datatypes::{DataType, TimeUnit}; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_common::{ - DFSchema, Diagnostic, Result, Span, Spans, TableReference, not_impl_err, plan_err, + DFSchema, Diagnostic, Result, ScalarValue, Span, Spans, TableReadSelector, + TableReference, internal_err, not_impl_err, plan_err, }; use datafusion_expr::builder::subquery_alias; use datafusion_expr::planner::{ - PlannedRelation, RelationPlannerContext, RelationPlanning, TableVersionExpr, + PlannedRelation, RelationPlannerContext, RelationPlanning, TableReadSelectorContext, + TableReadSelectorPlanning, }; use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, Volatility, expr::Unnest}; use datafusion_expr::{Subquery, SubqueryAlias}; use sqlparser::ast::{ Expr as SQLExpr, FunctionArg, FunctionArgExpr, FunctionArguments, Spanned, Statement, - TableFactor, TableVersion as SQLTableVersion, Visit, Visitor, + TableFactor, TableVersion, }; mod join; @@ -38,13 +42,6 @@ fn get_cte_for_table_reference<'a>( planner_context.get_cte(&name.to_string()) } -/// Returns the expression of a time travel clause -fn version_expr(version: &TableVersionExpr) -> &Expr { - match version { - TableVersionExpr::Version(expr) | TableVersionExpr::Timestamp(expr) => expr, - } -} - /// Returns the argument name and value of an `AT ( => )` time /// travel clause, or `None` if `func` has any other form. fn at_clause_argument(func: &SQLExpr) -> Option<(&str, &SQLExpr)> { @@ -94,13 +91,17 @@ impl RelationPlannerContext for SqlToRelRelationContext<'_, get_cte_for_table_reference(self.planner_context, name) } - fn sql_to_expr(&mut self, expr: SQLExpr, schema: &DFSchema) -> Result { + fn sql_to_expr( + &mut self, + expr: sqlparser::ast::Expr, + schema: &DFSchema, + ) -> Result { self.planner.sql_to_expr(expr, schema, self.planner_context) } fn sql_expr_to_logical_expr( &mut self, - expr: SQLExpr, + expr: sqlparser::ast::Expr, schema: &DFSchema, ) -> Result { self.planner @@ -119,6 +120,123 @@ impl RelationPlannerContext for SqlToRelRelationContext<'_, } } +struct SqlToRelTableReadSelectorContext<'a, 'b, S: ContextProvider> { + planner: &'a SqlToRel<'b, S>, + /// The table factor whose selector is determined + relation: &'a TableFactor, + /// Whether the statement is being prepared, in which case expressions + /// must not depend on the time of execution + prepared: bool, + /// The error message for an expression that depends on the time of + /// execution, in a prepared statement + execution_dependent: Cell>, +} + +impl<'a, 'b, S: ContextProvider> SqlToRelTableReadSelectorContext<'a, 'b, S> { + fn new( + planner: &'a SqlToRel<'b, S>, + relation: &'a TableFactor, + prepared: bool, + ) -> Self { + Self { + planner, + relation, + prepared, + execution_dependent: Cell::new(None), + } + } +} + +impl TableReadSelectorContext + for SqlToRelTableReadSelectorContext<'_, '_, S> +{ + fn evaluate_const_expr(&self, expr: SQLExpr) -> Result { + // A fresh planner context, so planning the expression can neither + // depend on nor affect the planning of the query that references the + // table, and the selector is the same before and during planning + let expr = self.planner.sql_to_expr( + expr, + &DFSchema::empty(), + &mut PlannerContext::new(), + )?; + if expr.exists(|e| Ok(matches!(e, Expr::Placeholder(_))))? { + return not_impl_err!( + "Parameters are not supported in table read selectors: {}", + self.relation + ); + } + // The selector is determined when the statement is prepared, so + // functions such as `now()` would return the time of preparation + // rather than of execution + if self.prepared + && expr.exists(|e| { + Ok(matches!( + e, + Expr::ScalarFunction(f) + if f.func.signature().volatility != Volatility::Immutable + )) + })? + { + let message = format!( + "Table read selectors in prepared statements must not depend on the \ + time of execution, such as with now(): {}", + self.relation + ); + self.execution_dependent.set(Some(message.clone())); + return not_impl_err!("{message}"); + } + self.planner.context_provider.evaluate_const_expr(expr) + } +} + +/// Determines the [`TableReadSelector`] of a table factor with a time travel +/// clause, as DataFusion does by default. See [`SqlToRel::read_selector`] +/// for the mapping. +fn default_read_selector( + relation: &TableFactor, + context: &dyn TableReadSelectorContext, +) -> Result> { + let TableFactor::Table { + version: Some(version), + .. + } = relation + else { + return Ok(None); + }; + let (is_timestamp, expr) = match version { + TableVersion::VersionAsOf(expr) => (false, expr), + TableVersion::TimestampAsOf(expr) | TableVersion::ForSystemTimeAsOf(expr) => { + (true, expr) + } + TableVersion::Function(func) => match at_clause_argument(func) { + Some((name, expr)) if name.eq_ignore_ascii_case("VERSION") => (false, expr), + Some((name, expr)) if name.eq_ignore_ascii_case("TIMESTAMP") => (true, expr), + _ => return not_impl_err!("Unsupported time travel clause: {func}"), + }, + TableVersion::Changes { .. } => { + return not_impl_err!("CHANGES clauses are not supported"); + } + }; + let value = context.evaluate_const_expr(expr.clone())?; + if value.is_null() { + let kind = if is_timestamp { "timestamp" } else { "version" }; + return plan_err!( + "The {kind} of a time travel clause must not be NULL: {relation}" + ); + } + if !is_timestamp { + return Ok(Some(TableReadSelector::AsOfVersion(value))); + } + + let timestamp_type = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())); + match value.cast_to(×tamp_type)? { + ScalarValue::TimestampNanosecond(Some(ns), _) => { + Ok(Some(TableReadSelector::AsOfTimestamp(ns))) + } + value => internal_err!("Expected a timestamp, got: {value:?}"), + } +} + impl SqlToRel<'_, S> { /// Create a `LogicalPlan` that scans the named relation. /// @@ -188,6 +306,21 @@ impl SqlToRel<'_, S> { planner_context: &mut PlannerContext, ) -> Result { let relation_span = relation.span(); + // References to CTEs are read without a selector + let (table_ref, selector) = match &relation { + TableFactor::Table { + name, args: None, .. + } => { + let table_ref = self.object_name_to_table_reference(name.clone())?; + let selector = + match get_cte_for_table_reference(planner_context, &table_ref) { + Some(_) => None, + None => self.read_selector(&table_ref, &relation)?, + }; + (Some(table_ref), selector) + } + _ => (None, None), + }; let (plan, alias) = match relation { TableFactor::Table { name, @@ -233,20 +366,22 @@ impl SqlToRel<'_, S> { .build()?; (plan, alias) } else { - // Normalize name and alias - let table_ref = self.object_name_to_table_reference(name)?; + let Some(table_ref) = table_ref else { + return internal_err!( + "The name of table '{name}' was not normalized" + ); + }; let cte = get_cte_for_table_reference(planner_context, &table_ref); - if let Some(version) = version { - if cte.is_some() { - return plan_err!( - "Time travel is not supported for CTE '{table_ref}'" - ); - } - let plan = self.versioned_table_scan( + if cte.is_some() && version.is_some() { + return plan_err!( + "Time travel is not supported for CTE '{table_ref}'" + ); + } + if let Some(selector) = selector { + let plan = self.selected_table_scan( table_ref, - version, + &selector, relation_span, - planner_context, )?; return Ok(PlannedRelation::new(plan, alias)); } @@ -366,141 +501,126 @@ impl SqlToRel<'_, S> { Ok(PlannedRelation::new(plan, alias)) } - /// Plans a scan of a specific version of a table, as requested by a time - /// travel clause such as `VERSION AS OF 3`. - fn versioned_table_scan( + /// Derives a [`TableReadSelector`] from a table factor that references + /// `table`. The returned [`TableReadSelector`], if present, describes + /// which read of the table is requested by the table factor. + /// + /// The registered [`TableReadSelectorPlanner`]s are asked first, in order. If none + /// of them handles the table factor, its time travel clause, if any, is + /// mapped to a selector. The different syntaxes used by SQL dialects are + /// normalized to either a version or a timestamp: + /// + /// | SQL | [`TableReadSelector`] | + /// |----------------------------------------------|----------------------| + /// | `t VERSION AS OF 3` | [`AsOfVersion`] | + /// | `t AT (VERSION => 'v1')` | [`AsOfVersion`] | + /// | `t TIMESTAMP AS OF '2024-01-01'` | [`AsOfTimestamp`] | + /// | `t FOR SYSTEM_TIME AS OF now()` | [`AsOfTimestamp`] | + /// | `t AT (TIMESTAMP => now() - INTERVAL '1 h')` | [`AsOfTimestamp`] | + /// + /// The expressions in these clauses are evaluated during planning and are + /// required to be constant expressions. Other clauses, + /// such as Snowflake's `BEFORE (...)`, `AT (OFFSET => ...)` or + /// `CHANGES (...)`, are not supported. + /// + /// Returns `None` if the table is to be read without a selector. + /// + /// [`TableReadSelectorPlanner`]: datafusion_expr::planner::TableReadSelectorPlanner + /// [`AsOfVersion`]: TableReadSelector::AsOfVersion + /// [`AsOfTimestamp`]: TableReadSelector::AsOfTimestamp + pub fn read_selector( + &self, + table: &TableReference, + relation: &TableFactor, + ) -> Result> { + let context = SqlToRelTableReadSelectorContext::new(self, relation, false); + self.plan_read_selector(table, relation, &context) + } + + fn plan_read_selector( + &self, + table: &TableReference, + relation: &TableFactor, + context: &SqlToRelTableReadSelectorContext<'_, '_, S>, + ) -> Result> { + for planner in self.context_provider.get_table_read_selector_planners() { + match planner.plan_read_selector(table, relation, context)? { + TableReadSelectorPlanning::Planned(selector) => return Ok(selector), + TableReadSelectorPlanning::Original => {} + } + } + default_read_selector(relation, context) + } + + /// Plans a scan of a table with a [`TableReadSelector`]. + fn selected_table_scan( &self, table_ref: TableReference, - version: SQLTableVersion, + selector: &TableReadSelector, relation_span: sqlparser::tokenizer::Span, - planner_context: &mut PlannerContext, ) -> Result { - let version = self.sql_table_version_to_expr(version, planner_context)?; - if version_expr(&version).exists(|e| Ok(matches!(e, Expr::Placeholder(_))))? { - return not_impl_err!( - "Parameters are not supported in time travel clauses: {table_ref}" - ); - } - let version = self.context_provider.evaluate_table_version(version)?; let provider = self .context_provider - .get_table_source_at(table_ref.clone(), &version) + .get_table_source_with_read_selector(table_ref.clone(), selector) .map_err(|e| { e.with_diagnostic(Diagnostic::new_error( - format!("cannot read table '{table_ref}@{version}'"), + format!("cannot read table '{table_ref}@{selector}'"), Span::try_from_sqlparser_span(relation_span), )) })?; - // The source reports the version it reads, which may differ from the - // requested version, e.g. a snapshot id for a timestamp - if provider.version().is_none() { + // The source reports the selector it reads, which may differ from the + // requested selector, e.g. a snapshot id for a timestamp + if provider.read_selector().is_none() { return plan_err!( - "The source of table '{table_ref}@{version}' must report the version \ - it reads" + "The source of table '{table_ref}@{selector}' must report the \ + selector it reads" ); } LogicalPlanBuilder::scan(table_ref, provider, None)?.build() } - /// Converts a time travel clause to a [`TableVersionExpr`]. - /// - /// The different syntaxes used by SQL dialects are normalized to either a - /// version or a timestamp: - /// - /// * `VERSION AS OF ` and `AT (VERSION => )` become - /// [`TableVersionExpr::Version`] - /// * `TIMESTAMP AS OF `, `FOR SYSTEM_TIME AS OF ` and - /// `AT (TIMESTAMP => )` become [`TableVersionExpr::Timestamp`] - /// - /// Other clauses, such as Snowflake's `BEFORE (...)`, `AT (OFFSET => ...)` - /// or `CHANGES (...)`, are not supported. - pub fn sql_table_version_to_expr( - &self, - version: SQLTableVersion, - planner_context: &mut PlannerContext, - ) -> Result { - let (is_timestamp, expr) = match version { - SQLTableVersion::VersionAsOf(expr) => (false, expr), - SQLTableVersion::TimestampAsOf(expr) - | SQLTableVersion::ForSystemTimeAsOf(expr) => (true, expr), - SQLTableVersion::Function(func) => match at_clause_argument(&func) { - Some((name, expr)) if name.eq_ignore_ascii_case("VERSION") => { - (false, expr.clone()) - } - Some((name, expr)) if name.eq_ignore_ascii_case("TIMESTAMP") => { - (true, expr.clone()) - } - _ => return not_impl_err!("Unsupported time travel clause: {func}"), - }, - SQLTableVersion::Changes { .. } => { - return not_impl_err!("CHANGES clauses are not supported"); - } - }; - let expr = self.sql_to_expr(expr, &DFSchema::empty(), planner_context)?; - Ok(if is_timestamp { - TableVersionExpr::Timestamp(expr) - } else { - TableVersionExpr::Version(expr) - }) - } - - /// Rejects time travel clauses in a prepared statement that depend on the + /// Rejects table read selectors in a prepared statement that depend on the /// time of execution, such as `AT (TIMESTAMP => now())`. /// - /// Time travel clauses are evaluated, and the table version is resolved, - /// when the statement is prepared. Functions such as `now()` would - /// therefore return the time of preparation rather than of execution. - pub(crate) fn check_prepared_table_versions( + /// Table read selectors are determined, and the table is retrieved from the + /// catalog, when the statement is prepared. Functions such as `now()` + /// would therefore return the time of preparation rather than of + /// execution. + pub(crate) fn check_prepared_read_selectors( &self, statement: &Statement, ) -> Result<()> { - #[derive(Default)] - struct TableVersions(Vec); - - impl Visitor for TableVersions { - type Break = (); - - fn pre_visit_table_factor( - &mut self, - table_factor: &TableFactor, - ) -> ControlFlow { - if let TableFactor::Table { - version: Some(version), - .. - } = table_factor - { - self.0.push(version.clone()); + // Visit the same table factors as are retrieved from the catalog + // before planning, which excludes references to CTEs + let mut execution_dependent = false; + let result = crate::resolve::resolve_sql_references( + statement, + self.options.enable_ident_normalization, + |reference| { + let ResolvedReference::TableFactor(table, table_factor) = reference + else { + return Ok(()); + }; + let context = + SqlToRelTableReadSelectorContext::new(self, table_factor, true); + // Other errors are reported when the statement is planned + let _ = self.plan_read_selector(&table, table_factor, &context); + match context.execution_dependent.into_inner() { + Some(message) => { + execution_dependent = true; + not_impl_err!("{message}") + } + None => Ok(()), } - ControlFlow::Continue(()) - } - } - - let mut versions = TableVersions::default(); - let _ = statement.visit(&mut versions); - for version in versions.0 { - let display = version.to_string(); - // Errors are reported when the statement is planned - let Ok(expr) = - self.sql_table_version_to_expr(version, &mut PlannerContext::new()) - else { - continue; - }; - let depends_on_execution = version_expr(&expr).exists(|e| { - Ok(matches!( - e, - Expr::ScalarFunction(f) - if f.func.signature().volatility != Volatility::Immutable - )) - })?; - if depends_on_execution { - return not_impl_err!( - "Time travel clauses in prepared statements must not depend on \ - the time of execution, such as with now(): {}", - display.trim() - ); - } + }, + ); + // Other errors, such as for invalid table names, are reported when the + // statement is planned + if execution_dependent { + result.map(|_| ()) + } else { + Ok(()) } - Ok(()) } pub(crate) fn create_relation_subquery( diff --git a/datafusion/sql/src/resolve.rs b/datafusion/sql/src/resolve.rs index c7422359e2d27..94e72673fd00f 100644 --- a/datafusion/sql/src/resolve.rs +++ b/datafusion/sql/src/resolve.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashSet}; use std::ops::ControlFlow; use datafusion_common::{DataFusionError, Result}; @@ -14,100 +14,177 @@ use sqlparser::ast::*; // Collect table/CTE references as `TableReference`s and normalize them during traversal. // This avoids a second normalization/conversion pass after visiting the AST. -struct RelationVisitor { - relations: BTreeSet, - versioned_relations: BTreeSet<(TableReference, TableVersion)>, - all_ctes: BTreeSet, +// +// # Non-recursive CTEs +// +// The name of a non-recursive CTE is in scope after its definition, but not +// within its own body. The `Visitor` trait has no hooks for individual CTEs, +// so `pre_visit_query` visits these CTE bodies itself, putting each name in +// scope after its body. +// The AST traversal cannot be instructed to skip subtrees so the CTE bodies +// are visited a second time, with all CTEs of the query in scope. During this +// second visit we shouldn't record anything: it's redundant, and its scope is +// wrong. +// +// For example, in this query the inner `b` is a CTE inside the body of CTE +// `a`, and the `c` in `a`'s body is a table because CTE `c` is defined later: +// +// ```sql +// WITH a AS ( +// WITH b AS (SELECT * FROM t) -- [1] +// SELECT * FROM b, c, u -- [2] +// ), +// c AS (SELECT * FROM a) -- [3] +// SELECT * FROM c -- [4] +// ``` +// +// First pass, by `pre_visit_query` of the outer query, with +// `skipped_cte_body_depth` 0: +// +// * The body of `a`, with no CTEs in scope. Its own `pre_visit_query` visits +// [1]: `t` is recorded, then `b` is put in scope. The traversal then visits +// [1] again, as described below, which records nothing. At [2], `b` is a CTE +// reference, and `c` and `u` are recorded as tables. Then `a` is put in +// scope. +// * [3], with `a` in scope: `a` is a CTE reference. Then `c` is put in scope. +// +// Second pass, by the traversal of the outer query. Each body visited in the +// first pass increments `skipped_cte_body_depth` on entry and decrements it on +// exit: +// +// * The body of `a`: `skipped_cte_body_depth` becomes 1. `pre_visit_query` no +// longer visits CTE bodies itself, but still puts `b` in scope. +// * [1]: `skipped_cte_body_depth` becomes 2, and is back to 1 afterwards. +// * [2]: `skipped_cte_body_depth` is still 1, so nothing is recorded. Here `c` +// is in scope and would wrongly be seen as a CTE reference. With a flag +// instead of a counter, leaving [1] would have cleared it, and `u` would +// have been recorded twice. +// * [3]: skipped the same way, after which `skipped_cte_body_depth` is back +// to 0. +// * [4]: the main body of the query, which was not visited in the first pass. +// Nothing is skipped, but `c` is a CTE reference, so it is not recorded. +// +// Each table factor, `t`, `c` and `u`, was reported once. +struct RelationVisitor { ctes_in_scope: Vec, enable_ident_normalization: bool, - /// The names of the table factors with a time travel clause that are - /// being visited, innermost last. These names are recorded in - /// `versioned_relations`, so they are skipped when visited as relations. - /// Only used for pointer comparison. - versioned_names: Vec<*const ObjectName>, + /// The names of the table factors that are being visited, innermost + /// last. These names are reported with their table factor, so they are + /// skipped when visited as relations. Only used for pointer comparison. + table_factor_names: Vec<*const ObjectName>, + /// The bodies of the non-recursive CTEs that have been visited before + /// their name was put in scope. They are skipped when visited again as + /// part of their query. Only used for pointer comparison. + visited_cte_bodies: HashSet<*const Query>, + /// The number of CTE bodies in `visited_cte_bodies` that are being + /// visited again. While positive, nothing is reported. + skipped_cte_body_depth: usize, + /// Called for each reference + on_reference: F, } -impl RelationVisitor { - /// Record the reference to `relation`, if it's not a CTE reference. - fn insert_relation(&mut self, relation: &ObjectName) -> ControlFlow { +impl) -> Result<()>> RelationVisitor { + fn new(enable_ident_normalization: bool, on_reference: F) -> Self { + Self { + ctes_in_scope: vec![], + enable_ident_normalization, + table_factor_names: vec![], + visited_cte_bodies: HashSet::new(), + skipped_cte_body_depth: 0, + on_reference, + } + } + + fn report( + &mut self, + reference: ResolvedReference<'_>, + ) -> ControlFlow { + match (self.on_reference)(reference) { + Ok(()) => ControlFlow::Continue(()), + Err(e) => ControlFlow::Break(e), + } + } + + /// Normalizes `name`, and returns `None` if it references a CTE + fn table_reference( + &self, + name: &ObjectName, + ) -> ControlFlow> { match object_name_to_table_reference( - relation.clone(), + name.clone(), self.enable_ident_normalization, ) { - Ok(relation) => { - if !self.relations.contains(&relation) - && !self.ctes_in_scope.contains(&relation) - { - self.relations.insert(relation); - } - ControlFlow::Continue(()) + Ok(table) if self.ctes_in_scope.contains(&table) => { + ControlFlow::Continue(None) } + Ok(table) => ControlFlow::Continue(Some(table)), Err(e) => ControlFlow::Break(e), } } - /// Record the reference to `relation` with a time travel clause, such as - /// `VERSION AS OF 3`, if it's not a CTE reference. - fn insert_versioned_relation( + /// Report the reference to the table `relation`, if it's not a CTE + /// reference + fn visit_relation(&mut self, relation: &ObjectName) -> ControlFlow { + if let Some(table) = self.table_reference(relation)? { + self.report(ResolvedReference::Table(table))?; + } + ControlFlow::Continue(()) + } + + /// Report `table_factor`, which references the table `relation`, if it's + /// not a CTE reference + fn visit_table_factor( &mut self, relation: &ObjectName, - version: &TableVersion, + table_factor: &TableFactor, ) -> ControlFlow { - match object_name_to_table_reference( - relation.clone(), - self.enable_ident_normalization, - ) { - Ok(relation) => { - if !self.ctes_in_scope.contains(&relation) { - self.versioned_relations.insert((relation, version.clone())); - } - ControlFlow::Continue(()) - } - Err(e) => ControlFlow::Break(e), + if let Some(table) = self.table_reference(relation)? { + self.report(ResolvedReference::TableFactor(table, table_factor))?; } + ControlFlow::Continue(()) } } -/// Returns the name and time travel clause of `table_factor`, if it is a -/// table with a time travel clause, such as `t VERSION AS OF 3` -fn versioned_table(table_factor: &TableFactor) -> Option<(&ObjectName, &TableVersion)> { +/// Returns the name of the table referenced by `table_factor`, if it +/// references a table or view, rather than for example a subquery +fn table_factor_name(table_factor: &TableFactor) -> Option<&ObjectName> { match table_factor { TableFactor::Table { - name, - args: None, - version: Some(version), - .. - } => Some((name, version)), + name, args: None, .. + } => Some(name), _ => None, } } -impl Visitor for RelationVisitor { +impl) -> Result<()>> Visitor for RelationVisitor { type Break = DataFusionError; fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow { - if self - .versioned_names - .iter() - .any(|name| std::ptr::eq(*name, relation)) + if self.skipped_cte_body_depth > 0 + || self + .table_factor_names + .iter() + .any(|name| std::ptr::eq(*name, relation)) { return ControlFlow::Continue(()); } - self.insert_relation(relation) + self.visit_relation(relation) } fn pre_visit_table_factor( &mut self, table_factor: &TableFactor, ) -> ControlFlow { - if let Some((name, version)) = versioned_table(table_factor) { + if let Some(name) = table_factor_name(table_factor) { // The table name is also visited as a relation. Skip exactly that - // node, so it is not recorded without its version as well. Other - // relations within this table factor, such as those in a subquery - // in the time travel clause, are still recorded. - self.versioned_names.push(name); - self.insert_versioned_relation(name, version)?; + // node, so it is not reported without its table factor as well. + // Other relations within this table factor, such as those in a + // subquery in a time travel clause, are still reported. + self.table_factor_names.push(name); + if self.skipped_cte_body_depth == 0 { + self.visit_table_factor(name, table_factor)?; + } } ControlFlow::Continue(()) } @@ -116,23 +193,27 @@ impl Visitor for RelationVisitor { &mut self, table_factor: &TableFactor, ) -> ControlFlow { - if versioned_table(table_factor).is_some() { - self.versioned_names.pop(); + if table_factor_name(table_factor).is_some() { + self.table_factor_names.pop(); } ControlFlow::Continue(()) } fn pre_visit_query(&mut self, q: &Query) -> ControlFlow { + if self.visited_cte_bodies.contains(&std::ptr::from_ref(q)) { + self.skipped_cte_body_depth += 1; + } if let Some(with) = &q.with { for cte in &with.cte_tables { // The non-recursive CTE name is not in scope when evaluating the CTE itself, so this is valid: // `WITH t AS (SELECT * FROM t) SELECT * FROM t` // Where the first `t` refers to a predefined table. So we are careful here - // to visit the CTE first, before putting it in scope. - if !with.recursive { - // This is a bit hackish as the CTE will be visited again as part of visiting `q`, - // but thankfully `insert_relation` is idempotent. + // to visit the CTE first, before putting it in scope. Its body is skipped when + // visited again as part of `q`; see "Non-recursive CTEs" on `RelationVisitor`. + if !with.recursive && self.skipped_cte_body_depth == 0 { cte.visit(self)?; + self.visited_cte_bodies + .insert(std::ptr::from_ref(cte.query.as_ref())); } let cte_name = ObjectName::from(vec![cte.alias.name.clone()]); match object_name_to_table_reference( @@ -151,9 +232,16 @@ impl Visitor for RelationVisitor { if let Some(with) = &q.with { for _ in &with.cte_tables { // Unwrap: We just pushed these in `pre_visit_query` - self.all_ctes.insert(self.ctes_in_scope.pop().unwrap()); + let cte = self.ctes_in_scope.pop().unwrap(); + // CTEs in skipped bodies were reported when first visited + if self.skipped_cte_body_depth == 0 { + self.report(ResolvedReference::Cte(cte))?; + } } } + if self.visited_cte_bodies.contains(&std::ptr::from_ref(q)) { + self.skipped_cte_body_depth -= 1; + } ControlFlow::Continue(()) } @@ -163,7 +251,7 @@ impl Visitor for RelationVisitor { obj_name, } = statement { - self.insert_relation(obj_name)?; + self.visit_relation(obj_name)?; } ControlFlow::Continue(()) @@ -177,17 +265,20 @@ fn control_flow_to_result(flow: ControlFlow) -> Result<()> { } } -fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) -> Result<()> { +fn visit_statement) -> Result<()>>( + statement: &DFStatement, + visitor: &mut RelationVisitor, +) -> Result<()> { match statement { DFStatement::Statement(s) => { control_flow_to_result(s.as_ref().visit(visitor))?; } DFStatement::CreateExternalTable(table) => { - control_flow_to_result(visitor.insert_relation(&table.name))?; + control_flow_to_result(visitor.visit_relation(&table.name))?; } DFStatement::CopyTo(CopyToStatement { source, .. }) => match source { CopyToSource::Relation(table_name) => { - control_flow_to_result(visitor.insert_relation(table_name))?; + control_flow_to_result(visitor.visit_relation(table_name))?; } CopyToSource::Query(query) => { control_flow_to_result(query.visit(visitor))?; @@ -203,25 +294,24 @@ fn visit_statement(statement: &DFStatement, visitor: &mut RelationVisitor) -> Re Ok(()) } -/// Tables, views and CTEs referenced in a SQL statement, as collected by -/// [`resolve_table_references_with_versions`]. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct TableReferences { - /// Tables and views referenced without a time travel clause - pub tables: Vec, - /// Tables and views referenced with a time travel clause, such as - /// `VERSION AS OF 3`, together with that clause - pub versioned_tables: Vec<(TableReference, TableVersion)>, - /// CTE aliases that were defined and possibly referenced - pub ctes: Vec, +/// A reference in a SQL statement, as reported by [`resolve_references`] +#[derive(Debug, Clone, PartialEq)] +pub enum ResolvedReference<'a> { + /// A table or view referenced by a table factor, such as `t` in + /// `SELECT * FROM t` + TableFactor(TableReference, &'a TableFactor), + /// A table or view referenced other than by a table factor, such as the + /// target of an `INSERT` statement + Table(TableReference), + /// A CTE defined in the statement + Cte(TableReference), } /// Collects all tables and views referenced in the SQL statement. CTEs are collected separately. /// This can be used to determine which tables need to be in the catalog for a query to be planned. /// -/// Tables referenced with a time travel clause, such as `VERSION AS OF 3`, -/// are included without their version. Use -/// [`resolve_table_references_with_versions`] to also obtain those versions. +/// Only the tables are returned, not the table factors that reference them. +/// Use [`resolve_references`] to also visit the table factors. /// /// # Returns /// @@ -258,70 +348,84 @@ pub fn resolve_table_references( statement: &crate::parser::Statement, enable_ident_normalization: bool, ) -> Result<(Vec, Vec)> { - let visitor = visit_relations(statement, enable_ident_normalization)?; - let mut relations = visitor.relations; - relations.extend(visitor.versioned_relations.into_iter().map(|(r, _)| r)); - Ok(( - relations.into_iter().collect(), - visitor.all_ctes.into_iter().collect(), - )) + let mut tables = BTreeSet::new(); + let mut ctes = BTreeSet::new(); + resolve_references(statement, enable_ident_normalization, |reference| { + match reference { + ResolvedReference::TableFactor(table, _) + | ResolvedReference::Table(table) => { + tables.insert(table); + } + ResolvedReference::Cte(cte) => { + ctes.insert(cte); + } + } + Ok(()) + })?; + Ok((tables.into_iter().collect(), ctes.into_iter().collect())) } -/// Collects all tables, views and CTEs referenced in the SQL statement, like -/// [`resolve_table_references`], and also the time travel clauses of tables, -/// such as `VERSION AS OF 3`. +/// Finds the tables, views and CTEs referenced in a SQL statement. +/// +/// Unlike [`resolve_table_references`], this also reports the table factor +/// through which a table is referenced, which determines the +/// [`TableReadSelector`] used to read the table. +/// +/// [`TableReadSelector`]: datafusion_common::TableReadSelector +/// +/// Each reference is passed to `on_reference` as a [`ResolvedReference`]. +/// Callers should not make assumptions on the order in which references are passed +/// to the callback. Table factors that reference a CTE are not reported, but the +/// definition of the CTE is. /// -/// A table referenced with a time travel clause is only included in -/// [`TableReferences::tables`] if it is also referenced without one. +/// References are reported as often as they occur, so the same table or CTE +/// may be reported more than once. For example, in +/// `INSERT INTO t SELECT * FROM t JOIN t AS t2 ON true`, `t` is reported as a +/// [`ResolvedReference::Table`] and twice as a +/// [`ResolvedReference::TableFactor`]. Callers that need the distinct set of tables +/// must remove duplicates themselves, as [`resolve_table_references`] does. +/// +/// [`TableReference`]s are reported in the form they are encountered. For instance, `t` will be +/// reported as a [`TableReference::Bare`] and `public.t` as a [`TableReference::Partial`]. /// /// ## Example /// /// ``` -/// # use datafusion_sql::parser::DFParserBuilder; -/// # use datafusion_sql::resolve::resolve_table_references_with_versions; -/// # use datafusion_sql::sqlparser::dialect::DatabricksDialect; -/// let query = "SELECT * FROM foo JOIN bar VERSION AS OF 3 ON foo.id = bar.id"; -/// let statement = DFParserBuilder::new(query) -/// .with_dialect(&DatabricksDialect {}) -/// .build() -/// .unwrap() -/// .parse_statements() -/// .unwrap() -/// .pop_back() -/// .unwrap(); -/// let references = resolve_table_references_with_versions(&statement, true).unwrap(); -/// assert_eq!(references.tables.len(), 1); -/// assert_eq!(references.tables[0].to_string(), "foo"); -/// assert_eq!(references.versioned_tables.len(), 1); -/// assert_eq!(references.versioned_tables[0].0.to_string(), "bar"); -/// assert_eq!(references.versioned_tables[0].1.to_string(), "VERSION AS OF 3"); +/// # use datafusion_sql::parser::DFParser; +/// # use datafusion_sql::resolve::{ResolvedReference, resolve_references}; +/// let query = "INSERT INTO foo SELECT * FROM bar AS b"; +/// let statement = DFParser::parse_sql(query).unwrap().pop_back().unwrap(); +/// let mut references = vec![]; +/// resolve_references(&statement, true, |reference| { +/// references.push(match reference { +/// ResolvedReference::TableFactor(table, table_factor) => { +/// format!("table factor {table}: {table_factor}") +/// } +/// ResolvedReference::Table(table) => format!("table {table}"), +/// ResolvedReference::Cte(cte) => format!("cte {cte}"), +/// }); +/// Ok(()) +/// }) +/// .unwrap(); +/// assert_eq!(references, ["table foo", "table factor bar: bar AS b"]); /// ``` -pub fn resolve_table_references_with_versions( +pub fn resolve_references( statement: &crate::parser::Statement, enable_ident_normalization: bool, -) -> Result { - let visitor = visit_relations(statement, enable_ident_normalization)?; - Ok(TableReferences { - tables: visitor.relations.into_iter().collect(), - versioned_tables: visitor.versioned_relations.into_iter().collect(), - ctes: visitor.all_ctes.into_iter().collect(), - }) + on_reference: impl FnMut(ResolvedReference<'_>) -> Result<()>, +) -> Result<()> { + let mut visitor = RelationVisitor::new(enable_ident_normalization, on_reference); + visit_statement(statement, &mut visitor) } -fn visit_relations( - statement: &crate::parser::Statement, +/// Like [`resolve_references`], for a SQL statement as parsed by sqlparser +pub(crate) fn resolve_sql_references( + statement: &Statement, enable_ident_normalization: bool, -) -> Result { - let mut visitor = RelationVisitor { - relations: BTreeSet::new(), - versioned_relations: BTreeSet::new(), - all_ctes: BTreeSet::new(), - ctes_in_scope: vec![], - enable_ident_normalization, - versioned_names: vec![], - }; - visit_statement(statement, &mut visitor)?; - Ok(visitor) + on_reference: impl FnMut(ResolvedReference<'_>) -> Result<()>, +) -> Result<()> { + let mut visitor = RelationVisitor::new(enable_ident_normalization, on_reference); + control_flow_to_result(statement.visit(&mut visitor)) } #[cfg(test)] @@ -342,112 +446,120 @@ mod tests { .unwrap() } + /// Returns the references reported for `query`, in the Snowflake dialect + fn references_of(query: &str) -> Vec { + let mut references = vec![]; + resolve_references(&parse_snowflake(query), true, |reference| { + references.push(match reference { + ResolvedReference::TableFactor(table, table_factor) => { + format!("{table}: {table_factor}") + } + ResolvedReference::Table(table) => format!("table {table}"), + ResolvedReference::Cte(cte) => format!("cte {cte}"), + }); + Ok(()) + }) + .unwrap(); + references + } + #[test] - fn resolve_table_references_with_versions_collects_versions() { - let statement = parse_snowflake( - "SELECT * FROM a \ + fn resolve_references_reports_table_factors() { + let query = "SELECT * FROM a \ JOIN b AT (VERSION => 1) ON true \ JOIN B AT (VERSION => 1) ON true \ JOIN b TIMESTAMP AS OF now() ON true \ - WHERE a.x IN (SELECT x FROM c VERSION AS OF 2)", - ); - let references = - resolve_table_references_with_versions(&statement, true).unwrap(); - let tables: Vec<_> = references.tables.iter().map(|t| t.to_string()).collect(); - assert_eq!(tables, ["a"]); - let versioned: Vec<_> = references - .versioned_tables - .iter() - .map(|(t, v)| format!("{t} {v}")) - .collect(); + WHERE a.x IN (SELECT x FROM c VERSION AS OF 2)"; assert_eq!( - versioned, + references_of(query), [ - "b TIMESTAMP AS OF now()", - "b AT(VERSION => 1)", - "c VERSION AS OF 2" + "a: a", + "b: b AT(VERSION => 1)", + "b: B AT(VERSION => 1)", + "b: b TIMESTAMP AS OF now()", + "c: c VERSION AS OF 2" ] ); - assert!(references.ctes.is_empty()); - // `resolve_table_references` returns all tables, without versions - let (tables, ctes) = resolve_table_references(&statement, true).unwrap(); + // `resolve_table_references` returns all tables, without their table + // factors + let (tables, ctes) = + resolve_table_references(&parse_snowflake(query), true).unwrap(); let tables: Vec<_> = tables.iter().map(|t| t.to_string()).collect(); assert_eq!(tables, ["a", "b", "c"]); - assert_eq!(ctes, references.ctes); + assert!(ctes.is_empty()); } #[test] - fn resolve_table_references_with_versions_mixed() { - // A table referenced both with and without a time travel clause - let statement = parse_snowflake( - "SELECT * FROM t AS cur JOIN t VERSION AS OF 1 AS old ON true", + fn resolve_references_other_references() { + // Tables referenced other than by a table factor + assert_eq!( + references_of("INSERT INTO t SELECT * FROM t AS cur JOIN t AS old ON true"), + ["table t", "t: t AS cur", "t: t AS old"] ); - let references = - resolve_table_references_with_versions(&statement, true).unwrap(); - let tables: Vec<_> = references.tables.iter().map(|t| t.to_string()).collect(); - assert_eq!(tables, ["t"]); - let versioned: Vec<_> = references - .versioned_tables - .iter() - .map(|(t, v)| format!("{t} {v}")) - .collect(); - assert_eq!(versioned, ["t VERSION AS OF 1"]); } #[test] - fn resolve_table_references_with_versions_subquery_in_version() { - // Tables in a subquery in the time travel clause are still recorded. + fn resolve_references_subquery_in_version() { + // Tables in a subquery in a time travel clause are still reported. // Planning reports an error for the non-constant version. - let tables_of = |query: &str| { - let references = - resolve_table_references_with_versions(&parse_snowflake(query), true) - .unwrap(); - let tables: Vec<_> = - references.tables.iter().map(|t| t.to_string()).collect(); - let versioned: Vec<_> = references - .versioned_tables - .iter() - .map(|(t, _)| t.to_string()) - .collect(); - (tables, versioned) - }; - - let (tables, versioned) = - tables_of("SELECT * FROM t TIMESTAMP AS OF (SELECT max(ts) FROM log)"); - assert_eq!(tables, ["log"]); - assert_eq!(versioned, ["t"]); - - let (tables, versioned) = tables_of( - "SELECT * FROM t AT (VERSION => (SELECT max(v) FROM a JOIN b ON true))", + assert_eq!( + references_of("SELECT * FROM t TIMESTAMP AS OF (SELECT max(ts) FROM log)"), + ["t: t TIMESTAMP AS OF (SELECT max(ts) FROM log)", "log: log"] + ); + assert_eq!( + references_of( + "SELECT * FROM t AT (VERSION => \ + (SELECT max(v) FROM log VERSION AS OF 1 JOIN a ON true))" + ), + [ + "t: t AT(VERSION => (SELECT max(v) FROM log VERSION AS OF 1 JOIN a ON true))", + "log: log VERSION AS OF 1", + "a: a" + ] + ); + } + + #[test] + fn resolve_references_visits_ctes_once() { + // The body of a non-recursive CTE is visited before the CTE is in + // scope, and not again afterwards + assert_eq!( + references_of("WITH a AS (SELECT * FROM t VERSION AS OF 1) SELECT * FROM a"), + ["t: t VERSION AS OF 1", "cte a"] ); - assert_eq!(tables, ["a", "b"]); - assert_eq!(versioned, ["t"]); - let (tables, versioned) = tables_of( - "SELECT * FROM t AT (VERSION => \ - (SELECT max(v) FROM log VERSION AS OF 1 JOIN a ON true))", + // The example of "Non-recursive CTEs" on `RelationVisitor` + assert_eq!( + references_of( + "WITH a AS (WITH b AS (SELECT * FROM t) SELECT * FROM b, c, u), \ + c AS (SELECT * FROM a) \ + SELECT * FROM c" + ), + ["t: t", "c: c", "u: u", "cte b", "cte c", "cte a"] + ); + + // The body of a recursive CTE is only visited with the CTE in scope + assert_eq!( + references_of( + "WITH RECURSIVE a AS (SELECT * FROM t UNION ALL SELECT * FROM a) \ + SELECT * FROM a" + ), + ["t: t", "cte a"] ); - assert_eq!(tables, ["a"]); - assert_eq!(versioned, ["log", "t"]); } #[test] - fn resolve_table_references_with_versions_skips_ctes() { - // A versioned reference to a CTE is not a table reference. Planning - // reports an error for it. - let statement = parse_snowflake( - "WITH t AS (SELECT * FROM t VERSION AS OF 1) SELECT * FROM t VERSION AS OF 2", + fn resolve_references_skips_cte_references() { + // A reference to a CTE is not a table reference. Planning reports an + // error for its time travel clause. + assert_eq!( + references_of( + "WITH t AS (SELECT * FROM t VERSION AS OF 1) \ + SELECT * FROM t VERSION AS OF 2" + ), + ["t: t VERSION AS OF 1", "cte t"] ); - let references = - resolve_table_references_with_versions(&statement, true).unwrap(); - let versioned: Vec<_> = references - .versioned_tables - .iter() - .map(|(t, v)| format!("{t} {v}")) - .collect(); - assert_eq!(versioned, ["t VERSION AS OF 1"]); - assert_eq!(references.ctes.len(), 1); } #[test] diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 6aadcdf317f63..f16927e89490f 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -853,7 +853,7 @@ impl SqlToRel<'_, S> { .map(|t| self.convert_data_type_to_field(&t)) .collect::>()?; - self.check_prepared_table_versions(&statement)?; + self.check_prepared_read_selectors(&statement)?; // Create planner context with parameters let mut planner_context = PlannerContext::new() diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rs index 7d9dd9ef85c47..07c49f0fefa30 100644 --- a/datafusion/sql/src/unparser/dialect.rs +++ b/datafusion/sql/src/unparser/dialect.rs @@ -362,8 +362,8 @@ pub enum CharacterLengthStyle { CharacterLength, } -/// `TableVersionStyle` to use for unparsing the time travel clause of a table -/// scan, such as `VERSION AS OF 3` +/// Syntax to use for unparsing the time travel clause of a table scan, such as +/// `VERSION AS OF 3` #[derive(Clone, Copy, PartialEq)] pub enum TableVersionStyle { /// `AT (VERSION => v)` and `AT (TIMESTAMP => t)`, as used by DuckDB @@ -1273,7 +1273,7 @@ impl CustomDialectBuilder { self } - /// Customize the dialect with a specific time travel syntax listed in `TableVersionStyle` + /// Customize the dialect with a specific time travel syntax listed in [`TableVersionStyle`] pub fn with_table_version_style( mut self, table_version_style: TableVersionStyle, diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 037ab4c11242d..bf45b1a924753 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -31,8 +31,9 @@ use crate::unparser::{ }; use crate::utils::UNNEST_PLACEHOLDER; use datafusion_common::{ - Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference, TableVersion, - assert_or_internal_err, internal_datafusion_err, internal_err, not_impl_err, + Column, DFSchema, DataFusionError, Result, ScalarValue, TableReadSelector, + TableReference, assert_or_internal_err, internal_datafusion_err, internal_err, + not_impl_err, tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion}, utils::combine_limit, }; @@ -872,8 +873,8 @@ impl Unparser<'_> { builder.name(ast::ObjectName::from(table_parts)); builder.version( scan.source - .version() - .map(|v| self.table_version_to_sql(v)) + .read_selector() + .map(|v| self.read_selector_to_sql(v)) .transpose()?, ); relation.table(builder); @@ -2344,15 +2345,24 @@ impl Unparser<'_> { Ok(Some(flatten)) } - /// Converts the time travel clause of a table scan to SQL, using the - /// syntax of the dialect - fn table_version_to_sql(&self, version: &TableVersion) -> Result { - let value = self.expr_to_sql(&Expr::Literal(version.value(), None))?; - Ok(match (self.dialect.table_version_style(), version) { + /// Converts the selector of a table scan to a time travel clause, using + /// the syntax of the dialect + fn read_selector_to_sql( + &self, + selector: &TableReadSelector, + ) -> Result { + let value = match selector { + TableReadSelector::AsOfVersion(value) => value.clone(), + TableReadSelector::AsOfTimestamp(ns) => { + ScalarValue::TimestampNanosecond(Some(*ns), Some("UTC".into())) + } + }; + let value = self.expr_to_sql(&Expr::Literal(value, None))?; + Ok(match (self.dialect.table_version_style(), selector) { (TableVersionStyle::At, _) => { - let name = match version { - TableVersion::Version(_) => "VERSION", - TableVersion::Timestamp(_) => "TIMESTAMP", + let name = match selector { + TableReadSelector::AsOfVersion(_) => "VERSION", + TableReadSelector::AsOfTimestamp(_) => "TIMESTAMP", }; ast::TableVersion::Function(ast::Expr::Function(ast::Function { name: ast::ObjectName::from(vec![Ident::new("AT")]), @@ -2373,18 +2383,18 @@ impl Unparser<'_> { within_group: vec![], })) } - (TableVersionStyle::AsOf, TableVersion::Version(_)) => { + (TableVersionStyle::AsOf, TableReadSelector::AsOfVersion(_)) => { ast::TableVersion::VersionAsOf(value) } - (TableVersionStyle::AsOf, TableVersion::Timestamp(_)) => { + (TableVersionStyle::AsOf, TableReadSelector::AsOfTimestamp(_)) => { ast::TableVersion::TimestampAsOf(value) } - (TableVersionStyle::ForSystemTime, TableVersion::Timestamp(_)) => { + (TableVersionStyle::ForSystemTime, TableReadSelector::AsOfTimestamp(_)) => { ast::TableVersion::ForSystemTimeAsOf(value) } - (TableVersionStyle::ForSystemTime, TableVersion::Version(_)) => { + (TableVersionStyle::ForSystemTime, TableReadSelector::AsOfVersion(_)) => { return not_impl_err!( - "This dialect does not support time travel to version {version}" + "This dialect does not support time travel to version {selector}" ); } }) diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs index 73d69df2b4f08..53f2da257c4b2 100644 --- a/datafusion/sql/tests/cases/plan_to_sql.rs +++ b/datafusion/sql/tests/cases/plan_to_sql.rs @@ -6,8 +6,8 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::tree_node::{Transformed, TransformedResult}; use datafusion_common::{ - Column, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, TableReference, - TableVersion, assert_contains, + Column, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, + TableReadSelector, TableReference, assert_contains, }; use datafusion_expr::expr::{WindowFunction, WindowFunctionParams}; use datafusion_expr::test::function_stub::{ @@ -1700,7 +1700,7 @@ fn test_table_scan_version_to_sql() -> Result<()> { /// A source that reads a specific version of a table struct VersionedSource { schema: SchemaRef, - version: TableVersion, + selector: TableReadSelector, } impl TableSource for VersionedSource { @@ -1708,15 +1708,15 @@ fn test_table_scan_version_to_sql() -> Result<()> { Arc::clone(&self.schema) } - fn version(&self) -> Option<&TableVersion> { - Some(&self.version) + fn read_selector(&self) -> Option<&TableReadSelector> { + Some(&self.selector) } } - let scan = |version: TableVersion, filters: Vec| -> Result { + let scan = |selector: TableReadSelector, filters: Vec| -> Result { let source = Arc::new(VersionedSource { schema: Arc::new(schema.clone()), - version, + selector, }); Ok(LogicalPlan::TableScan( TableScanBuilder::new("t1", source) @@ -1724,10 +1724,10 @@ fn test_table_scan_version_to_sql() -> Result<()> { .build()?, )) }; - let version = TableVersion::Version(ScalarValue::Int64(Some(3))); - let tag = TableVersion::Version(ScalarValue::from("v1")); + let version = TableReadSelector::AsOfVersion(ScalarValue::Int64(Some(3))); + let tag = TableReadSelector::AsOfVersion(ScalarValue::from("v1")); // 2024-01-01T00:00:00Z - let timestamp = TableVersion::Timestamp(1_704_067_200_000_000_000); + let timestamp = TableReadSelector::AsOfTimestamp(1_704_067_200_000_000_000); let sql = plan_to_sql(&scan(version.clone(), vec![])?)?; assert_snapshot!(sql, @"SELECT * FROM t1 AT(VERSION => 3)"); diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index 58af0fa63ab24..2206858a47fb7 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -13,11 +13,10 @@ use datafusion_common::config::ConfigOptions; use datafusion_common::datatype::DataTypeExt; use datafusion_common::file_options::file_type::FileType; use datafusion_common::{ - DFSchema, GetExt, Result, TableReference, TableVersion, not_impl_err, plan_err, -}; -use datafusion_expr::planner::{ - ExprPlanner, PlannerResult, TableVersionExpr, TypePlanner, + DFSchema, GetExt, Result, ScalarValue, TableReadSelector, TableReference, + not_impl_err, plan_err, }; +use datafusion_expr::planner::{ExprPlanner, PlannerResult, TypePlanner}; use datafusion_expr::{ AggregateUDF, Expr, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF, }; @@ -337,32 +336,23 @@ impl ContextProvider for MockContextProvider { Ok(Arc::new(EmptyTable::new(schema))) } - /// Only literal versions are supported, as evaluating expressions - /// requires an `ExprSimplifier` - fn evaluate_table_version(&self, version: TableVersionExpr) -> Result { - match version { - TableVersionExpr::Version(Expr::Literal(value, _)) => { - Ok(TableVersion::Version(value)) - } - version => not_impl_err!("Unsupported table version: {version:?}"), + /// Only literals are supported, as evaluating expressions requires an + /// `ExprSimplifier` + fn evaluate_const_expr(&self, expr: Expr) -> Result { + match expr { + Expr::Literal(value, _) => Ok(value), + expr => not_impl_err!("Unsupported constant: {expr:?}"), } } - /// Only `person` supports time travel. For `j1`, the current version is - /// returned without reporting a version, which is invalid. - fn get_table_source_at( + /// Returns the current table without reporting a selector, which is + /// invalid. The planner must reject it. + fn get_table_source_with_read_selector( &self, name: TableReference, - version: &TableVersion, + _selector: &TableReadSelector, ) -> Result> { - match name.table() { - "person" => Ok(Arc::new(EmptyTable::new_with_version( - self.get_table_source(name)?.schema(), - version.clone(), - ))), - "j1" => self.get_table_source(name), - _ => not_impl_err!("Time travel is not supported for table '{name}'"), - } + self.get_table_source(name) } fn udf_names(&self) -> Vec { @@ -396,22 +386,11 @@ impl ContextProvider for MockContextProvider { struct EmptyTable { table_schema: SchemaRef, - version: Option, } impl EmptyTable { fn new(table_schema: SchemaRef) -> Self { - Self { - table_schema, - version: None, - } - } - - fn new_with_version(table_schema: SchemaRef, version: TableVersion) -> Self { - Self { - table_schema, - version: Some(version), - } + Self { table_schema } } } @@ -419,10 +398,6 @@ impl TableSource for EmptyTable { fn schema(&self) -> SchemaRef { Arc::clone(&self.table_schema) } - - fn version(&self) -> Option<&TableVersion> { - self.version.as_ref() - } } #[derive(Debug)] diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 798aa97c60be1..6b94546642fa4 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -6175,25 +6175,15 @@ fn test_reserved_column_name_explain_copy() { ); } -#[test] -fn test_time_travel() { - let dialect = &DatabricksDialect {}; - - // The version is reported by the source and shown with the scan. It does - // not change how columns are qualified. - let plan = logical_plan_with_dialect( - "SELECT person.id FROM person VERSION AS OF 3 WHERE person.age > 21", - dialect, +// A source that does not report the selector it reads is rejected. With a +// `SessionState`, the catalog's table provider is checked before planning +// instead, see `time_travel.slt`. +#[test] +fn test_time_travel_source_must_report_read_selector() { + let err = logical_plan_with_dialect( + "SELECT * FROM person VERSION AS OF 3", + &DatabricksDialect {}, ) - .unwrap(); - assert_snapshot!(plan, @r" - Projection: person.id - Filter: person.age > Int64(21) - TableScan: person@3 - "); - - // A source that does not report the version it reads is rejected - let err = logical_plan_with_dialect("SELECT * FROM j1 VERSION AS OF 3", dialect) - .unwrap_err(); - assert_snapshot!(err.strip_backtrace(), @"Error during planning: The source of table 'j1@3' must report the version it reads"); + .unwrap_err(); + assert_snapshot!(err.strip_backtrace(), @"Error during planning: The source of table 'person@3' must report the selector it reads"); } diff --git a/datafusion/sqllogictest/src/test_context/time_travel.rs b/datafusion/sqllogictest/src/test_context/time_travel.rs index a7cb4448962d5..9487fcfc7c3c5 100644 --- a/datafusion/sqllogictest/src/test_context/time_travel.rs +++ b/datafusion/sqllogictest/src/test_context/time_travel.rs @@ -4,12 +4,15 @@ use std::sync::Arc; -use arrow::array::{Int64Array, StringArray}; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::array::{ + ArrayRef, Float32Array, Float64Array, Int32Array, Int64Array, StringArray, + StructArray, +}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion::catalog::SchemaProvider; -use datafusion::common::{Result, ScalarValue, TableVersion, plan_err}; +use datafusion::common::{Result, ScalarValue, TableReadSelector, plan_err}; use datafusion::datasource::{MemTable, TableProvider}; use datafusion::prelude::SessionContext; @@ -47,11 +50,17 @@ const VERSIONS: [Version; 3] = [ /// Named versions (tags) of table `t` const TAGS: [(&str, i64); 2] = [("first", 1), ("latest", 3)]; -/// A schema with a single table `t` that supports time travel queries. +/// The latest version of table `evolving` +const EVOLVING_VERSIONS: i64 = 5; + +/// A schema with tables that support time travel queries. /// -/// The table has three versions, which can be selected by version number, by -/// tag or by timestamp. Time travel on table `no_version` returns a table that -/// does not report its version, which is invalid. +/// Table `t` has three versions, which can be selected by version number, by +/// tag or by timestamp. Table `evolving` has five versions with different +/// schemas, as created by the schema evolution of table formats such as +/// Iceberg, and can be selected by version number. Time travel on table +/// `no_version` returns a table that does not report its version, which is +/// invalid. #[derive(Debug)] struct VersionedSchema {} @@ -76,9 +85,157 @@ impl VersionedSchema { )?; // Report the version that is read, which may differ from the requested // version, e.g. when it was requested by tag or timestamp - let table = MemTable::try_new(schema, vec![vec![batch]])?.with_version(Some( - TableVersion::Version(ScalarValue::Int64(Some(version))), - )); + let table = + MemTable::try_new(schema, vec![vec![batch]])?.with_read_selector(Some( + TableReadSelector::AsOfVersion(ScalarValue::Int64(Some(version))), + )); + Ok(Arc::new(table)) + } + + /// Returns a version of table `evolving`, in which: + /// + /// 1. the table is created as `(id INT, name VARCHAR NOT NULL, score FLOAT)` + /// 2. column `email` is added + /// 3. column `name` is renamed to `full_name`, and `id` and `score` are + /// widened to `BIGINT` and `DOUBLE` + /// 4. column `score` is dropped, `full_name` becomes nullable and struct + /// column `address` is added + /// 5. column `email` is moved first, and field `zip` is added to `address` + fn evolving_version(version: i64) -> Result> { + let emails = || -> ArrayRef { + Arc::new(StringArray::from(vec![ + None, + Some("b@example.com"), + Some("c@example.com"), + ])) + }; + let address = |zips: bool| -> ArrayRef { + let mut fields = vec![( + Arc::new(Field::new("city", DataType::Utf8, true)), + Arc::new(StringArray::from(vec![ + Some("Ghent"), + Some("Leuven"), + None, + Some("Brussels"), + ])) as ArrayRef, + )]; + if zips { + fields.push(( + Arc::new(Field::new("zip", DataType::Utf8, true)), + Arc::new(StringArray::from(vec![ + Some("9000"), + Some("3000"), + None, + Some("1000"), + ])) as ArrayRef, + )); + } + Arc::new(StructArray::from(fields)) + }; + let address_type = |zips: bool| { + let mut fields = vec![Field::new("city", DataType::Utf8, true)]; + if zips { + fields.push(Field::new("zip", DataType::Utf8, true)); + } + DataType::Struct(Fields::from(fields)) + }; + let (fields, columns): (Vec, Vec) = match version { + 1 => ( + vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Float32, false), + ], + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["a", "b"])), + Arc::new(Float32Array::from(vec![1.5, 2.5])), + ], + ), + 2 => ( + vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("score", DataType::Float32, false), + Field::new("email", DataType::Utf8, true), + ], + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(Float32Array::from(vec![1.5, 2.5, 3.5])), + emails(), + ], + ), + 3 => ( + vec![ + Field::new("id", DataType::Int64, false), + Field::new("full_name", DataType::Utf8, false), + Field::new("score", DataType::Float64, false), + Field::new("email", DataType::Utf8, true), + ], + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(Float64Array::from(vec![1.5, 2.5, 3.5])), + emails(), + ], + ), + 4 => ( + vec![ + Field::new("id", DataType::Int64, false), + Field::new("full_name", DataType::Utf8, true), + Field::new("email", DataType::Utf8, true), + Field::new("address", address_type(false), true), + ], + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4])), + Arc::new(StringArray::from(vec![ + Some("a"), + Some("b"), + Some("c"), + None, + ])), + Arc::new(StringArray::from(vec![ + None, + Some("b@example.com"), + Some("c@example.com"), + Some("d@example.com"), + ])), + address(false), + ], + ), + 5 => ( + vec![ + Field::new("email", DataType::Utf8, true), + Field::new("id", DataType::Int64, false), + Field::new("full_name", DataType::Utf8, true), + Field::new("address", address_type(true), true), + ], + vec![ + Arc::new(StringArray::from(vec![ + None, + Some("b@example.com"), + Some("c@example.com"), + Some("d@example.com"), + ])), + Arc::new(Int64Array::from(vec![1, 2, 3, 4])), + Arc::new(StringArray::from(vec![ + Some("a"), + Some("b"), + Some("c"), + None, + ])), + address(true), + ], + ), + _ => return plan_err!("Table evolving has no version {version}"), + }; + let schema = Arc::new(Schema::new(fields)); + let batch = RecordBatch::try_new(Arc::clone(&schema), columns)?; + let table = + MemTable::try_new(schema, vec![vec![batch]])?.with_read_selector(Some( + TableReadSelector::AsOfVersion(ScalarValue::Int64(Some(version))), + )); Ok(Arc::new(table)) } } @@ -86,32 +243,46 @@ impl VersionedSchema { #[async_trait] impl SchemaProvider for VersionedSchema { fn table_names(&self) -> Vec { - vec!["t".to_string()] + vec!["t".to_string(), "evolving".to_string()] } async fn table(&self, name: &str) -> Result>> { - if name != "t" { - return Ok(None); + match name { + "t" => Self::table_version(VERSIONS[VERSIONS.len() - 1].version).map(Some), + "evolving" => Self::evolving_version(EVOLVING_VERSIONS).map(Some), + _ => Ok(None), } - Self::table_version(VERSIONS[VERSIONS.len() - 1].version).map(Some) } - async fn table_at( + async fn table_with_read_selector( &self, name: &str, - version: &TableVersion, + selector: &TableReadSelector, ) -> Result>> { if name == "no_version" { - // Invalid: the provider does not report the version it reads + // Invalid: the provider does not report the selector it reads let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); return Ok(Some(Arc::new(MemTable::try_new(schema, vec![vec![]])?))); } + if name == "evolving" { + return match selector { + TableReadSelector::AsOfVersion(value) => { + match value.cast_to(&DataType::Int64)? { + ScalarValue::Int64(Some(version)) => { + Self::evolving_version(version).map(Some) + } + _ => plan_err!("Invalid version {value}"), + } + } + _ => plan_err!("Table read selector {selector} is not supported"), + }; + } if name != "t" { return Ok(None); } - let version = match version { - TableVersion::Version(value) => { + let version = match selector { + TableReadSelector::AsOfVersion(value) => { if let Some(Some(tag)) = value.try_as_str() { match TAGS.iter().find(|(t, _)| *t == tag) { Some((_, version)) => *version, @@ -124,10 +295,10 @@ impl SchemaProvider for VersionedSchema { } } } - TableVersion::Timestamp(ts) => { + TableReadSelector::AsOfTimestamp(ts) => { match VERSIONS.iter().rev().find(|v| v.committed <= *ts) { Some(v) => v.version, - None => return plan_err!("Table t did not exist at {version}"), + None => return plan_err!("Table t did not exist at {selector}"), } } }; @@ -135,7 +306,7 @@ impl SchemaProvider for VersionedSchema { } fn table_exist(&self, name: &str) -> bool { - name == "t" + name == "t" || name == "evolving" } } diff --git a/datafusion/sqllogictest/test_files/time_travel.slt b/datafusion/sqllogictest/test_files/time_travel.slt index 085b76875fe75..a8e5c1a2a8c53 100644 --- a/datafusion/sqllogictest/test_files/time_travel.slt +++ b/datafusion/sqllogictest/test_files/time_travel.slt @@ -204,6 +204,184 @@ SELECT * FROM first_version statement ok DROP VIEW first_version; +# Time travel clauses in CTE bodies +query IT rowsort +WITH old AS (SELECT * FROM versioned.t VERSION AS OF 1) SELECT * FROM old +---- +1 a + +statement ok +SET datafusion.catalog.default_schema = 'versioned' + +# A CTE body is not in the scope of its own name, so `t` in the body is the +# table, while `t` in the query is the CTE +query IT rowsort +WITH t AS (SELECT * FROM t VERSION AS OF 1) SELECT * FROM t +---- +1 a + +# Nor in the scope of CTEs defined after it: `t VERSION AS OF 2` in the body +# of `a` is the table, not the CTE `t` +query IT rowsort +WITH a AS ( + WITH b AS (SELECT * FROM t VERSION AS OF 1) + SELECT * FROM b UNION ALL SELECT * FROM t VERSION AS OF 2 +), +t AS (SELECT * FROM a) +SELECT * FROM t +---- +1 a +1 a +2 b + +statement ok +SET datafusion.catalog.default_schema = 'public' + +########## +## Schema evolution +## +## Table `evolving` has five versions with different schemas, as created by +## the schema evolution of table formats such as Iceberg: +## +## | version | change | +## |---------|---------------------------------------------------------------| +## | 1 | created as (id INT, name VARCHAR NOT NULL, score FLOAT) | +## | 2 | email added | +## | 3 | name renamed to full_name, id and score widened | +## | 4 | score dropped, full_name nullable, struct address(city) added | +## | 5 | email moved first, field zip added to address | +########## + +# Each version is read with its own schema +query ITR rowsort +SELECT * FROM versioned.evolving VERSION AS OF 1 +---- +1 a 1.5 +2 b 2.5 + +query ITRT rowsort +SELECT * FROM versioned.evolving VERSION AS OF 2 +---- +1 a 1.5 NULL +2 b 2.5 b@example.com +3 c 3.5 c@example.com + +query ITRT rowsort +SELECT * FROM versioned.evolving VERSION AS OF 3 +---- +1 a 1.5 NULL +2 b 2.5 b@example.com +3 c 3.5 c@example.com + +query ITT? rowsort +SELECT * FROM versioned.evolving VERSION AS OF 4 +---- +1 a NULL {city: Ghent} +2 b b@example.com {city: Leuven} +3 c c@example.com {city: NULL} +4 NULL d@example.com {city: Brussels} + +# The current version, with email moved first +query TIT? rowsort +SELECT * FROM versioned.evolving +---- +NULL 1 a {city: Ghent, zip: 9000} +b@example.com 2 b {city: Leuven, zip: 3000} +c@example.com 3 c {city: NULL, zip: NULL} +d@example.com 4 NULL {city: Brussels, zip: 1000} + +# An added column does not exist in earlier versions +statement error No field named email +SELECT email FROM versioned.evolving VERSION AS OF 1 + +# A renamed column has its old name in earlier versions, and its new name in +# later versions +statement error No field named full_name +SELECT full_name FROM versioned.evolving VERSION AS OF 2 + +statement error No field named name +SELECT name FROM versioned.evolving VERSION AS OF 3 + +# A dropped column does not exist in later versions +statement error No field named score +SELECT score FROM versioned.evolving VERSION AS OF 4 + +# Widened types +query TT +SELECT arrow_typeof(id), arrow_typeof(score) FROM versioned.evolving VERSION AS OF 2 LIMIT 1 +---- +Int32 Float32 + +query TT +SELECT arrow_typeof(id), arrow_typeof(score) FROM versioned.evolving VERSION AS OF 3 LIMIT 1 +---- +Int64 Float64 + +# A required column that became optional +query I +SELECT count(*) FROM versioned.evolving VERSION AS OF 4 WHERE full_name IS NULL +---- +1 + +# A field added to a struct does not exist in earlier versions +statement error DataFusion error: Error during planning: Field zip not found in struct +SELECT address['zip'] FROM versioned.evolving VERSION AS OF 4 + +query TT rowsort +SELECT address['city'], address['zip'] FROM versioned.evolving +---- +Brussels 1000 +Ghent 9000 +Leuven 3000 +NULL NULL + +# Versions with different schemas can be combined in one query: a column +# renamed between the versions +query ITT rowsort +SELECT old.id, old.name, cur.full_name +FROM versioned.evolving VERSION AS OF 2 AS old +JOIN versioned.evolving AS cur ON old.id = cur.id +---- +1 a a +2 b b +3 c c + +# A column widened between the versions +query IT rowsort +SELECT id, arrow_typeof(id) FROM ( + SELECT id FROM versioned.evolving VERSION AS OF 1 + UNION ALL + SELECT id FROM versioned.evolving +) +---- +1 Int64 +1 Int64 +2 Int64 +2 Int64 +3 Int64 +4 Int64 + +# Rows added between the versions, while id was widened and moved +query I rowsort +SELECT id FROM versioned.evolving EXCEPT SELECT id FROM versioned.evolving VERSION AS OF 1 +---- +3 +4 + +# Views capture the schema of their version +statement ok +CREATE VIEW evolving_v1 AS SELECT * FROM versioned.evolving VERSION AS OF 1; + +query TTT +DESCRIBE evolving_v1 +---- +id Int32 NO +name Utf8 NO +score Float32 NO + +statement ok +DROP VIEW evolving_v1; + ########## ## EXPLAIN shows the resolved version ########## @@ -226,7 +404,8 @@ physical_plan 01)FilterExec: id@0 = 1, projection=[name@1] 02)--DataSourceExec: partitions=1, partition_sizes=[1] -# The timestamp resolves to version 1, the latest version committed before it +# The timestamp resolves to version 1, the latest version committed at or +# before it query TT EXPLAIN SELECT * FROM versioned.t TIMESTAMP AS OF '2024-01-15T12:30:00+01:00' ---- @@ -249,30 +428,37 @@ SELECT * FROM versioned.t TIMESTAMP AS OF '2023-01-01' statement error DataFusion error: Error during planning: table 'datafusion.versioned.unknown@1' not found SELECT * FROM versioned.unknown VERSION AS OF 1 -# Catalogs must return a table that reports the version it reads -statement error DataFusion error: Error during planning: The source of table 'versioned.no_version@1' must report the version it reads +# Catalogs must return a table that reports the selector it reads +statement error DataFusion error: Error during planning: The table provider of 'datafusion\.versioned\.no_version@1' must report the selector it reads SELECT * FROM versioned.no_version VERSION AS OF 1 # Tables that do not support time travel statement ok CREATE TABLE plain AS VALUES (1); -statement error DataFusion error: This feature is not implemented: Time travel is not supported for table 'plain@1' +statement error DataFusion error: This feature is not implemented: Table read selectors are not supported for table 'plain@1' SELECT * FROM plain VERSION AS OF 1 statement ok DROP TABLE plain; -# The version must be a non-null constant +# The version must be a constant, which is not NULL statement error DataFusion error: Schema error: No field named id\. SELECT * FROM versioned.t AT (VERSION => id) -statement error DataFusion error: Error during planning: Time travel version must evaluate to a non-null constant, got: NULL +statement error DataFusion error: Error during planning: The version of a time travel clause must not be NULL: versioned\.t AT\(VERSION => NULL\) SELECT * FROM versioned.t AT (VERSION => NULL) -statement error DataFusion error: Error during planning: Time travel version must evaluate to a non-null constant +statement error DataFusion error: Error during planning: The timestamp of a time travel clause must not be NULL: versioned\.t TIMESTAMP AS OF NULL +SELECT * FROM versioned.t TIMESTAMP AS OF NULL + +statement error DataFusion error: Error during planning: Expression must evaluate to a constant, got: random\(\) SELECT * FROM versioned.t AT (VERSION => random()) +# Errors evaluating the expression are reported as is +statement error DataFusion error: Arrow error: Divide by zero error +SELECT * FROM versioned.t AT (VERSION => 1 / 0) + statement error Error parsing timestamp from 'yesterday' SELECT * FROM versioned.t AT (TIMESTAMP => 'yesterday') @@ -316,22 +502,27 @@ statement ok DEALLOCATE versioned_lookup; # Parameters are not supported in time travel clauses -statement error DataFusion error: This feature is not implemented: Parameters are not supported in time travel clauses: versioned\.t +statement error DataFusion error: This feature is not implemented: Parameters are not supported in table read selectors: versioned\.t AT\(VERSION => \$1\) PREPARE p AS SELECT * FROM versioned.t AT (VERSION => $1) # Clauses that depend on the time of execution would be evaluated when the # statement is prepared instead -statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution, such as with now\(\): AT\(TIMESTAMP => now\(\) \- INTERVAL '1 hour'\) +statement error DataFusion error: This feature is not implemented: Table read selectors in prepared statements must not depend on the time of execution, such as with now\(\): versioned\.t AT\(TIMESTAMP => now\(\) \- INTERVAL '1 hour'\) PREPARE p AS SELECT * FROM versioned.t AT (TIMESTAMP => now() - INTERVAL '1 hour') # Also within subqueries and DML statements -statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution +statement error DataFusion error: This feature is not implemented: Table read selectors in prepared statements must not depend on the time of execution PREPARE p AS SELECT * FROM (SELECT * FROM versioned.t AT (TIMESTAMP => now())) +# References to CTEs are read without a selector, so their time travel clause is +# rejected as for unprepared statements +statement error DataFusion error: Error during planning: Time travel is not supported for CTE 'c' +PREPARE p AS WITH c AS (SELECT 1) SELECT * FROM c AT (TIMESTAMP => now()) + statement ok CREATE TABLE copied (id BIGINT, name VARCHAR); -statement error DataFusion error: This feature is not implemented: Time travel clauses in prepared statements must not depend on the time of execution +statement error DataFusion error: This feature is not implemented: Table read selectors in prepared statements must not depend on the time of execution PREPARE p AS INSERT INTO copied SELECT * FROM versioned.t AT (TIMESTAMP => now()) statement ok diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs index 2308194ffac6d..61f29a06b5679 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs @@ -50,9 +50,9 @@ pub fn from_table_scan( producer: &mut impl SubstraitProducer, scan: &TableScan, ) -> datafusion::common::Result> { - if let Some(version) = scan.source.version() { + if let Some(selector) = scan.source.read_selector() { return not_impl_err!( - "Time travel table scans are not supported in Substrait: {}@{version}", + "Table scans with a read selector are not supported in Substrait: {}@{selector}", scan.table_name ); } diff --git a/docs/source/library-user-guide/catalogs.md b/docs/source/library-user-guide/catalogs.md index 041e2907e66b8..1c2e659843251 100644 --- a/docs/source/library-user-guide/catalogs.md +++ b/docs/source/library-user-guide/catalogs.md @@ -203,59 +203,70 @@ impl SchemaProvider for Schema { } ``` -### Time travel +### Table read selectors and time travel -Time travel queries, such as `SELECT * FROM t VERSION AS OF 3` or -`SELECT * FROM t TIMESTAMP AS OF '2024-01-01'`, read a table as it existed at an -earlier version or point in time. DataFusion evaluates the version expression -during planning and passes the resulting [`TableVersion`] to -`SchemaProvider::table_at`. A `TableVersion` is either a `Version`, holding any -`ScalarValue` such as a snapshot id or a tag, or a `Timestamp` in nanoseconds -since the Unix epoch (UTC). +A [`TableReadSelector`] specifies which read of a table to perform. For example, time +travel queries such as `SELECT * FROM t VERSION AS OF 3` or +`SELECT * FROM t TIMESTAMP AS OF '2024-01-01'` read a table as it existed at an +earlier version or point in time. DataFusion evaluates the expression of the +time travel clause during planning and passes the resulting `TableReadSelector` to +`SchemaProvider::table_with_read_selector`. A `TableReadSelector` is one of: -The default implementation of `table_at` returns a "not implemented" error. -Schema providers that support time travel return a table provider that reads the -requested version of the table. +- an `AsOfVersion`, holding any `ScalarValue` such as a snapshot id, a tag, a branch + name or a commit hash, or +- an `AsOfTimestamp` in nanoseconds since the Unix epoch (UTC). -The returned provider must report the version it reads from -`TableProvider::version`. This may differ from the requested version: a tag or a -timestamp typically resolves to a snapshot id. The built-in providers, such as -`ListingTable` and `MemTable`, report the version set with `with_version`: +An `AsOfTimestamp` selects the latest state committed at or before that time, so +a commit made exactly at that time is included. + +A selector determines which data is read, not how it is read: hints about how +to scan a table do not belong in a selector. The schema of the table may differ +between selectors, for example after the schema of the table evolved. + +The default implementation of `table_with_read_selector` returns a "not implemented" error. +Schema providers that support table read selectors return a table provider that +performs the requested read. + +The returned provider must report the selector it reads from +`TableProvider::read_selector`. This may differ from the requested selector: a tag +or a timestamp typically resolves to a snapshot id. The built-in providers, such +as `ListingTable` and `MemTable`, report the selector set with `with_read_selector`: ```rust # use async_trait::async_trait; # use std::sync::Arc; # use arrow::datatypes::DataType; # use datafusion::catalog::{MemTable, TableProvider, SchemaProvider}; -# use datafusion::common::{Result, ScalarValue, TableVersion}; +# use datafusion::common::{Result, ScalarValue, TableReadSelector}; # # #[derive(Debug)] # struct Schema {} # # impl Schema { # fn snapshot(&self, _name: &str, _id: i64) -> Result> { todo!() } -# fn latest_snapshot_before(&self, _nanos: i64) -> Result { todo!() } +# fn latest_snapshot_at(&self, _nanos: i64) -> Result { todo!() } # } # #[async_trait] impl SchemaProvider for Schema { - async fn table_at( + async fn table_with_read_selector( &self, name: &str, - version: &TableVersion, + selector: &TableReadSelector, ) -> Result>> { - // Resolve the requested version to a snapshot id - let id = match version { - TableVersion::Version(value) => { + // Resolve the requested selector to a snapshot id + let id = match selector { + TableReadSelector::AsOfVersion(value) => { i64::try_from(value.cast_to(&DataType::Int64)?)? } - TableVersion::Timestamp(nanos) => self.latest_snapshot_before(*nanos)?, + TableReadSelector::AsOfTimestamp(nanos) => self.latest_snapshot_at(*nanos)?, }; let Some(table) = self.snapshot(name, id)? else { return Ok(None); }; // Report the snapshot that is read - let table = table.with_version(Some(TableVersion::Version(ScalarValue::from(id)))); + let table = + table.with_read_selector(Some(TableReadSelector::AsOfVersion(ScalarValue::from(id)))); Ok(Some(Arc::new(table))) } @@ -265,14 +276,23 @@ impl SchemaProvider for Schema { } ``` -EXPLAIN shows the version that each table scan reads. Scans of the same version -compare equal, however that version was requested: +EXPLAIN shows the selector that each table scan reads. Scans of the same table +with equal selectors are assumed to read the same data, and compare equal +however the selector was requested: ```text TableScan: t@3 projection=[id, name] ``` -[`tableversion`]: https://docs.rs/datafusion/latest/datafusion/common/enum.TableVersion.html +Providers returned by `SchemaProvider::table` may also report a selector, such +as the id of the current snapshot, to identify the state of the table they +read. + +SQL syntax other than time travel clauses can be mapped to a `TableReadSelector` by +implementing a `TableReadSelectorPlanner`, registered with +`SessionContext::register_table_read_selector_planner`. + +[`tablereadselector`]: https://docs.rs/datafusion/latest/datafusion/common/enum.TableReadSelector.html ## Implementing `MemoryCatalogProvider` diff --git a/docs/source/library-user-guide/upgrading/57.0.0.md b/docs/source/library-user-guide/upgrading/57.0.0.md index aee010e68e0a6..76fc3a649ac89 100644 --- a/docs/source/library-user-guide/upgrading/57.0.0.md +++ b/docs/source/library-user-guide/upgrading/57.0.0.md @@ -78,25 +78,37 @@ The trait already required `'static`, which `Any` implies, so existing implementations need no changes. `ParquetSource::parquet_file_reader_factory` returns the trait object to downcast. -### Time travel queries +### Time travel queries and table read selectors DataFusion now supports time travel queries such as -`SELECT * FROM t VERSION AS OF 3`. `SchemaProvider` gained a `table_at` method -that catalogs implement to return a specific version of a table, and -`TableProvider` and `TableSource` gained a `version` method that reports the -version a table reads. All have default implementations. See the -[catalog guide](../catalogs.md#time-travel) for details. +`SELECT * FROM t VERSION AS OF 3`. These are planned using a `TableReadSelector`, +which specifies which read of a table to perform, such as reading it as of a +version or a timestamp. `SchemaProvider` gained a +`table_with_read_selector` method that catalogs implement to return a table for a selector, +and `TableProvider` and `TableSource` gained a `read_selector` method that reports +the selector a table reads. All have default implementations. A new +`TableReadSelectorPlanner` trait, registered with +`SessionContext::register_table_read_selector_planner`, maps other SQL syntax to +a selector. See the [catalog guide](../catalogs.md#table-read-selectors-and-time-travel) +for details. **Who is affected:** -- Implementations of `SchemaProvider::table_at` must return a provider whose - `TableProvider::version` reports the version it reads. `ListingTable`, - `MemTable`, `ViewTable` and `EmptyTable` support this with `with_version`. +- Users of SQL dialects that parse time travel clauses, such as Snowflake or + Databricks: these clauses were previously ignored, so + `SELECT * FROM t VERSION AS OF 3` read the current state of `t`. Such queries + now fail with a "not implemented" error, unless the catalog implements + `SchemaProvider::table_with_read_selector`. `DELETE` and `UPDATE` statements with + a time travel clause on their target table now always fail. +- Implementations of `SchemaProvider::table_with_read_selector` must return a provider whose + `TableProvider::read_selector` reports the selector it reads. `ListingTable`, + `MemTable`, `ViewTable` and `EmptyTable` support this with `with_read_selector`. - Custom `ContextProvider` implementations that implement - `get_table_source_at` must return a source whose `version` reports the - version it reads. + `get_table_source_with_read_selector` must return a source whose `read_selector` reports the + selector it reads. They must also implement `evaluate_const_expr` to evaluate + the expressions of time travel clauses. - Users serializing logical plans with `datafusion-proto`: the `ListingTableScanNode`, `ViewTableScanNode`, `CustomTableScanNode` and - `EmptyTableScanNode` messages gained a `version` field. A - `LogicalExtensionCodec` must restore the version of the table providers it + `EmptyTableScanNode` messages gained a `read_selector` field. A + `LogicalExtensionCodec` must restore the selector of the table providers it decodes, or decoding fails. diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.md index d25a6d8d0fa8b..78388368d3a14 100644 --- a/docs/source/user-guide/sql/select.md +++ b/docs/source/user-guide/sql/select.md @@ -306,7 +306,9 @@ time_travel: The expression is evaluated once during planning and must be a constant. It can use functions such as `now()`, but cannot reference columns or parameters such as `$1`. A version can be any value that the catalog understands, such as a snapshot -id or a tag. A timestamp is converted to a `TIMESTAMP` in the UTC time zone. +id or a tag. A timestamp is converted to a `TIMESTAMP` in the UTC time zone, and +selects the latest version committed at or before that time, so a commit made +exactly at that time is included. In a prepared statement, the expression is evaluated when the statement is prepared rather than when it is executed. It therefore cannot use functions that @@ -329,12 +331,12 @@ WHERE cur.status <> prev.status; ``` Time travel requires a catalog that supports it (see -[`SchemaProvider::table_at`]); the built-in in-memory and listing tables do not. +[`SchemaProvider::table_with_read_selector`]); the built-in in-memory and listing tables do not. Time travel clauses are currently only parsed by the `Snowflake`, `Databricks`, `BigQuery` and `MsSQL` dialects (see `datafusion.sql_parser.dialect`), and not every dialect supports every syntax. -[`schemaprovider::table_at`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html#method.table_at +[`schemaprovider::table_with_read_selector`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html#method.table_with_read_selector ## WHERE clause From c148ca070cdc301c9b859dcba244adff70aef401 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Fri, 13 Nov 2026 17:07:40 +0100 Subject: [PATCH 3/3] Add table read selector extensions Add `TableReadSelector::Extension` for reads that are specific to a catalog or engine, defined by implementing `TableReadSelectorExtension`. Extensions are serialized with the new `LogicalExtensionCodec::try_encode_read_selector` and `try_decode_read_selector` methods, and unparsed to SQL with the new `Dialect::read_selector_extension_to_sql` hook. Co-Authored-By: Claude Opus 5.5 --- datafusion/common/src/lib.rs | 2 +- datafusion/common/src/table_read_selector.rs | 149 +++++++++++++++++- .../table_read_selector_planner.rs | 122 +++++++++++++- .../proto-models/proto/datafusion.proto | 3 + datafusion/proto-models/src/from_proto.rs | 8 +- .../proto-models/src/generated/pbjson.rs | 14 ++ .../proto-models/src/generated/prost.rs | 6 +- datafusion/proto-models/src/to_proto.rs | 5 + datafusion/proto/src/logical_plan/mod.rs | 78 +++++++-- .../tests/cases/roundtrip_logical_plan.rs | 131 ++++++++++++++- datafusion/sql/src/unparser/dialect.rs | 15 +- datafusion/sql/src/unparser/plan.rs | 29 +++- datafusion/sql/tests/cases/plan_to_sql.rs | 90 ++++++++++- .../src/test_context/time_travel.rs | 3 + docs/source/library-user-guide/catalogs.md | 13 +- .../library-user-guide/upgrading/57.0.0.md | 8 +- 16 files changed, 634 insertions(+), 42 deletions(-) diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 0500e131913c6..85e425ab6f974 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -90,7 +90,7 @@ pub use scalar::{ScalarType, ScalarValue}; pub use schema_reference::SchemaReference; pub use spans::{Location, Span, Spans}; pub use stats::{ColumnStatistics, Statistics}; -pub use table_read_selector::TableReadSelector; +pub use table_read_selector::{TableReadSelector, TableReadSelectorExtension}; pub use table_reference::{ResolvedTableReference, TableReference}; pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; diff --git a/datafusion/common/src/table_read_selector.rs b/datafusion/common/src/table_read_selector.rs index 2d3af97e72e03..843d22b23806f 100644 --- a/datafusion/common/src/table_read_selector.rs +++ b/datafusion/common/src/table_read_selector.rs @@ -2,7 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use std::fmt::{Display, Formatter}; +use std::any::Any; +use std::cmp::Ordering; +use std::fmt::{Debug, Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; use arrow::temporal_conversions::timestamp_ns_to_datetime; use chrono::SecondsFormat; @@ -12,9 +16,10 @@ use crate::ScalarValue; /// Selects which read of a table to perform, such as reading the table as of /// a version or point in time for a time travel query. /// -/// A selector typically picks a point in the history of a table, such as a -/// snapshot. It determines *which* data is read, not *how* it is read: hints -/// about how to scan a table do not belong in a selector. +/// A selector typically picks a point or a range in the history of a table, +/// such as a snapshot, or the changes between two snapshots. It determines +/// *which* data is read, not *how* it is read: hints about how to scan a table +/// do not belong in a selector. /// /// A selector plays two roles: /// @@ -31,6 +36,9 @@ use crate::ScalarValue; /// may differ between selectors, for example after the schema of the table /// evolved. /// +/// Selectors that are specific to a catalog or engine can be defined with +/// [`TableReadSelector::Extension`]. +/// /// See [`SqlToRel::read_selector`] for how SQL time travel clauses are /// mapped to selectors. /// @@ -55,6 +63,104 @@ pub enum TableReadSelector { /// time, so a commit made exactly at this time is included. As an /// identity, it is the state of the table at this time. AsOfTimestamp(i64), + /// A selector defined outside DataFusion, see [`TableReadSelectorExtension`]. + Extension(Arc), +} + +/// A [`TableReadSelector`] defined outside DataFusion, for a read of a table +/// that is specific to a catalog or engine, such as the state of the table +/// before a given statement was executed, or the changes between two versions. +/// +/// Like other selectors, an extension is used both as a request and as an +/// identity. Its [`Display`] implementation is shown in `EXPLAIN`, and scans +/// of the same table with equal extensions are assumed to read the same data. +/// A table provider for a read that is not a state of the table, such as a +/// change feed, must not report the constraints of the table, such as its +/// primary key, as they may not hold for the data it reads. +/// +/// To serialize plans with `datafusion-proto`, implement +/// [`LogicalExtensionCodec::try_encode_read_selector`] and +/// [`LogicalExtensionCodec::try_decode_read_selector`]. To unparse plans +/// to SQL, implement [`Dialect::read_selector_extension_to_sql`]. +/// +/// [`LogicalExtensionCodec::try_encode_read_selector`]: https://docs.rs/datafusion-proto/latest/datafusion_proto/logical_plan/trait.LogicalExtensionCodec.html#method.try_encode_read_selector +/// [`LogicalExtensionCodec::try_decode_read_selector`]: https://docs.rs/datafusion-proto/latest/datafusion_proto/logical_plan/trait.LogicalExtensionCodec.html#method.try_decode_read_selector +/// [`Dialect::read_selector_extension_to_sql`]: https://docs.rs/datafusion/latest/datafusion/sql/unparser/dialect/trait.Dialect.html#method.read_selector_extension_to_sql +/// +/// # Example +/// +/// ``` +/// # use std::any::Any; +/// # use std::cmp::Ordering; +/// # use std::fmt::{Display, Formatter}; +/// # use std::hash::{Hash, Hasher}; +/// # use datafusion_common::TableReadSelectorExtension; +/// /// The state of a table before a statement was executed +/// #[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] +/// struct BeforeStatement(String); +/// +/// impl Display for BeforeStatement { +/// fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { +/// write!(f, "before statement {:?}", self.0) +/// } +/// } +/// +/// impl TableReadSelectorExtension for BeforeStatement { +/// fn as_any(&self) -> &dyn Any { +/// self +/// } +/// +/// fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool { +/// other.as_any().downcast_ref::() == Some(self) +/// } +/// +/// fn dyn_hash(&self, mut state: &mut dyn Hasher) { +/// self.hash(&mut state) +/// } +/// +/// fn dyn_ord(&self, other: &dyn TableReadSelectorExtension) -> Option { +/// other +/// .as_any() +/// .downcast_ref::() +/// .and_then(|other| self.partial_cmp(other)) +/// } +/// } +/// ``` +pub trait TableReadSelectorExtension: Debug + Display + Send + Sync { + /// Returns the selector as [`Any`], so it can be downcast to its concrete + /// type. + fn as_any(&self) -> &dyn Any; + + /// Compares `other`, respecting the requirements of [`Eq`]. Selectors of + /// different types are not equal. + fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool; + + /// Hashes the selector, consistently with [`Self::dyn_eq`]. + fn dyn_hash(&self, state: &mut dyn Hasher); + + /// Compares `other`, respecting the requirements of [`PartialOrd`]. Must + /// return `Some(Ordering::Equal)` if and only if `self.dyn_eq(other)`. + fn dyn_ord(&self, other: &dyn TableReadSelectorExtension) -> Option; +} + +impl PartialEq for dyn TableReadSelectorExtension { + fn eq(&self, other: &Self) -> bool { + self.dyn_eq(other) + } +} + +impl Eq for dyn TableReadSelectorExtension {} + +impl Hash for dyn TableReadSelectorExtension { + fn hash(&self, state: &mut H) { + self.dyn_hash(state); + } +} + +impl PartialOrd for dyn TableReadSelectorExtension { + fn partial_cmp(&self, other: &Self) -> Option { + self.dyn_ord(other) + } } impl Display for TableReadSelector { @@ -74,6 +180,7 @@ impl Display for TableReadSelector { ), None => write!(f, "{ns}"), }, + TableReadSelector::Extension(extension) => write!(f, "{extension}"), } } } @@ -82,6 +189,36 @@ impl Display for TableReadSelector { mod tests { use super::*; + #[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] + struct Offset(i64); + + impl Display for Offset { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "offset {}", self.0) + } + } + + impl TableReadSelectorExtension for Offset { + fn as_any(&self) -> &dyn Any { + self + } + + fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool { + other.as_any().downcast_ref::() == Some(self) + } + + fn dyn_hash(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state) + } + + fn dyn_ord(&self, other: &dyn TableReadSelectorExtension) -> Option { + other + .as_any() + .downcast_ref::() + .and_then(|other| self.partial_cmp(other)) + } + } + #[test] fn display() { assert_eq!( @@ -106,5 +243,9 @@ mod tests { .to_string(), "\"3\"" ); + assert_eq!( + TableReadSelector::Extension(Arc::new(Offset(-60))).to_string(), + "offset -60" + ); } } diff --git a/datafusion/core/tests/user_defined/table_read_selector_planner.rs b/datafusion/core/tests/user_defined/table_read_selector_planner.rs index 93d15641e26aa..bff476c92ea6a 100644 --- a/datafusion/core/tests/user_defined/table_read_selector_planner.rs +++ b/datafusion/core/tests/user_defined/table_read_selector_planner.rs @@ -4,6 +4,9 @@ //! Tests for the TableReadSelectorPlanner extension point +use std::any::Any; +use std::fmt::{self, Display, Formatter}; +use std::hash::{Hash, Hasher}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -13,11 +16,17 @@ use async_trait::async_trait; use datafusion::catalog::memory::MemTable; use datafusion::catalog::{SchemaProvider, TableProvider}; use datafusion::prelude::*; -use datafusion_common::{Result, TableReadSelector, TableReference, not_impl_err}; +use datafusion_common::{ + Result, TableReadSelector, TableReadSelectorExtension, TableReference, not_impl_err, + plan_err, +}; use datafusion_expr::planner::{ TableReadSelectorContext, TableReadSelectorPlanner, TableReadSelectorPlanning, }; -use datafusion_sql::sqlparser::ast::TableFactor; +use datafusion_sql::sqlparser::ast::{ + Expr as SQLExpr, FunctionArg, FunctionArgExpr, FunctionArguments, TableFactor, + TableVersion, +}; use insta::assert_snapshot; /// A schema with a single table `t`, which counts how often the current and @@ -155,6 +164,90 @@ impl TableReadSelectorPlanner for IgnoreTimeTravelPlanner { } } +/// A table read selector extension for the state of a table before a statement was +/// executed +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] +struct BeforeStatement(String); + +impl Display for BeforeStatement { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "before statement {:?}", self.0) + } +} + +impl TableReadSelectorExtension for BeforeStatement { + fn as_any(&self) -> &dyn Any { + self + } + + fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool { + other.as_any().downcast_ref::() == Some(self) + } + + fn dyn_hash(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state) + } + + fn dyn_ord( + &self, + other: &dyn TableReadSelectorExtension, + ) -> Option { + other + .as_any() + .downcast_ref::() + .and_then(|other| self.partial_cmp(other)) + } +} + +/// Example planner that maps Snowflake's `BEFORE (STATEMENT => )` clause, +/// which DataFusion does not support, to the [`BeforeStatement`] selector +#[derive(Debug)] +struct BeforeStatementPlanner; + +impl TableReadSelectorPlanner for BeforeStatementPlanner { + fn plan_read_selector( + &self, + _table: &TableReference, + relation: &TableFactor, + context: &dyn TableReadSelectorContext, + ) -> Result { + let TableFactor::Table { + version: Some(TableVersion::Function(SQLExpr::Function(func))), + .. + } = relation + else { + return Ok(TableReadSelectorPlanning::Original); + }; + let FunctionArguments::List(list) = &func.args else { + return Ok(TableReadSelectorPlanning::Original); + }; + let [ + FunctionArg::Named { + name, + arg: FunctionArgExpr::Expr(statement), + .. + }, + ] = list.args.as_slice() + else { + return Ok(TableReadSelectorPlanning::Original); + }; + if !func.name.to_string().eq_ignore_ascii_case("BEFORE") + || !name.value.eq_ignore_ascii_case("STATEMENT") + { + return Ok(TableReadSelectorPlanning::Original); + } + let statement = context.evaluate_const_expr(statement.clone())?; + let Some(Some(statement)) = statement.try_as_str() else { + return plan_err!("The statement must be a string: {relation}"); + }; + Ok(TableReadSelectorPlanning::Planned(Some( + TableReadSelector::Extension(Arc::new(BeforeStatement( + statement.to_string(), + ))), + ))) + } +} + // A TableReadSelectorPlanner can determine the selector of a table from // syntax other than a time travel clause #[tokio::test] @@ -252,3 +345,28 @@ async fn planner_ignores_time_travel() { .expect_err("table read selectors should not be supported"); assert_snapshot!(error.strip_backtrace(), @"This feature is not implemented: Time travel is not supported"); } + +// A TableReadSelectorPlanner can determine a table read selector extension, +// which the catalog receives to look up the table +#[tokio::test] +async fn planner_determines_read_selector_extension() { + let ctx = create_context(vec![Arc::new(BeforeStatementPlanner)]); + ctx.sql("SET datafusion.sql_parser.dialect = 'Snowflake'") + .await + .unwrap(); + let schema = register_counting_schema(&ctx, true).await; + + let plan = ctx + .sql("SELECT * FROM counting.t BEFORE (STATEMENT => 'q1')") + .await + .unwrap() + .into_unoptimized_plan(); + assert_snapshot!(plan, @r#" + Projection: counting.t.id, counting.t.name + TableScan: counting.t@before statement "q1" + "#); + assert_eq!( + schema.table_with_read_selector_calls.load(Ordering::SeqCst), + 1 + ); +} diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index b406daccc1130..05105f40536fb 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -898,6 +898,9 @@ message TableReadSelector { datafusion_common.ScalarValue as_of_version = 1; // A point in time, in nanoseconds since the Unix epoch (UTC) int64 as_of_timestamp = 2; + // A selector defined outside DataFusion, encoded by a + // LogicalExtensionCodec + bytes extension = 3; } } diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs index 72aa69d685a4d..91d03f4aa967b 100644 --- a/datafusion/proto-models/src/from_proto.rs +++ b/datafusion/proto-models/src/from_proto.rs @@ -112,9 +112,11 @@ impl TryFrom<&protobuf::TableReadSelector> for TableReadSelector { TableReadSelectorEnum::AsOfVersion(value) => { Ok(TableReadSelector::AsOfVersion(value.try_into()?)) } - TableReadSelectorEnum::AsOfTimestamp(ns) => { - Ok(TableReadSelector::AsOfTimestamp(*ns)) - } + TableReadSelectorEnum::AsOfTimestamp(ns) => Ok(TableReadSelector::AsOfTimestamp(*ns)), + TableReadSelectorEnum::Extension(_) => Err(Error::General( + "Table read selector extensions must be decoded with a LogicalExtensionCodec" + .to_string(), + )), } } } diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 2382040e71102..316d0c6be2f11 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -28895,6 +28895,11 @@ impl serde::Serialize for TableReadSelector { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("asOfTimestamp", ToString::to_string(&v).as_str())?; } + table_read_selector::TableReadSelectorEnum::Extension(v) => { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("extension", pbjson::private::base64::encode(&v).as_str())?; + } } } struct_ser.end() @@ -28911,12 +28916,14 @@ impl<'de> serde::Deserialize<'de> for TableReadSelector { "asOfVersion", "as_of_timestamp", "asOfTimestamp", + "extension", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { AsOfVersion, AsOfTimestamp, + Extension, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -28940,6 +28947,7 @@ impl<'de> serde::Deserialize<'de> for TableReadSelector { match value { "asOfVersion" | "as_of_version" => Ok(GeneratedField::AsOfVersion), "asOfTimestamp" | "as_of_timestamp" => Ok(GeneratedField::AsOfTimestamp), + "extension" => Ok(GeneratedField::Extension), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -28975,6 +28983,12 @@ impl<'de> serde::Deserialize<'de> for TableReadSelector { } table_read_selector_enum__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| table_read_selector::TableReadSelectorEnum::AsOfTimestamp(x.0)); } + GeneratedField::Extension => { + if table_read_selector_enum__.is_some() { + return Err(serde::de::Error::duplicate_field("extension")); + } + table_read_selector_enum__ = map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| table_read_selector::TableReadSelectorEnum::Extension(x.0)); + } } } Ok(TableReadSelector { diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index cf2cbbd4aaea6..efb2ea43b50cb 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1405,7 +1405,7 @@ pub mod table_reference { /// version or point in time for time travel queries #[derive(Clone, PartialEq, ::prost::Message)] pub struct TableReadSelector { - #[prost(oneof = "table_read_selector::TableReadSelectorEnum", tags = "1, 2")] + #[prost(oneof = "table_read_selector::TableReadSelectorEnum", tags = "1, 2, 3")] pub table_read_selector_enum: ::core::option::Option< table_read_selector::TableReadSelectorEnum, >, @@ -1421,6 +1421,10 @@ pub mod table_read_selector { /// A point in time, in nanoseconds since the Unix epoch (UTC) #[prost(int64, tag = "2")] AsOfTimestamp(i64), + /// A selector defined outside DataFusion, encoded by a + /// LogicalExtensionCodec + #[prost(bytes, tag = "3")] + Extension(::prost::alloc::vec::Vec), } } /// PhysicalPlanNode is a nested type diff --git a/datafusion/proto-models/src/to_proto.rs b/datafusion/proto-models/src/to_proto.rs index 8cf549900dcdb..f2501d74b8458 100644 --- a/datafusion/proto-models/src/to_proto.rs +++ b/datafusion/proto-models/src/to_proto.rs @@ -127,6 +127,11 @@ impl TryFrom<&TableReadSelector> for protobuf::TableReadSelector { TableReadSelector::AsOfTimestamp(ns) => { TableReadSelectorEnum::AsOfTimestamp(*ns) } + TableReadSelector::Extension(extension) => { + return Err(datafusion_proto_common::ToProtoError::General(format!( + "Table read selector {extension} must be encoded with a LogicalExtensionCodec" + ))); + } }; Ok(protobuf::TableReadSelector { table_read_selector_enum: Some(table_read_selector_enum), diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index 06496c9469943..b2797dbd66d0d 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -28,8 +28,9 @@ use datafusion_common::format::{ }; use datafusion_common::utils::{usize_from_wire, usize_to_wire}; use datafusion_common::{ - NullEquality, Result, TableReadSelector, TableReference, assert_or_internal_err, - context, internal_datafusion_err, internal_err, not_impl_err, plan_err, + NullEquality, Result, TableReadSelector, TableReadSelectorExtension, TableReference, + assert_or_internal_err, context, internal_datafusion_err, internal_err, not_impl_err, + plan_err, }; use datafusion_datasource::file_format::FileFormat; use datafusion_datasource::file_format::{ @@ -208,6 +209,27 @@ pub trait LogicalExtensionCodec: Debug + Send + Sync + std::any::Any { fn try_encode_udwf(&self, _node: &WindowUDF, _buf: &mut Vec) -> Result<()> { Ok(()) } + + /// Decodes a [`TableReadSelector::Extension`] encoded by + /// [`Self::try_encode_read_selector`] + fn try_decode_read_selector( + &self, + _buf: &[u8], + ) -> Result> { + not_impl_err!("LogicalExtensionCodec is not provided for table read selectors") + } + + /// Encodes a [`TableReadSelector::Extension`], to be decoded by + /// [`Self::try_decode_read_selector`] + fn try_encode_read_selector( + &self, + selector: &dyn TableReadSelectorExtension, + _buf: &mut Vec, + ) -> Result<()> { + not_impl_err!( + "LogicalExtensionCodec is not provided for table read selector {selector}" + ) + } } #[derive(Debug, Clone)] @@ -362,8 +384,37 @@ fn from_table_reference( fn from_read_selector( selector: Option<&protobuf::TableReadSelector>, + extension_codec: &dyn LogicalExtensionCodec, ) -> Result> { - Ok(selector.map(TableReadSelector::try_from).transpose()?) + use protobuf::table_read_selector::TableReadSelectorEnum; + let Some(selector) = selector else { + return Ok(None); + }; + if let Some(TableReadSelectorEnum::Extension(buf)) = + &selector.table_read_selector_enum + { + let extension = extension_codec.try_decode_read_selector(buf)?; + return Ok(Some(TableReadSelector::Extension(extension))); + } + Ok(Some(TableReadSelector::try_from(selector)?)) +} + +fn to_read_selector( + selector: Option<&TableReadSelector>, + extension_codec: &dyn LogicalExtensionCodec, +) -> Result> { + use protobuf::table_read_selector::TableReadSelectorEnum; + let Some(selector) = selector else { + return Ok(None); + }; + if let TableReadSelector::Extension(extension) = selector { + let mut buf = vec![]; + extension_codec.try_encode_read_selector(extension.as_ref(), &mut buf)?; + return Ok(Some(protobuf::TableReadSelector { + table_read_selector_enum: Some(TableReadSelectorEnum::Extension(buf)), + })); + } + Ok(Some(protobuf::TableReadSelector::try_from(selector)?)) } /// Converts [LogicalPlan::TableScan] to [TableSource] @@ -682,7 +733,10 @@ impl AsLogicalPlan for LogicalPlanNode { .with_cache( ctx.runtime_env().cache_manager.get_file_statistic_cache(), ) - .with_read_selector(from_read_selector(scan.read_selector.as_ref())?); + .with_read_selector(from_read_selector( + scan.read_selector.as_ref(), + extension_codec, + )?); let table_name = from_table_reference(scan.table_name.as_ref(), "ListingTableScan")?; @@ -735,7 +789,8 @@ impl AsLogicalPlan for LogicalPlanNode { )?; // The codec restores the selector as part of the provider's // state. Fail rather than silently reading another state. - let selector = from_read_selector(scan.read_selector.as_ref())?; + let selector = + from_read_selector(scan.read_selector.as_ref(), extension_codec)?; if provider.read_selector() != selector.as_ref() { return internal_err!( "Decoded table provider for '{table_name}' reads selector {:?}, \ @@ -1242,8 +1297,9 @@ impl AsLogicalPlan for LogicalPlanNode { None }; - let provider = ViewTable::new(input, definition) - .with_read_selector(from_read_selector(scan.read_selector.as_ref())?); + let provider = ViewTable::new(input, definition).with_read_selector( + from_read_selector(scan.read_selector.as_ref(), extension_codec)?, + ); let table_name = from_table_reference(scan.table_name.as_ref(), "ViewScan")?; @@ -1393,7 +1449,7 @@ impl AsLogicalPlan for LogicalPlanNode { let provider = Arc::new(EmptyTable::new(Arc::clone(&schema)).with_read_selector( - from_read_selector(scan.read_selector.as_ref())?, + from_read_selector(scan.read_selector.as_ref(), extension_codec)?, )); let table_scan = @@ -1457,10 +1513,8 @@ impl AsLogicalPlan for LogicalPlanNode { }) => { let provider = source_as_provider(source)?; let schema = provider.schema(); - let read_selector = source - .read_selector() - .map(protobuf::TableReadSelector::try_from) - .transpose()?; + let read_selector = + to_read_selector(source.read_selector(), extension_codec)?; let projection = match projection { None => None, diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index 58a19e232f773..41712e934437c 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -26,8 +26,10 @@ use datafusion_common::parsers::CompressionTypeVariant; use datafusion_functions_aggregate::sum::sum_distinct; use prost::Message; use std::any::Any; +use std::cmp::Ordering; use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; +use std::hash::{Hash, Hasher}; use std::mem::size_of_val; use std::sync::Arc; use std::vec; @@ -64,8 +66,8 @@ use datafusion_common::format::{ use datafusion_common::scalar::ScalarStructBuilder; use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, - SplitPoint, TableReadSelector, TableReference, internal_datafusion_err, internal_err, - not_impl_err, plan_err, + SplitPoint, TableReadSelector, TableReadSelectorExtension, TableReference, + internal_datafusion_err, internal_err, not_impl_err, plan_err, }; use datafusion_execution::TaskContext; use datafusion_expr::dml::CopyTo; @@ -3877,6 +3879,131 @@ async fn roundtrip_table_scan_with_read_selector() -> Result<()> { Ok(()) } +/// A table read selector for a number of seconds before the start of the query +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] +struct OffsetSelector(i64); + +impl fmt::Display for OffsetSelector { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "offset {}", self.0) + } +} + +impl TableReadSelectorExtension for OffsetSelector { + fn as_any(&self) -> &dyn Any { + self + } + + fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool { + other.as_any().downcast_ref::() == Some(self) + } + + fn dyn_hash(&self, mut state: &mut dyn Hasher) { + self.hash(&mut state) + } + + fn dyn_ord(&self, other: &dyn TableReadSelectorExtension) -> Option { + other + .as_any() + .downcast_ref::() + .and_then(|other| self.partial_cmp(other)) + } +} + +/// Encodes [`OffsetSelector`]s +#[derive(Debug)] +struct OffsetSelectorCodec {} + +impl LogicalExtensionCodec for OffsetSelectorCodec { + fn try_decode( + &self, + _buf: &[u8], + _inputs: &[LogicalPlan], + _ctx: &TaskContext, + ) -> Result { + not_impl_err!("No extension codec provided") + } + + fn try_encode(&self, _node: &Extension, _buf: &mut Vec) -> Result<()> { + not_impl_err!("No extension codec provided") + } + + fn try_decode_table_provider( + &self, + _buf: &[u8], + _table_ref: &TableReference, + _schema: SchemaRef, + _ctx: &TaskContext, + ) -> Result> { + not_impl_err!("No table provider codec provided") + } + + fn try_encode_table_provider( + &self, + _table_ref: &TableReference, + _node: Arc, + _buf: &mut Vec, + ) -> Result<()> { + not_impl_err!("No table provider codec provided") + } + + fn try_decode_read_selector( + &self, + buf: &[u8], + ) -> Result> { + let offset = buf + .try_into() + .map_err(|_| internal_datafusion_err!("Invalid offset selector"))?; + Ok(Arc::new(OffsetSelector(i64::from_le_bytes(offset)))) + } + + fn try_encode_read_selector( + &self, + selector: &dyn TableReadSelectorExtension, + buf: &mut Vec, + ) -> Result<()> { + let Some(OffsetSelector(offset)) = selector.as_any().downcast_ref() else { + return not_impl_err!("Unsupported table read selector {selector}"); + }; + buf.extend_from_slice(&offset.to_le_bytes()); + Ok(()) + } +} + +// Table read selector extensions are encoded with the `LogicalExtensionCodec` +#[tokio::test] +async fn roundtrip_table_scan_with_read_selector_extension() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let selector = TableReadSelector::Extension(Arc::new(OffsetSelector(-60))); + let table = + Arc::new(EmptyTable::new(schema).with_read_selector(Some(selector.clone()))); + let plan = LogicalPlanBuilder::scan( + "empty", + Arc::new(DefaultTableSource::new(table)), + None, + )? + .build()?; + let ctx = SessionContext::new(); + let codec = OffsetSelectorCodec {}; + + let bytes = logical_plan_to_bytes_with_extension_codec(&plan, &codec)?; + let restored = + logical_plan_from_bytes_with_extension_codec(&bytes, &ctx.task_ctx(), &codec)?; + assert_eq!(plan, restored); + let LogicalPlan::TableScan(scan) = restored else { + panic!("expected a TableScan"); + }; + assert_eq!(scan.source.read_selector(), Some(&selector)); + + // Without a codec for it, the selector cannot be encoded + let err = logical_plan_to_bytes(&plan).unwrap_err().strip_backtrace(); + assert_eq!( + err, + "This feature is not implemented: LogicalExtensionCodec is not provided for table read selector offset -60" + ); + Ok(()) +} + #[tokio::test] async fn roundtrip_empty_table_scan_with_projection() -> Result<()> { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rs index 07c49f0fefa30..a7b0c95f61de5 100644 --- a/datafusion/sql/src/unparser/dialect.rs +++ b/datafusion/sql/src/unparser/dialect.rs @@ -12,7 +12,7 @@ use super::{ use arrow::array::timezone::Tz; use arrow::datatypes::TimeUnit; use chrono::DateTime; -use datafusion_common::{Result, internal_err}; +use datafusion_common::{Result, TableReadSelectorExtension, internal_err}; use datafusion_expr::Expr; use regex::Regex; use sqlparser::tokenizer::Span; @@ -98,6 +98,19 @@ pub trait Dialect: Send + Sync { TableVersionStyle::At } + /// Unparses a [`TableReadSelector::Extension`] to the time travel clause of a + /// table scan. Returns `None` if the dialect does not support the + /// selector, in which case unparsing fails. + /// + /// [`TableReadSelector::Extension`]: datafusion_common::TableReadSelector::Extension + fn read_selector_extension_to_sql( + &self, + _unparser: &Unparser, + _selector: &dyn TableReadSelectorExtension, + ) -> Result> { + Ok(None) + } + /// The SQL type to use for Arrow Int64 unparsing /// Most dialects use BigInt, but some, like MySQL, require SIGNED fn int64_cast_dtype(&self) -> ast::DataType { diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index bf45b1a924753..52f880fb75c8f 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -2351,19 +2351,27 @@ impl Unparser<'_> { &self, selector: &TableReadSelector, ) -> Result { - let value = match selector { - TableReadSelector::AsOfVersion(value) => value.clone(), - TableReadSelector::AsOfTimestamp(ns) => { - ScalarValue::TimestampNanosecond(Some(*ns), Some("UTC".into())) + let (name, value) = match selector { + TableReadSelector::AsOfVersion(value) => ("VERSION", value.clone()), + TableReadSelector::AsOfTimestamp(ns) => ( + "TIMESTAMP", + ScalarValue::TimestampNanosecond(Some(*ns), Some("UTC".into())), + ), + TableReadSelector::Extension(extension) => { + return match self + .dialect + .read_selector_extension_to_sql(self, extension.as_ref())? + { + Some(version) => Ok(version), + None => not_impl_err!( + "This dialect does not support table read selector {selector}" + ), + }; } }; let value = self.expr_to_sql(&Expr::Literal(value, None))?; Ok(match (self.dialect.table_version_style(), selector) { (TableVersionStyle::At, _) => { - let name = match selector { - TableReadSelector::AsOfVersion(_) => "VERSION", - TableReadSelector::AsOfTimestamp(_) => "TIMESTAMP", - }; ast::TableVersion::Function(ast::Expr::Function(ast::Function { name: ast::ObjectName::from(vec![Ident::new("AT")]), uses_odbc_syntax: false, @@ -2397,6 +2405,11 @@ impl Unparser<'_> { "This dialect does not support time travel to version {selector}" ); } + (_, TableReadSelector::Extension(_)) => { + return internal_err!( + "Table read selector extensions are unparsed by the dialect" + ); + } }) } diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs index 53f2da257c4b2..9a2957c56cc03 100644 --- a/datafusion/sql/tests/cases/plan_to_sql.rs +++ b/datafusion/sql/tests/cases/plan_to_sql.rs @@ -7,7 +7,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::tree_node::{Transformed, TransformedResult}; use datafusion_common::{ Column, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, - TableReadSelector, TableReference, assert_contains, + TableReadSelector, TableReadSelectorExtension, TableReference, assert_contains, }; use datafusion_expr::expr::{WindowFunction, WindowFunctionParams}; use datafusion_expr::test::function_stub::{ @@ -35,7 +35,7 @@ use datafusion_sql::unparser::dialect::{ }; use datafusion_sql::unparser::{Unparser, expr_to_sql, plan_to_sql}; use insta::assert_snapshot; -use sqlparser::ast::Statement; +use sqlparser::ast::{self, Statement}; use std::hash::Hash; use std::ops::Add; use std::sync::Arc; @@ -1759,9 +1759,95 @@ fn test_table_scan_version_to_sql() -> Result<()> { .strip_backtrace(); assert_snapshot!(err, @"This feature is not implemented: This dialect does not support time travel to version 3"); + // Extensions are unparsed by the dialect + let before = + TableReadSelector::Extension(Arc::new(BeforeStatement("q1".to_string()))); + let unparser = Unparser::new(&BeforeStatementDialect {}); + let sql = unparser.plan_to_sql(&scan(before.clone(), vec![])?)?; + assert_snapshot!(sql, @"SELECT * FROM t1 BEFORE(STATEMENT => 'q1')"); + let err = plan_to_sql(&scan(before, vec![])?) + .unwrap_err() + .strip_backtrace(); + assert_snapshot!(err, @r#"This feature is not implemented: This dialect does not support table read selector before statement "q1""#); + Ok(()) } +/// A table read selector for the state of a table before a statement was executed +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd)] +struct BeforeStatement(String); + +impl fmt::Display for BeforeStatement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "before statement {:?}", self.0) + } +} + +impl TableReadSelectorExtension for BeforeStatement { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn dyn_eq(&self, other: &dyn TableReadSelectorExtension) -> bool { + other.as_any().downcast_ref::() == Some(self) + } + + fn dyn_hash(&self, mut state: &mut dyn std::hash::Hasher) { + self.hash(&mut state) + } + + fn dyn_ord( + &self, + other: &dyn TableReadSelectorExtension, + ) -> Option { + other + .as_any() + .downcast_ref::() + .and_then(|other| self.partial_cmp(other)) + } +} + +/// A dialect that unparses [`BeforeStatement`] to Snowflake's +/// `BEFORE (STATEMENT => ...)` clause +struct BeforeStatementDialect {} + +impl UnparserDialect for BeforeStatementDialect { + fn identifier_quote_style(&self, _identifier: &str) -> Option { + None + } + + fn read_selector_extension_to_sql( + &self, + unparser: &Unparser, + selector: &dyn TableReadSelectorExtension, + ) -> Result> { + let Some(BeforeStatement(statement)) = selector.as_any().downcast_ref() else { + return Ok(None); + }; + let statement = unparser.expr_to_sql(&lit(statement.as_str()))?; + Ok(Some(ast::TableVersion::Function(ast::Expr::Function( + ast::Function { + name: ast::ObjectName::from(vec![ast::Ident::new("BEFORE")]), + uses_odbc_syntax: false, + parameters: ast::FunctionArguments::None, + args: ast::FunctionArguments::List(ast::FunctionArgumentList { + duplicate_treatment: None, + args: vec![ast::FunctionArg::Named { + name: ast::Ident::new("STATEMENT"), + arg: ast::FunctionArgExpr::Expr(statement), + operator: ast::FunctionArgOperator::RightArrow, + }], + clauses: vec![], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }, + )))) + } +} + #[test] fn test_table_scan_pushdown() -> Result<()> { let schema = Schema::new(vec![ diff --git a/datafusion/sqllogictest/src/test_context/time_travel.rs b/datafusion/sqllogictest/src/test_context/time_travel.rs index 9487fcfc7c3c5..31028b9c2839f 100644 --- a/datafusion/sqllogictest/src/test_context/time_travel.rs +++ b/datafusion/sqllogictest/src/test_context/time_travel.rs @@ -301,6 +301,9 @@ impl SchemaProvider for VersionedSchema { None => return plan_err!("Table t did not exist at {selector}"), } } + TableReadSelector::Extension(_) => { + return plan_err!("Table read selector {selector} is not supported"); + } }; Self::table_version(version).map(Some) } diff --git a/docs/source/library-user-guide/catalogs.md b/docs/source/library-user-guide/catalogs.md index 1c2e659843251..8e2f3f3f6a2cd 100644 --- a/docs/source/library-user-guide/catalogs.md +++ b/docs/source/library-user-guide/catalogs.md @@ -213,8 +213,10 @@ time travel clause during planning and passes the resulting `TableReadSelector` `SchemaProvider::table_with_read_selector`. A `TableReadSelector` is one of: - an `AsOfVersion`, holding any `ScalarValue` such as a snapshot id, a tag, a branch - name or a commit hash, or -- an `AsOfTimestamp` in nanoseconds since the Unix epoch (UTC). + name or a commit hash, +- an `AsOfTimestamp` in nanoseconds since the Unix epoch (UTC), or +- an `Extension`, for reads that are specific to a catalog or engine, such as + the changes between two versions. See `TableReadSelectorExtension`. An `AsOfTimestamp` selects the latest state committed at or before that time, so a commit made exactly at that time is included. @@ -237,7 +239,7 @@ as `ListingTable` and `MemTable`, report the selector set with `with_read_select # use std::sync::Arc; # use arrow::datatypes::DataType; # use datafusion::catalog::{MemTable, TableProvider, SchemaProvider}; -# use datafusion::common::{Result, ScalarValue, TableReadSelector}; +# use datafusion::common::{not_impl_err, Result, ScalarValue, TableReadSelector}; # # #[derive(Debug)] # struct Schema {} @@ -260,6 +262,7 @@ impl SchemaProvider for Schema { i64::try_from(value.cast_to(&DataType::Int64)?)? } TableReadSelector::AsOfTimestamp(nanos) => self.latest_snapshot_at(*nanos)?, + _ => return not_impl_err!("Unsupported table read selector {selector}"), }; let Some(table) = self.snapshot(name, id)? else { return Ok(None); @@ -291,6 +294,10 @@ read. SQL syntax other than time travel clauses can be mapped to a `TableReadSelector` by implementing a `TableReadSelectorPlanner`, registered with `SessionContext::register_table_read_selector_planner`. +Plans with a `TableReadSelector::Extension` are serialized with +`LogicalExtensionCodec::try_encode_read_selector` and +`try_decode_read_selector`, and unparsed to SQL with +`Dialect::read_selector_extension_to_sql`. [`tablereadselector`]: https://docs.rs/datafusion/latest/datafusion/common/enum.TableReadSelector.html diff --git a/docs/source/library-user-guide/upgrading/57.0.0.md b/docs/source/library-user-guide/upgrading/57.0.0.md index 76fc3a649ac89..d339ea742a8e9 100644 --- a/docs/source/library-user-guide/upgrading/57.0.0.md +++ b/docs/source/library-user-guide/upgrading/57.0.0.md @@ -82,8 +82,8 @@ returns the trait object to downcast. DataFusion now supports time travel queries such as `SELECT * FROM t VERSION AS OF 3`. These are planned using a `TableReadSelector`, -which specifies which read of a table to perform, such as reading it as of a -version or a timestamp. `SchemaProvider` gained a +which specifies which read of a table to perform: as of a version, as of a timestamp, or a +`TableReadSelectorExtension` defined outside DataFusion. `SchemaProvider` gained a `table_with_read_selector` method that catalogs implement to return a table for a selector, and `TableProvider` and `TableSource` gained a `read_selector` method that reports the selector a table reads. All have default implementations. A new @@ -111,4 +111,6 @@ for details. `ListingTableScanNode`, `ViewTableScanNode`, `CustomTableScanNode` and `EmptyTableScanNode` messages gained a `read_selector` field. A `LogicalExtensionCodec` must restore the selector of the table providers it - decodes, or decoding fails. + decodes, or decoding fails. Selector extensions are encoded with the new + `LogicalExtensionCodec::try_encode_read_selector` and + `try_decode_read_selector` methods.