diff --git a/ballista/core/src/serde/composed_codec.rs b/ballista/core/src/serde/composed_codec.rs index 092c1cae3e..51bfe012c5 100644 --- a/ballista/core/src/serde/composed_codec.rs +++ b/ballista/core/src/serde/composed_codec.rs @@ -114,6 +114,14 @@ impl ComposedLogicalExtensionCodec { } /// Combines the codecs' errors, so the owning codec's error isn't hidden. +/// +/// TODO: when moving this to `datafusion-proto`, return the error of a codec +/// that recognised the node (one other than `NotImplemented`) unchanged, and +/// join the errors only when every codec declined. Joining them into one string +/// loses the owning codec's error chain, so `DataFusionError::find_root` can't +/// reach it and callers can only match its text. For example, reading an +/// Iceberg table that can't be loaded reports `no composed codec could encode +/// it: codec 0: … not implemented …; codec 1: `. fn collect(mut errors: Vec) -> DataFusionError { match errors.len() { 0 => DataFusionError::NotImplemented("Empty list of composed codecs".to_owned()), diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 82e2b64c14..373e2df5f1 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -115,6 +115,13 @@ Metadata files never change, so every node rebuilds exactly that version. the client plans the query. DataFusion looks up each table a query names once, so every scan of it, such as each side of a self-join, reads that version, and each new query reads the latest one. +- **A view**, from `CREATE VIEW` or a `DataFrame`'s `into_view`, stores the + plan it was created with, so each query that names it pins its + catalog-backed tables again, and it reads their latest versions. +- **A table its catalog can't load**, because it was dropped or the catalog + can't be reached, fails a query only when the query reads it, as in + DataFusion. The table, and any view over it, can still be described, dropped + or replaced, and a view reads the table again once it loads. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. @@ -162,11 +169,11 @@ and executors must all set the same codecs in the same order. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. -- **A view keeps the version from when it was created.** DataFusion stores a - view as a planned query, and tables are pinned when a query is planned. The - same goes for a `DataFrame` or a Flight SQL prepared statement run more than - once: each run reads the version from when it was planned. To read newer - commits, recreate the view or plan the query again. +- **A reused `DataFrame` or prepared statement keeps its version.** Tables are + pinned when a query is planned, so a `DataFrame` or a Flight SQL prepared + statement run more than once reads the version from when it was planned. To + read newer commits, plan the query again, or register the `DataFrame` as a + view with `into_view` and query that. - **A separately deployed scheduler and executors** must link `ballista-iceberg` and be readied with `with_iceberg` (see [SQL clients](#sql-clients)). The tests cover standalone mode and an diff --git a/integrations/iceberg/src/catalog.rs b/integrations/iceberg/src/catalog.rs index 671434e00d..4a2ee2202e 100644 --- a/integrations/iceberg/src/catalog.rs +++ b/integrations/iceberg/src/catalog.rs @@ -17,29 +17,45 @@ //! Pins catalog-backed tables to one version while a query is planned. +use std::borrow::Cow; +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::view::ViewTable; use datafusion::catalog::{ CatalogProvider, CatalogProviderList, SchemaProvider, Session, TableProvider, }; -use datafusion::common::{Result, not_impl_err}; -use datafusion::datasource::TableType; +use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; +use datafusion::common::{DataFusionError, Result, not_impl_err}; use datafusion::datasource::source::DataSourceExec; +use datafusion::datasource::{DefaultTableSource, TableType, provider_as_source}; use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; +use datafusion::logical_expr::{ + Expr, LogicalPlan, TableProviderFilterPushDown, TableScan, +}; use datafusion::physical_plan::ExecutionPlan; +use futures::future::join_all; +use iceberg::TableIdent; use iceberg::table::Table; use iceberg_datafusion::read::IcebergDataSource; use iceberg_datafusion::{IcebergCatalogTableProvider, to_datafusion_error}; /// A session's catalogs, serving each catalog-backed Iceberg table as a -/// [`PinnedTable`]. +/// [`PinnedTable`], and each view with its Iceberg tables pinned again, as a +/// [`RepinnedView`]. /// /// DataFusion looks up each table a query names once, while planning it, so /// every scan of a table in the query reads the same version, and encoding the -/// plan needs no I/O. +/// plan needs no I/O. A view stores the plan it was created with, so its tables +/// are pinned again each time a query names it, and it reads the latest +/// versions, as it would in DataFusion. +/// +/// A lookup never fails because a table can't be loaded: the [`PinnedTable`] +/// holds the error instead, and a query fails with it only when it reads the +/// table, so the table, or a view over it, can still be described, dropped or +/// replaced. #[derive(Debug)] pub(crate) struct PinningCatalogList(pub(crate) Arc); @@ -109,18 +125,20 @@ impl SchemaProvider for PinningSchema { let Some(table) = self.0.table(name).await? else { return Ok(None); }; - let Some(provider) = table.downcast_ref::() else { - return Ok(Some(table)); + if let Some(provider) = table.downcast_ref::() { + let table = load(provider).await; + return Ok(Some(Arc::new(PinnedTable::pin(provider, table)))); + } + let repinned = match table.get_logical_plan() { + Some(plan) => match repin(&plan).await? { + Some(plan) => ViewTable::new(plan, None), + None => return Ok(Some(table)), + }, + None => return Ok(Some(table)), }; - let pinned = provider - .catalog() - .load_table(provider.table_ident()) - .await - .map_err(to_datafusion_error)?; - Ok(Some(Arc::new(PinnedTable { - table: pinned, - snapshot_id: None, - schema: provider.schema(), + Ok(Some(Arc::new(RepinnedView { + original: table, + repinned, }))) } @@ -145,8 +163,132 @@ impl SchemaProvider for PinningSchema { } } +/// A catalog-backed table: its catalog, by address, and its identifier. +type TableKey = (usize, TableIdent); + +fn key(provider: &IcebergCatalogTableProvider) -> TableKey { + let catalog = Arc::as_ptr(provider.catalog()).cast::<()>() as usize; + (catalog, provider.table_ident().clone()) +} + +/// A loaded table, or the error loading it returned, shared by every scan of it. +type Loaded = std::result::Result>; + +/// The version of `provider`'s table its catalog serves now, or why it couldn't +/// be loaded. +async fn load(provider: &IcebergCatalogTableProvider) -> Loaded { + provider + .catalog() + .load_table(provider.table_ident()) + .await + .map_err(|error| Arc::new(to_datafusion_error(error))) +} + +/// The catalog-backed table a scan's pinned table was pinned from, if any. +fn origin(scan: &TableScan) -> Option<&IcebergCatalogTableProvider> { + let source = scan.source.downcast_ref::()?; + let pinned = source.table_provider.downcast_ref::()?; + pinned.origin.as_ref() +} + +/// Returns a view's `plan` with each catalog-backed Iceberg table it reads +/// pinned to the version its catalog serves now, loading each table once, or +/// `None` if it reads none. A table that can't be loaded is pinned to the error, +/// so the plan still knows which table to load next time. DataFusion inlines a +/// view when it plans a scan of it, so the plan of a view over other views +/// already holds their tables, and needs no recursion. +async fn repin(plan: &LogicalPlan) -> Result> { + let mut providers = HashMap::new(); + plan.apply_with_subqueries(|node| { + if let LogicalPlan::TableScan(scan) = node + && let Some(origin) = origin(scan) + { + providers + .entry(key(origin)) + .or_insert_with(|| origin.clone()); + } + Ok(TreeNodeRecursion::Continue) + })?; + if providers.is_empty() { + return Ok(None); + } + let loads = providers + .into_iter() + .map(|(key, provider)| async move { (key, load(&provider).await) }); + let tables: HashMap<_, _> = join_all(loads).await.into_iter().collect(); + let plan = plan + .clone() + .transform_up_with_subqueries(|node| { + let LogicalPlan::TableScan(mut scan) = node else { + return Ok(Transformed::no(node)); + }; + let Some(origin) = origin(&scan) else { + return Ok(Transformed::no(LogicalPlan::TableScan(scan))); + }; + let table = tables[&key(origin)].clone(); + scan.source = provider_as_source(Arc::new(PinnedTable::pin(origin, table))); + Ok(Transformed::yes(LogicalPlan::TableScan(scan))) + })? + .data; + Ok(Some(plan)) +} + +/// A view with each catalog-backed Iceberg table it reads pinned again. +/// +/// A view is any table DataFusion inlines: one whose provider has a logical +/// plan, such as a [`ViewTable`] from `CREATE VIEW` or a `DataFrame`'s +/// `into_view`. It keeps the original view's type, schema and definition, and +/// replaces only its plan. +#[derive(Debug)] +struct RepinnedView { + original: Arc, + /// The re-pinned plan, held as a [`ViewTable`] to plan a scan of it. + repinned: ViewTable, +} + +#[async_trait] +impl TableProvider for RepinnedView { + fn schema(&self) -> SchemaRef { + self.original.schema() + } + + fn table_type(&self) -> TableType { + self.original.table_type() + } + + fn get_table_definition(&self) -> Option<&str> { + self.original.get_table_definition() + } + + fn get_logical_plan(&self) -> Option> { + self.repinned.get_logical_plan() + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> Result> { + self.repinned.supports_filters_pushdown(filters) + } + + async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> Result> { + self.repinned.scan(state, projection, filters, limit).await + } +} + /// One version of an Iceberg table, read with the schema the client planned -/// against. +/// against, or the error loading it from its catalog returned. +/// +/// A table that couldn't be loaded still knows the catalog-backed table it was +/// pinned from, so a view over it reads it once its catalog can load it again. +/// Pushing filters into it, scanning it or encoding it returns the error, as +/// reading a missing table does in DataFusion. /// /// Not an [`IcebergStaticTableProvider`]: that derives its schema from the /// table, which may have evolved since planning, and is built asynchronously, @@ -155,9 +297,33 @@ impl SchemaProvider for PinningSchema { /// [`IcebergStaticTableProvider`]: iceberg_datafusion::IcebergStaticTableProvider #[derive(Debug)] pub(crate) struct PinnedTable { - pub(crate) table: Table, + pub(crate) table: Loaded, pub(crate) snapshot_id: Option, pub(crate) schema: SchemaRef, + /// The catalog-backed table it was pinned from, to pin it again. Not sent + /// between nodes. + pub(crate) origin: Option, +} + +impl PinnedTable { + /// `table`, a version of `provider` loaded from its catalog or the error + /// loading it returned, read with the provider's schema, which it caches + /// when it is created. + fn pin(provider: &IcebergCatalogTableProvider, table: Loaded) -> Self { + Self { + table, + snapshot_id: None, + schema: provider.schema(), + origin: Some(provider.clone()), + } + } + + /// The pinned version, or the error loading it returned. + pub(crate) fn table(&self) -> Result<&Table> { + self.table + .as_ref() + .map_err(|error| DataFusionError::Shared(Arc::clone(error))) + } } #[async_trait] @@ -177,12 +343,13 @@ impl TableProvider for PinnedTable { filters: &[Expr], limit: Option, ) -> Result> { - let source = IcebergDataSource::builder(self.table.clone(), self.schema.clone()) - .with_snapshot_id(self.snapshot_id) - .with_projection(projection) - .with_filters(filters) - .with_fetch(limit) - .build()?; + let source = + IcebergDataSource::builder(self.table()?.clone(), self.schema.clone()) + .with_snapshot_id(self.snapshot_id) + .with_projection(projection) + .with_filters(filters) + .with_fetch(limit) + .build()?; Ok(DataSourceExec::from_data_source(source)) } @@ -190,6 +357,8 @@ impl TableProvider for PinnedTable { &self, filters: &[&Expr], ) -> Result> { + // Fails a filtered query while it's planned, before it's sent. + self.table()?; Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) } diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 68d520d2ab..6c644cfb57 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -56,8 +56,8 @@ //! - Reads only. //! - Each scan runs as a single task. //! - No metadata tables such as `t$snapshots`. -//! - A view, or a `DataFrame` or prepared statement reused across runs, reads -//! the version from when it was planned. +//! - A `DataFrame` or prepared statement reused across runs reads the version +//! from when it was planned. //! - No encrypted tables. //! - Storage credentials travel in plain text. //! - A `UNION` or `UNION ALL` can fail, or with an `ORDER BY` return rows more diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 23cd8a7b91..87b1b48079 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -87,9 +87,10 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { ) -> Result> { let ProviderWire { table, snapshot_id } = wire::decode(buf)?; Ok(Arc::new(PinnedTable { - table: table.load()?, + table: Ok(table.load()?), snapshot_id, schema, + origin: None, })) } @@ -108,7 +109,7 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { } } else if let Some(pinned) = node.downcast_ref::() { ProviderWire { - table: TableWire::new(&pinned.table)?, + table: TableWire::new(pinned.table()?)?, snapshot_id: pinned.snapshot_id, } } else if node.is::() { diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index 2eed8d97b0..bf8a7e788c 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -29,10 +29,12 @@ mod standalone { BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, }; use ballista_iceberg::register_iceberg; + use datafusion::datasource::TableType; use datafusion::error::Result; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion::{assert_batches_eq, assert_batches_sorted_eq}; + use iceberg::{Catalog, TableIdent}; use crate::common::{assert_standalone_scans, catalog_table, insert, memory_catalog}; @@ -42,36 +44,172 @@ mod standalone { assert_standalone_scans(memory_catalog(dir.path()).await).await } - // A known limitation: a view stores its planned query, and tables are - // pinned when a query is planned. #[tokio::test] - async fn should_keep_a_views_version() -> Result<()> { + async fn should_read_a_views_latest_version() -> Result<()> { let dir = tempfile::tempdir()?; - let catalog = memory_catalog(dir.path()).await; - let state = SessionStateBuilder::new() - .with_config(SessionConfig::new_with_ballista()) - .with_default_features() - .build(); - let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; - ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; + let (catalog, ctx) = view_context(dir.path()).await?; insert(&catalog, "(1, 'a')").await; ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + // A view over a view, reading `t` again in a subquery. + ctx.sql("CREATE VIEW w AS SELECT * FROM v WHERE id IN (SELECT id FROM t)") + .await?; + insert(&catalog, "(2, 'b')").await; + + for view in ["v", "w"] { + let sql = format!("SELECT * FROM {view} ORDER BY id"); + let result = ctx.sql(&sql).await?.collect().await?; + assert_batches_eq!(LATEST, &result); + } + + Ok(()) + } + + #[tokio::test] + async fn should_read_a_dataframe_views_latest_version() -> Result<()> { + let dir = tempfile::tempdir()?; + let (catalog, ctx) = view_context(dir.path()).await?; + + insert(&catalog, "(1, 'a')").await; + let df = ctx.sql("SELECT * FROM t").await?; + ctx.register_table("v", df.clone().into_view())?; + ctx.register_table("tv", df.into_temporary_view())?; insert(&catalog, "(2, 'b')").await; - let result = ctx.sql("SELECT * FROM v").await?.collect().await?; + for (view, table_type) in [("v", TableType::View), ("tv", TableType::Temporary)] { + let sql = format!("SELECT * FROM {view} ORDER BY id"); + let result = ctx.sql(&sql).await?.collect().await?; + assert_batches_eq!(LATEST, &result); + assert_eq!(ctx.table_provider(view).await?.table_type(), table_type); + } + + Ok(()) + } + + #[tokio::test] + async fn should_read_a_view_created_while_its_table_was_gone() -> Result<()> { + let dir = tempfile::tempdir()?; + let (catalog, ctx) = view_context(dir.path()).await?; + + insert(&catalog, "(1, 'a')").await; + ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + // While `t` is gone, `w` is created over `v`, and reading it fails. + rename_table(&catalog, "t", "gone").await; + ctx.sql("CREATE VIEW w AS SELECT * FROM v").await?; + assert_table_gone(&ctx, "SELECT * FROM w").await; + // Once `t` is back, `w` reads its latest version. + rename_table(&catalog, "gone", "t").await; + insert(&catalog, "(2, 'b')").await; + let result = ctx + .sql("SELECT * FROM w ORDER BY id") + .await? + .collect() + .await?; + assert_batches_eq!(LATEST, &result); + + Ok(()) + } + + #[tokio::test] + async fn should_fail_to_read_a_view_whose_table_is_gone() -> Result<()> { + let dir = tempfile::tempdir()?; + let (catalog, ctx) = view_context(dir.path()).await?; + ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + rename_table(&catalog, "t", "gone").await; + + // Without a filter, the client fails to encode the view's scan. + assert_table_gone(&ctx, "SELECT * FROM v").await; + // With one, pushing the filter into the view's scan fails first. + assert_eq!( + query_error(&ctx, "SELECT * FROM v WHERE id = 1").await, + "Optimizer rule 'push_down_filter' failed\ncaused by\n\ + IcebergError(TableNotFound)\ncaused by\nExecution error: TableNotFound => \ + No such table: TableIdent { namespace: NamespaceIdent([\"ns\"]), name: \"t\" }" + ); + + Ok(()) + } + + #[tokio::test] + async fn should_describe_drop_and_replace_while_a_table_is_gone() -> Result<()> { + let dir = tempfile::tempdir()?; + let (catalog, ctx) = view_context(dir.path()).await?; + ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + ctx.sql("CREATE VIEW w AS SELECT * FROM t").await?; + rename_table(&catalog, "t", "gone").await; + + let result = ctx.sql("DESCRIBE v").await?.collect().await?; let expected = [ - "+----+------+", - "| id | name |", - "+----+------+", - "| 1 | a |", - "+----+------+", + "+-------------+-----------+-------------+", + "| column_name | data_type | is_nullable |", + "+-------------+-----------+-------------+", + "| id | Int64 | NO |", + "| name | Utf8 | YES |", + "+-------------+-----------+-------------+", ]; assert_batches_eq!(expected, &result); + ctx.sql("DROP VIEW v").await?; + assert!(!ctx.table_exist("v")?); + + ctx.sql("CREATE OR REPLACE VIEW w AS SELECT 1 AS x").await?; + let result = ctx.sql("SELECT * FROM w").await?.collect().await?; + let expected = ["+---+", "| x |", "+---+", "| 1 |", "+---+"]; + assert_batches_eq!(expected, &result); + + ctx.sql("DROP TABLE t").await?; + assert!(!ctx.table_exist("t")?); + Ok(()) } + /// `ns.t` after the view tests' two commits. + const LATEST: [&str; 6] = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 2 | b |", + "+----+------+", + ]; + + /// Standalone Ballista over an empty `ns.t` in a memory catalog in `dir`, + /// registered as `t`. + async fn view_context(dir: &Path) -> Result<(Arc, SessionContext)> { + let catalog = memory_catalog(dir).await; + let state = SessionStateBuilder::new() + .with_config(SessionConfig::new_with_ballista()) + .with_default_features() + .build(); + let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; + ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; + Ok((catalog, ctx)) + } + + /// The error running `sql` returns. + async fn query_error(ctx: &SessionContext, sql: &str) -> String { + let error = ctx.sql(sql).await.unwrap().collect().await.unwrap_err(); + error.to_string() + } + + /// Checks that running `sql` fails because `t` can't be loaded. The client + /// fails to encode the plan, and Ballista wraps that error in text of its + /// own, so only the cause is checked. + async fn assert_table_gone(ctx: &SessionContext, sql: &str) { + let error = query_error(ctx, sql).await; + assert!(error.contains("No such table"), "{error}"); + } + + /// Renames `ns.{from}` to `ns.{to}`, so a table registered from `ns.t` can + /// be made to fail to load, and then to load again. + async fn rename_table(catalog: &Arc, from: &str, to: &str) { + let ident = |name| TableIdent::from_strs(["ns", name]).unwrap(); + catalog + .rename_table(&ident(from), &ident(to)) + .await + .unwrap(); + } + /// Standalone Ballista over `ns.t` holding `(1, 'a'), (2, 'b')`, splitting /// every stage into one task per partition, so each branch of a union runs /// in a task of its own.