diff --git a/AGENTS.md b/AGENTS.md index 1859768..71b0aad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,7 @@ The playground config holds `[[catalogs]]` entries with `name`, `type` (`rest` o ### Catalog → schema → table -`IcebergCatalogProvider` wraps an `iceberg::Catalog` and builds one `IcebergSchemaProvider` per namespace, each holding an `IcebergCatalogTableProvider` per table. Namespaces and tables are listed once, at construction, and are never refreshed. Tables created by other clients after that point are invisible. +`IcebergCatalogProvider` wraps an `iceberg::Catalog` and builds one `IcebergSchemaProvider` per namespace, each holding the names of its tables. Each table lookup loads the table and builds a new `IcebergCatalogTableProvider`, so every query sees the table's current schema. Namespaces and table names are listed once, at construction, and are never refreshed. Tables created by other clients after that point are invisible. - Names like `$snapshots`, `$manifests`, and `$history` resolve to `IcebergMetadataTableProvider`. - SQL `CREATE TABLE` and `DROP TABLE` reach the *synchronous* `SchemaProvider::register_table` and `deregister_table` methods, which call the async catalog through `spawn_blocking` + `block_on`. `register_table` rejects input tables that contain rows, so CTAS isn't supported. It auto-assigns field IDs and uses format V2 unless the schema needs V3. diff --git a/crates/datafusion/src/catalog/schema.rs b/crates/datafusion/src/catalog/schema.rs index 900e874..cc3e985 100644 --- a/crates/datafusion/src/catalog/schema.rs +++ b/crates/datafusion/src/catalog/schema.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use async_trait::async_trait; -use dashmap::DashMap; +use dashmap::DashSet; use datafusion::catalog::SchemaProvider; use datafusion::common::{exec_datafusion_err, exec_err, plan_datafusion_err}; use datafusion::datasource::TableProvider; @@ -26,47 +26,42 @@ use datafusion::error::Result; use datafusion::execution::TaskContext; use datafusion::prelude::SessionContext; use futures::TryStreamExt; -use futures::future::try_join_all; use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids; use iceberg::inspect::MetadataTableType; use iceberg::spec::FormatVersion; use iceberg::{Catalog, NamespaceIdent, TableCreation, TableIdent}; +use crate::metadata::IcebergMetadataTableProvider; use crate::table_provider::IcebergCatalogTableProvider; use crate::to_datafusion_error; /// Represents a [`SchemaProvider`] for the Iceberg [`Catalog`], managing /// access to table providers within a specific namespace. +/// +/// Each lookup loads the table from the catalog and returns a new provider, so +/// a query sees the table's schema as it is when the query is planned. #[derive(Debug)] pub(crate) struct IcebergSchemaProvider { /// Reference to the Iceberg catalog catalog: Arc, /// The namespace this schema represents namespace: NamespaceIdent, - /// A concurrent map where keys are table names - /// and values are dynamic references to objects implementing the - /// [`TableProvider`] trait. + /// The names of the namespace's tables. /// Wrapped in Arc to allow sharing across async boundaries in register_table. - tables: Arc>>, + tables: Arc>, } impl IcebergSchemaProvider { /// Asynchronously tries to construct a new [`IcebergSchemaProvider`] - /// using the given client to fetch and initialize table providers for - /// the provided namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves a list of table names - /// attempts to create a table provider for each table name, and - /// collects these providers into a `HashMap`. + /// for the provided namespace in the Iceberg [`Catalog`], listing the + /// namespace's tables. pub(crate) async fn try_new( client: Arc, namespace: NamespaceIdent, ) -> Result { - // TODO: - // Tables and providers should be cached based on table_name - // if we have a cache miss; we update our internal cache & check again - // As of right now; tables might become stale. - let table_names: Vec<_> = client + // TODO: The table names are listed once, so tables created or dropped + // by other clients after this point are not seen (#24). + let tables = client .list_tables(&namespace) .await .map_err(to_datafusion_error)? @@ -74,29 +69,10 @@ impl IcebergSchemaProvider { .map(|tbl| tbl.name().to_string()) .collect(); - let providers = try_join_all( - table_names - .iter() - .map(|name| { - IcebergCatalogTableProvider::try_new( - client.clone(), - namespace.clone(), - name, - ) - }) - .collect::>(), - ) - .await?; - - let tables = Arc::new(DashMap::new()); - for (name, provider) in table_names.into_iter().zip(providers) { - tables.insert(name, Arc::new(provider)); - } - Ok(IcebergSchemaProvider { catalog: client, namespace, - tables, + tables: Arc::new(tables), }) } } @@ -119,29 +95,51 @@ impl SchemaProvider for IcebergSchemaProvider { fn table_exist(&self, name: &str) -> bool { if let Some((table_name, metadata_table_name)) = name.split_once('$') { - self.tables.contains_key(table_name) + self.tables.contains(table_name) && MetadataTableType::try_from(metadata_table_name).is_ok() } else { - self.tables.contains_key(name) + self.tables.contains(name) } } async fn table(&self, name: &str) -> Result>> { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - let metadata_table_type = MetadataTableType::try_from(metadata_table_name) - .map_err(|e| plan_datafusion_err!("{e}"))?; - if let Some(table) = self.tables.get(table_name) { - let metadata_table = table.metadata_table(metadata_table_type).await?; - return Ok(Some(Arc::new(metadata_table))); - } else { - return Ok(None); + let (table_name, metadata_table_type) = match name.split_once('$') { + Some((table_name, metadata_table_name)) => { + let metadata_table_type = + MetadataTableType::try_from(metadata_table_name) + .map_err(|e| plan_datafusion_err!("{e}"))?; + (table_name, Some(metadata_table_type)) } + None => (name, None), + }; + if !self.tables.contains(table_name) { + return Ok(None); } - Ok(self - .tables - .get(name) - .map(|entry| entry.value().clone() as Arc)) + let provider: Arc = match metadata_table_type { + None => Arc::new( + IcebergCatalogTableProvider::try_new( + self.catalog.clone(), + self.namespace.clone(), + table_name, + ) + .await?, + ), + Some(metadata_table_type) => { + let table_ident = + TableIdent::new(self.namespace.clone(), table_name.into()); + let table = self + .catalog + .load_table(&table_ident) + .await + .map_err(to_datafusion_error)?; + Arc::new(IcebergMetadataTableProvider::try_new( + table, + metadata_table_type, + )?) + } + }; + Ok(Some(provider)) } fn register_table( @@ -190,16 +188,7 @@ impl SchemaProvider for IcebergSchemaProvider { .await .map_err(to_datafusion_error)?; - // Create a new table provider using the catalog reference - let table_provider = IcebergCatalogTableProvider::try_new( - catalog.clone(), - namespace.clone(), - name_clone.clone(), - ) - .await?; - - // Store the new table provider - tables.insert(name_clone, Arc::new(table_provider)); + tables.insert(name_clone); Ok(None) }) @@ -226,20 +215,23 @@ impl SchemaProvider for IcebergSchemaProvider { let result = tokio::task::spawn_blocking(move || { let rt = tokio::runtime::Handle::current(); rt.block_on(async move { - let table_ident = TableIdent::new(namespace, table_name.clone()); + // Load the table before dropping it, to return its provider + let removed = IcebergCatalogTableProvider::try_new( + catalog.clone(), + namespace, + table_name.clone(), + ) + .await?; // Drop the table from the Iceberg catalog catalog - .drop_table(&table_ident) + .drop_table(removed.table_ident()) .await .map_err(to_datafusion_error)?; - // Remove from local cache and return the removed provider - let removed = tables - .remove(&table_name) - .map(|(_, table)| table as Arc); + tables.remove(&table_name); - Ok(removed) + Ok(Some(Arc::new(removed) as Arc)) }) }); diff --git a/crates/datafusion/src/table_provider/catalog_table_provider.rs b/crates/datafusion/src/table_provider/catalog_table_provider.rs index 70fd7d8..584aa13 100644 --- a/crates/datafusion/src/table_provider/catalog_table_provider.rs +++ b/crates/datafusion/src/table_provider/catalog_table_provider.rs @@ -30,12 +30,10 @@ use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; use iceberg::spec::TableProperties; use iceberg::{Catalog, NamespaceIdent, TableIdent}; use crate::error::to_datafusion_error; -use crate::metadata::table::IcebergMetadataTableProvider; use crate::read::{FileScanTaskProviderBuilder, IcebergDataSource}; use crate::write::commit::IcebergCommitExec; use crate::write::exec::IcebergWriteExec; @@ -90,19 +88,6 @@ impl IcebergCatalogTableProvider { }) } - pub(crate) async fn metadata_table( - &self, - r#type: MetadataTableType, - ) -> Result { - // Load fresh table metadata for metadata table access - let table = self - .catalog - .load_table(&self.table_ident) - .await - .map_err(to_datafusion_error)?; - IcebergMetadataTableProvider::try_new(table, r#type) - } - /// The catalog this provider loads its table from and commits through. pub fn catalog(&self) -> &Arc { &self.catalog diff --git a/crates/datafusion/tests/main.rs b/crates/datafusion/tests/main.rs index 263af20..e762f48 100644 --- a/crates/datafusion/tests/main.rs +++ b/crates/datafusion/tests/main.rs @@ -28,3 +28,4 @@ mod plan_nodes; mod projection_pushdown; mod schema_evolution; mod statistics; +mod table_lookup; diff --git a/crates/datafusion/tests/table_lookup.rs b/crates/datafusion/tests/table_lookup.rs new file mode 100644 index 0000000..e93fe38 --- /dev/null +++ b/crates/datafusion/tests/table_lookup.rs @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Each query looks its tables up in the catalog afresh, so it sees changes +//! other clients made to a table's schema after the catalog was registered. + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::error::Error; + use std::sync::Arc; + + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::prelude::SessionContext; + use iceberg::spec::{PrimitiveType, Type}; + use iceberg::transaction::{ + AddColumn, ApplyTransactionAction, Transaction, UpdateSchemaAction, + }; + use iceberg::{NamespaceIdent, TableIdent}; + use iceberg_datafusion::IcebergCatalogProvider; + use iceberg_datafusion_test_tools::FixtureCatalog; + + type Result = std::result::Result>; + + /// Creates a catalog with the empty namespace `ns`. Returns a session with + /// the catalog registered as `c`, and another client of the same catalog. + async fn setup() -> Result<(Session, OtherClient)> { + let fixtures = FixtureCatalog::try_new().await?; + fixtures + .catalog() + .create_namespace(&NamespaceIdent::new("ns".to_string()), HashMap::new()) + .await?; + Ok((Session::new(&fixtures).await?, OtherClient { fixtures })) + } + + /// A DataFusion session with the catalog registered as `c`. + struct Session { + ctx: SessionContext, + } + + impl Session { + async fn new(fixtures: &FixtureCatalog) -> Result { + let ctx = SessionContext::new(); + let provider = IcebergCatalogProvider::try_new(fixtures.catalog()).await?; + ctx.register_catalog("c", Arc::new(provider)); + Ok(Self { ctx }) + } + + async fn sql(&self, sql: &str) -> Result { + let batches = self.ctx.sql(sql).await?.collect().await?; + Ok(pretty_format_batches(&batches)?.to_string()) + } + } + + /// Another client of the catalog, changing table `ns.t` as another engine + /// would, without the session knowing. + struct OtherClient { + fixtures: FixtureCatalog, + } + + impl OtherClient { + async fn update_schema( + &self, + change: impl FnOnce(UpdateSchemaAction) -> UpdateSchemaAction, + ) -> Result<()> { + let ident = TableIdent::from_strs(["ns", "t"])?; + let catalog = self.fixtures.catalog(); + let table = catalog.load_table(&ident).await?; + let tx = Transaction::new(&table); + change(tx.update_schema()) + .apply(tx)? + .commit(catalog.as_ref()) + .await?; + Ok(()) + } + + /// Runs `sql` in a session of its own, started now. + async fn sql(&self, sql: &str) -> Result { + Session::new(&self.fixtures).await?.sql(sql).await + } + } + + #[tokio::test] + async fn inserts_into_a_column_another_client_added() -> Result<()> { + let (session, other_client) = setup().await?; + session + .sql("CREATE TABLE c.ns.t (id INT NOT NULL, name VARCHAR NOT NULL)") + .await?; + session.sql("INSERT INTO c.ns.t VALUES (1, 'a')").await?; + + other_client + .update_schema(|schema| { + schema.add_column(AddColumn::optional( + "email", + Type::Primitive(PrimitiveType::String), + )) + }) + .await?; + + session + .sql("INSERT INTO c.ns.t (id, name) VALUES (2, 'b')") + .await?; + session + .sql("INSERT INTO c.ns.t VALUES (3, 'c', 'c@example.com')") + .await?; + let rows = session.sql("SELECT * FROM c.ns.t ORDER BY id").await?; + insta::assert_snapshot!(rows, @r" + +----+------+---------------+ + | id | name | email | + +----+------+---------------+ + | 1 | a | | + | 2 | b | | + | 3 | c | c@example.com | + +----+------+---------------+ + "); + + Ok(()) + } + + /// Dropping a column and adding it back is how a column's type is changed + /// when the change isn't an allowed promotion. + #[tokio::test] + async fn reads_a_column_another_client_dropped_and_added_back() -> Result<()> { + let (session, other_client) = setup().await?; + session + .sql("CREATE TABLE c.ns.t (id INT NOT NULL, price DOUBLE NOT NULL)") + .await?; + session.sql("INSERT INTO c.ns.t VALUES (1, 1.5)").await?; + + other_client + .update_schema(|schema| schema.delete_column("price")) + .await?; + other_client + .update_schema(|schema| { + schema.add_column(AddColumn::optional( + "price", + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + )) + }) + .await?; + other_client + .sql("INSERT INTO c.ns.t VALUES (2, 2.25)") + .await?; + + let rows = session + .sql("SELECT *, price * 2 AS doubled FROM c.ns.t ORDER BY id") + .await?; + insta::assert_snapshot!(rows, @r" + +----+-------+---------+ + | id | price | doubled | + +----+-------+---------+ + | 1 | | | + | 2 | 2.25 | 4.50 | + +----+-------+---------+ + "); + let nulls = session + .sql("SELECT count(*) AS nulls FROM c.ns.t WHERE price IS NULL") + .await?; + insta::assert_snapshot!(nulls, @r" + +-------+ + | nulls | + +-------+ + | 1 | + +-------+ + "); + + Ok(()) + } +}