diff --git a/AGENTS.md b/AGENTS.md
index 1859768..71b0aad 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -77,7 +77,7 @@ The playground config holds `[[catalogs]]` entries with `name`, `type` (`rest` o
### Catalog → schema → table
-`IcebergCatalogProvider` wraps an `iceberg::Catalog` and builds one `IcebergSchemaProvider` per namespace, each holding an `IcebergCatalogTableProvider` per table. Namespaces and tables are listed once, at construction, and are never refreshed. Tables created by other clients after that point are invisible.
+`IcebergCatalogProvider` wraps an `iceberg::Catalog` and builds one `IcebergSchemaProvider` per namespace, each holding the names of its tables. Each table lookup loads the table and builds a new `IcebergCatalogTableProvider`, so every query sees the table's current schema. Namespaces and table names are listed once, at construction, and are never refreshed. Tables created by other clients after that point are invisible.
- Names like `
$snapshots`, `$manifests`, and `$history` resolve to `IcebergMetadataTableProvider`.
- SQL `CREATE TABLE` and `DROP TABLE` reach the *synchronous* `SchemaProvider::register_table` and `deregister_table` methods, which call the async catalog through `spawn_blocking` + `block_on`. `register_table` rejects input tables that contain rows, so CTAS isn't supported. It auto-assigns field IDs and uses format V2 unless the schema needs V3.
diff --git a/crates/datafusion/src/catalog/schema.rs b/crates/datafusion/src/catalog/schema.rs
index 900e874..cc3e985 100644
--- a/crates/datafusion/src/catalog/schema.rs
+++ b/crates/datafusion/src/catalog/schema.rs
@@ -18,7 +18,7 @@
use std::sync::Arc;
use async_trait::async_trait;
-use dashmap::DashMap;
+use dashmap::DashSet;
use datafusion::catalog::SchemaProvider;
use datafusion::common::{exec_datafusion_err, exec_err, plan_datafusion_err};
use datafusion::datasource::TableProvider;
@@ -26,47 +26,42 @@ use datafusion::error::Result;
use datafusion::execution::TaskContext;
use datafusion::prelude::SessionContext;
use futures::TryStreamExt;
-use futures::future::try_join_all;
use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids;
use iceberg::inspect::MetadataTableType;
use iceberg::spec::FormatVersion;
use iceberg::{Catalog, NamespaceIdent, TableCreation, TableIdent};
+use crate::metadata::IcebergMetadataTableProvider;
use crate::table_provider::IcebergCatalogTableProvider;
use crate::to_datafusion_error;
/// Represents a [`SchemaProvider`] for the Iceberg [`Catalog`], managing
/// access to table providers within a specific namespace.
+///
+/// Each lookup loads the table from the catalog and returns a new provider, so
+/// a query sees the table's schema as it is when the query is planned.
#[derive(Debug)]
pub(crate) struct IcebergSchemaProvider {
/// Reference to the Iceberg catalog
catalog: Arc,
/// The namespace this schema represents
namespace: NamespaceIdent,
- /// A concurrent map where keys are table names
- /// and values are dynamic references to objects implementing the
- /// [`TableProvider`] trait.
+ /// The names of the namespace's tables.
/// Wrapped in Arc to allow sharing across async boundaries in register_table.
- tables: Arc>>,
+ tables: Arc>,
}
impl IcebergSchemaProvider {
/// Asynchronously tries to construct a new [`IcebergSchemaProvider`]
- /// using the given client to fetch and initialize table providers for
- /// the provided namespace in the Iceberg [`Catalog`].
- ///
- /// This method retrieves a list of table names
- /// attempts to create a table provider for each table name, and
- /// collects these providers into a `HashMap`.
+ /// for the provided namespace in the Iceberg [`Catalog`], listing the
+ /// namespace's tables.
pub(crate) async fn try_new(
client: Arc,
namespace: NamespaceIdent,
) -> Result {
- // TODO:
- // Tables and providers should be cached based on table_name
- // if we have a cache miss; we update our internal cache & check again
- // As of right now; tables might become stale.
- let table_names: Vec<_> = client
+ // TODO: The table names are listed once, so tables created or dropped
+ // by other clients after this point are not seen (#24).
+ let tables = client
.list_tables(&namespace)
.await
.map_err(to_datafusion_error)?
@@ -74,29 +69,10 @@ impl IcebergSchemaProvider {
.map(|tbl| tbl.name().to_string())
.collect();
- let providers = try_join_all(
- table_names
- .iter()
- .map(|name| {
- IcebergCatalogTableProvider::try_new(
- client.clone(),
- namespace.clone(),
- name,
- )
- })
- .collect::>(),
- )
- .await?;
-
- let tables = Arc::new(DashMap::new());
- for (name, provider) in table_names.into_iter().zip(providers) {
- tables.insert(name, Arc::new(provider));
- }
-
Ok(IcebergSchemaProvider {
catalog: client,
namespace,
- tables,
+ tables: Arc::new(tables),
})
}
}
@@ -119,29 +95,51 @@ impl SchemaProvider for IcebergSchemaProvider {
fn table_exist(&self, name: &str) -> bool {
if let Some((table_name, metadata_table_name)) = name.split_once('$') {
- self.tables.contains_key(table_name)
+ self.tables.contains(table_name)
&& MetadataTableType::try_from(metadata_table_name).is_ok()
} else {
- self.tables.contains_key(name)
+ self.tables.contains(name)
}
}
async fn table(&self, name: &str) -> Result