diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de0d42c..23afd59 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -148,6 +148,9 @@ jobs: dotnet pack "$proj" -c Release -o nupkg $vprop done dotnet pack src/Apache.DataFusion/Apache.DataFusion.csproj -c Release -o nupkg $vprop + for proj in src/Apache.DataFusion.TableProviders.*/Apache.DataFusion.TableProviders.*.csproj; do + dotnet pack "$proj" -c Release -o nupkg $vprop + done ls -l nupkg - name: Upload packages artifact diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs index 60f8b8a..34fbc0d 100644 --- a/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs +++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs @@ -23,17 +23,23 @@ string customerRegionsPath = WriteCustomerRegionsCsv(); using SessionContext context = new(); -context.RegisterSqlite(sqliteConnectionString); +context.RegisterSqlite(new SqliteDatabaseOptions +{ + ConnectionString = sqliteConnectionString, + SourceName = "orders", +}); context.RegisterCsv("customer_regions", customerRegionsPath, new CsvReadOptions { HasHeader = true }); -Console.WriteLine("Federated query joining SQLite orders with CSV customer regions:"); +Console.WriteLine("Federated query joining SQLite orders and customers with CSV customer regions:"); using DataFrame df = context.Sql(""" - SELECT r.region, o.customer, SUM(o.total) AS spend - FROM datafusion_orders AS o + SELECT r.region, c.loyalty_tier, o.customer, SUM(o.total) AS spend + FROM orders.datafusion_orders AS o + INNER JOIN orders.datafusion_customers AS c + ON o.customer = c.customer INNER JOIN customer_regions AS r ON o.customer = r.customer - GROUP BY r.region, o.customer - ORDER BY r.region, o.customer + GROUP BY r.region, c.loyalty_tier, o.customer + ORDER BY r.region, c.loyalty_tier, o.customer """); df.Show(); @@ -56,13 +62,24 @@ CREATE TABLE IF NOT EXISTS datafusion_orders ( total REAL NOT NULL ); + CREATE TABLE IF NOT EXISTS datafusion_customers ( + customer TEXT PRIMARY KEY, + loyalty_tier TEXT NOT NULL + ); + DELETE FROM datafusion_orders; + DELETE FROM datafusion_customers; INSERT INTO datafusion_orders (id, customer, total) VALUES (1, 'alice', 19.99), (2, 'bob', 7.50), (3, 'alice', 100.00), (4, 'carol', 42.25); + + INSERT INTO datafusion_customers (customer, loyalty_tier) VALUES + ('alice', 'gold'), + ('bob', 'silver'), + ('carol', 'bronze'); """; command.ExecuteNonQuery(); diff --git a/native/src/callback_table.rs b/native/src/callback_table.rs index 4ef65ba..28a9084 100644 --- a/native/src/callback_table.rs +++ b/native/src/callback_table.rs @@ -38,8 +38,8 @@ use std::sync::Arc; use ::arrow::array::RecordBatch; use ::arrow::datatypes::SchemaRef; use ::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; -use datafusion::catalog::{Session, TableProvider}; -use datafusion::common::ScalarValue; +use datafusion::catalog::{MemorySchemaProvider, Session, TableProvider}; +use datafusion::common::{ScalarValue, TableReference}; use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown, TableType}; @@ -404,17 +404,85 @@ pub extern "C" fn df_session_context_register_callback_table( take_result(move || { let ctx = unsafe { &*require_ptr(handle, "SessionContext handle")? }; let name = cstr(name, "name")?; - let schema: SchemaRef = Arc::new( - decode_optional_schema(schema_ptr, schema_len)? - .ok_or("callback table requires a non-empty schema")?, - ); - let table = ManagedCallbackTable { - schema, + register_callback_table( + ctx, + name, + schema_ptr, + schema_len, + supports_pushdown, callback, - context: Arc::new(guard), - supports_pushdown: supports_pushdown != 0, - }; - ctx.register_table(name, Arc::new(table))?; - Ok(()) + guard, + ) + }) +} + +#[no_mangle] +pub extern "C" fn df_session_context_register_callback_table_in_schema( + handle: *mut SessionContext, + schema_name: *const c_char, + table_name: *const c_char, + schema_ptr: *const u8, + schema_len: usize, + supports_pushdown: c_int, + callback: ScanCallback, + context: *mut c_void, + release: ReleaseCallback, +) -> c_int { + // Built before any fallible work so its Drop releases the managed handle on + // every early-return path; on success it moves into the adapter and fires + // when the table is dropped. + let guard = ContextGuard { + context, + release: Some(release), + }; + take_result(move || { + let ctx = unsafe { &*require_ptr(handle, "SessionContext handle")? }; + let schema_name = cstr(schema_name, "schema_name")?; + let table_name = cstr(table_name, "table_name")?; + register_default_schema(ctx, &schema_name)?; + register_callback_table( + ctx, + TableReference::partial(schema_name, table_name), + schema_ptr, + schema_len, + supports_pushdown, + callback, + guard, + ) }) } + +fn register_callback_table( + ctx: &SessionContext, + table_ref: impl Into, + schema_ptr: *const u8, + schema_len: usize, + supports_pushdown: c_int, + callback: ScanCallback, + guard: ContextGuard, +) -> crate::NativeResult<()> { + let schema: SchemaRef = Arc::new( + decode_optional_schema(schema_ptr, schema_len)? + .ok_or("callback table requires a non-empty schema")?, + ); + let table = ManagedCallbackTable { + schema, + callback, + context: Arc::new(guard), + supports_pushdown: supports_pushdown != 0, + }; + ctx.register_table(table_ref, Arc::new(table))?; + Ok(()) +} + +fn register_default_schema(ctx: &SessionContext, schema_name: &str) -> crate::NativeResult<()> { + let state = ctx.state(); + let default_catalog = state.config().options().catalog.default_catalog.as_str(); + let catalog = ctx + .catalog(default_catalog) + .ok_or_else(|| format!("failed to resolve catalog: {default_catalog}"))?; + if catalog.schema(schema_name).is_none() { + catalog.register_schema(schema_name, Arc::new(MemorySchemaProvider::new()))?; + } + Ok(()) +} diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs index 4440322..2866949 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs @@ -21,6 +21,8 @@ public sealed class ClickHouseDatabaseOptions { public required string ConnectionString { get; init; } + public string? SourceName { get; init; } + public string? DatabaseName { get; init; } public bool IncludeViews { get; init; } = true; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs index 740e39b..15185d6 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs @@ -46,7 +46,7 @@ public static void RegisterClickHouse(this SessionContext context, ClickHouseDat foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) { - context.RegisterStreamingTable(table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions + RegisterSourceTable(context, options.SourceName, table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions { ConnectionString = options.ConnectionString, DatabaseName = table.DatabaseName, @@ -108,4 +108,15 @@ ORDER BY name } private sealed record ClickHouseTable(string DatabaseName, string TableName); + + private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider) + { + if (string.IsNullOrWhiteSpace(sourceName)) + { + context.RegisterStreamingTable(tableName, provider); + return; + } + + context.RegisterStreamingTable(sourceName, tableName, provider); + } } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs index 64250bf..0b7a8ed 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs @@ -21,6 +21,8 @@ public sealed class MongoDbDatabaseOptions { public required string ConnectionString { get; init; } + public string? SourceName { get; init; } + public string? DatabaseName { get; init; } public int BatchSize { get; init; } = 1024; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs index 643894f..86467d3 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs @@ -53,7 +53,7 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO IMongoDatabase database = client.GetDatabase(databaseName); foreach (string collectionName in database.ListCollectionNames().ToEnumerable()) { - context.RegisterStreamingTable(collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions + RegisterSourceTable(context, options.SourceName, collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions { ConnectionString = options.ConnectionString, DatabaseName = databaseName, @@ -113,4 +113,15 @@ private static string ResolveDatabaseName(string connectionString, string? datab throw new ArgumentException("Database name must be provided in MongoDB options or connection string.", nameof(connectionString)); } + + private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider) + { + if (string.IsNullOrWhiteSpace(sourceName)) + { + context.RegisterStreamingTable(tableName, provider); + return; + } + + context.RegisterStreamingTable(sourceName, tableName, provider); + } } diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs index 6ff05bb..f3f0053 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs @@ -21,6 +21,8 @@ public sealed class MySqlDatabaseOptions { public required string ConnectionString { get; init; } + public string? SourceName { get; init; } + public string? DatabaseName { get; init; } public bool IncludeViews { get; init; } = true; diff --git a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs index f7e87de..2f39496 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs @@ -45,7 +45,7 @@ public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptio foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) { - context.RegisterStreamingTable(table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions + RegisterSourceTable(context, options.SourceName, table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions { ConnectionString = options.ConnectionString, DatabaseName = table.DatabaseName, @@ -100,4 +100,15 @@ ORDER BY table_name } private sealed record MySqlTable(string DatabaseName, string TableName); + + private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider) + { + if (string.IsNullOrWhiteSpace(sourceName)) + { + context.RegisterStreamingTable(tableName, provider); + return; + } + + context.RegisterStreamingTable(sourceName, tableName, provider); + } } diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs index aef7378..4e53a06 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs @@ -21,6 +21,8 @@ public sealed class PostgreSqlDatabaseOptions { public required string ConnectionString { get; init; } + public string? SourceName { get; init; } + public IReadOnlyCollection Schemas { get; init; } = ["public"]; public bool IncludeViews { get; init; } = true; diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs index 6843e0d..68ad98b 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs @@ -57,7 +57,7 @@ public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDat foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews)) { string registrationName = RegistrationName(table, schemas); - context.RegisterStreamingTable(registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions + RegisterSourceTable(context, options.SourceName, registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions { ConnectionString = options.ConnectionString, SchemaName = table.SchemaName, @@ -126,4 +126,15 @@ private static string RegistrationName(PostgreSqlTable table, IReadOnlyCollectio : $"{table.SchemaName}_{table.TableName}"; private sealed record PostgreSqlTable(string SchemaName, string TableName); + + private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider) + { + if (string.IsNullOrWhiteSpace(sourceName)) + { + context.RegisterStreamingTable(tableName, provider); + return; + } + + context.RegisterStreamingTable(sourceName, tableName, provider); + } } diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs index 15fa8d7..7a5406b 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs @@ -46,7 +46,7 @@ public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOpt foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews)) { - context.RegisterStreamingTable(tableName, new SqliteStreamingTableProvider(new SqliteTableOptions + RegisterSourceTable(context, options.SourceName, tableName, new SqliteStreamingTableProvider(new SqliteTableOptions { ConnectionString = options.ConnectionString, TableName = tableName, @@ -98,4 +98,15 @@ ORDER BY name yield return reader.GetString(0); } } + + private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider) + { + if (string.IsNullOrWhiteSpace(sourceName)) + { + context.RegisterStreamingTable(tableName, provider); + return; + } + + context.RegisterStreamingTable(sourceName, tableName, provider); + } } diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs index 9f7eb32..c9a696f 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs @@ -21,6 +21,8 @@ public sealed class SqliteDatabaseOptions { public required string ConnectionString { get; init; } + public string? SourceName { get; init; } + public bool IncludeViews { get; init; } = true; public int BatchSize { get; init; } = 1024; diff --git a/src/Apache.DataFusion/Interop/NativeMethods/TableProviderNativeMethods.cs b/src/Apache.DataFusion/Interop/NativeMethods/TableProviderNativeMethods.cs index 8fb80ea..68df7f6 100644 --- a/src/Apache.DataFusion/Interop/NativeMethods/TableProviderNativeMethods.cs +++ b/src/Apache.DataFusion/Interop/NativeMethods/TableProviderNativeMethods.cs @@ -69,4 +69,16 @@ internal static extern int df_session_context_register_callback_table( CallbackTableScan scan, IntPtr context, CallbackTableRelease release); + + [DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)] + internal static extern int df_session_context_register_callback_table_in_schema( + IntPtr handle, + IntPtr schemaName, + IntPtr tableName, + IntPtr schemaPtr, + nuint schemaLen, + int supportsPushdown, + CallbackTableScan scan, + IntPtr context, + CallbackTableRelease release); } diff --git a/src/Apache.DataFusion/Session/SessionContext.Registration.cs b/src/Apache.DataFusion/Session/SessionContext.Registration.cs index 9254d53..f9ec47a 100644 --- a/src/Apache.DataFusion/Session/SessionContext.Registration.cs +++ b/src/Apache.DataFusion/Session/SessionContext.Registration.cs @@ -78,6 +78,46 @@ public void RegisterStreamingTable(string name, StreamingTableProvider provider) StreamingReleaseCallback)); } + /// + /// Register a lazy, streaming table in a named DataFusion schema. The table + /// can be referenced from SQL as schemaName.tableName. + /// + public void RegisterStreamingTable(string schemaName, string tableName, StreamingTableProvider provider) + { + ArgumentNullException.ThrowIfNull(provider); + if (string.IsNullOrWhiteSpace(schemaName)) + { + throw new ArgumentException("Schema name cannot be null or whitespace.", nameof(schemaName)); + } + + if (string.IsNullOrWhiteSpace(tableName)) + { + throw new ArgumentException("Table name cannot be null or whitespace.", nameof(tableName)); + } + + byte[] schemaIpc = SerializeSchema(provider.Schema); + GCHandle gcHandle = GCHandle.Alloc(provider); + IntPtr context = GCHandle.ToIntPtr(gcHandle); + using NativeUtf8String nativeSchemaName = new(schemaName); + using NativeUtf8String nativeTableName = new(tableName); + using NativeByteArray schema = new(schemaIpc); + + // The native adapter takes ownership of the context handle and always + // invokes the release callback exactly once - synchronously here on + // failure, or when the table is dropped on success - so the handle is + // freed in StreamingReleaseThunk and never here. + NativeMethods.ThrowIfError(NativeMethods.df_session_context_register_callback_table_in_schema( + Handle, + nativeSchemaName.Pointer, + nativeTableName.Pointer, + schema.Pointer, + schema.Length, + provider.SupportsPushdown ? 1 : 0, + StreamingScanCallback, + context, + StreamingReleaseCallback)); + } + public void RegisterScalarUdf(ScalarUdf udf) { ArgumentNullException.ThrowIfNull(udf); diff --git a/src/Apache.DataFusion/TableProvider/StreamingTableProvider.cs b/src/Apache.DataFusion/TableProvider/StreamingTableProvider.cs index 70dd91b..593c185 100644 --- a/src/Apache.DataFusion/TableProvider/StreamingTableProvider.cs +++ b/src/Apache.DataFusion/TableProvider/StreamingTableProvider.cs @@ -22,7 +22,7 @@ namespace Apache.DataFusion; /// /// A lazy, streaming table provider implemented in managed code. Register an -/// instance with ; DataFusion +/// instance with ; DataFusion /// then calls on every scan and reads the returned stream /// over the Arrow C Data Interface, so rows are pulled on demand straight from /// the managed source (for example an ADO.NET reader) without a native database diff --git a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs index 2f7a639..098ed2a 100644 --- a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs +++ b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs @@ -44,6 +44,52 @@ public void RegisterStreamingTable_QueriesRows() Assert.Equal(1UL, filtered.Count()); } + [Fact] + public void RegisterStreamingTable_InSchemaQueriesRows() + { + InMemoryStreamingProvider provider = + new(BuildSchema(), [1, 2], ["Alice", "Bob"]); + + using SessionContext context = new(); + context.RegisterStreamingTable("sqlite1", "people", provider); + + using DataFrame result = context.Sql("SELECT name FROM sqlite1.people WHERE id = 2"); + Assert.Equal(1UL, result.Count()); + } + + [Fact] + public void RegisterStreamingTable_InDifferentSchemasAllowsSameTableName() + { + InMemoryStreamingProvider primary = + new(BuildSchema(), [1, 2], ["Alice", "Bob"]); + InMemoryStreamingProvider secondary = + new(BuildSchema(), [2, 3], ["Bob", "Carol"]); + + using SessionContext context = new(); + context.RegisterStreamingTable("primary_source", "people", primary); + context.RegisterStreamingTable("secondary_source", "people", secondary); + + using DataFrame result = context.Sql(""" + SELECT p.name + FROM primary_source.people AS p + INNER JOIN secondary_source.people AS s + ON p.id = s.id + """); + Assert.Equal(1UL, result.Count()); + } + + [Fact] + public void RegisterStreamingTable_InSchemaValidatesNames() + { + InMemoryStreamingProvider provider = + new(BuildSchema(), [1], ["Alice"]); + + using SessionContext context = new(); + + Assert.Throws(() => context.RegisterStreamingTable("", "people", provider)); + Assert.Throws(() => context.RegisterStreamingTable("source", "", provider)); + } + [Fact] public void RegisterStreamingTable_ScansLazilyOnEachQuery() {