Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ jobs:
dotnet pack "$proj" -c Release -o nupkg $vprop
done
dotnet pack src/Apache.DataFusion/Apache.DataFusion.csproj -c Release -o nupkg $vprop
for proj in src/Apache.DataFusion.TableProviders.*/Apache.DataFusion.TableProviders.*.csproj; do
dotnet pack "$proj" -c Release -o nupkg $vprop
done
ls -l nupkg

- name: Upload packages artifact
Expand Down
29 changes: 23 additions & 6 deletions examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,23 @@
string customerRegionsPath = WriteCustomerRegionsCsv();

using SessionContext context = new();
context.RegisterSqlite(sqliteConnectionString);
context.RegisterSqlite(new SqliteDatabaseOptions
{
ConnectionString = sqliteConnectionString,
SourceName = "orders",
});
context.RegisterCsv("customer_regions", customerRegionsPath, new CsvReadOptions { HasHeader = true });

Console.WriteLine("Federated query joining SQLite orders with CSV customer regions:");
Console.WriteLine("Federated query joining SQLite orders and customers with CSV customer regions:");
using DataFrame df = context.Sql("""
SELECT r.region, o.customer, SUM(o.total) AS spend
FROM datafusion_orders AS o
SELECT r.region, c.loyalty_tier, o.customer, SUM(o.total) AS spend
FROM orders.datafusion_orders AS o
INNER JOIN orders.datafusion_customers AS c
ON o.customer = c.customer
INNER JOIN customer_regions AS r
ON o.customer = r.customer
GROUP BY r.region, o.customer
ORDER BY r.region, o.customer
GROUP BY r.region, c.loyalty_tier, o.customer
ORDER BY r.region, c.loyalty_tier, o.customer
""");
df.Show();

Expand All @@ -56,13 +62,24 @@ CREATE TABLE IF NOT EXISTS datafusion_orders (
total REAL NOT NULL
);

CREATE TABLE IF NOT EXISTS datafusion_customers (
customer TEXT PRIMARY KEY,
loyalty_tier TEXT NOT NULL
);

DELETE FROM datafusion_orders;
DELETE FROM datafusion_customers;

INSERT INTO datafusion_orders (id, customer, total) VALUES
(1, 'alice', 19.99),
(2, 'bob', 7.50),
(3, 'alice', 100.00),
(4, 'carol', 42.25);

INSERT INTO datafusion_customers (customer, loyalty_tier) VALUES
('alice', 'gold'),
('bob', 'silver'),
('carol', 'bronze');
""";
command.ExecuteNonQuery();

Expand Down
94 changes: 81 additions & 13 deletions native/src/callback_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ use std::sync::Arc;
use ::arrow::array::RecordBatch;
use ::arrow::datatypes::SchemaRef;
use ::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
use datafusion::catalog::{Session, TableProvider};
use datafusion::common::ScalarValue;
use datafusion::catalog::{MemorySchemaProvider, Session, TableProvider};
use datafusion::common::{ScalarValue, TableReference};
use datafusion::error::DataFusionError;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown, TableType};
Expand Down Expand Up @@ -404,17 +404,85 @@ pub extern "C" fn df_session_context_register_callback_table(
take_result(move || {
let ctx = unsafe { &*require_ptr(handle, "SessionContext handle")? };
let name = cstr(name, "name")?;
let schema: SchemaRef = Arc::new(
decode_optional_schema(schema_ptr, schema_len)?
.ok_or("callback table requires a non-empty schema")?,
);
let table = ManagedCallbackTable {
schema,
register_callback_table(
ctx,
name,
schema_ptr,
schema_len,
supports_pushdown,
callback,
context: Arc::new(guard),
supports_pushdown: supports_pushdown != 0,
};
ctx.register_table(name, Arc::new(table))?;
Ok(())
guard,
)
})
}

#[no_mangle]
pub extern "C" fn df_session_context_register_callback_table_in_schema(
handle: *mut SessionContext,
schema_name: *const c_char,
table_name: *const c_char,
schema_ptr: *const u8,
schema_len: usize,
supports_pushdown: c_int,
callback: ScanCallback,
context: *mut c_void,
release: ReleaseCallback,
) -> c_int {
// Built before any fallible work so its Drop releases the managed handle on
// every early-return path; on success it moves into the adapter and fires
// when the table is dropped.
let guard = ContextGuard {
context,
release: Some(release),
};
take_result(move || {
let ctx = unsafe { &*require_ptr(handle, "SessionContext handle")? };
let schema_name = cstr(schema_name, "schema_name")?;
let table_name = cstr(table_name, "table_name")?;
register_default_schema(ctx, &schema_name)?;
register_callback_table(
ctx,
TableReference::partial(schema_name, table_name),
schema_ptr,
schema_len,
supports_pushdown,
callback,
guard,
)
})
}

fn register_callback_table(
ctx: &SessionContext,
table_ref: impl Into<TableReference>,
schema_ptr: *const u8,
schema_len: usize,
supports_pushdown: c_int,
callback: ScanCallback,
guard: ContextGuard,
) -> crate::NativeResult<()> {
let schema: SchemaRef = Arc::new(
decode_optional_schema(schema_ptr, schema_len)?
.ok_or("callback table requires a non-empty schema")?,
);
let table = ManagedCallbackTable {
schema,
callback,
context: Arc::new(guard),
supports_pushdown: supports_pushdown != 0,
};
ctx.register_table(table_ref, Arc::new(table))?;
Ok(())
}

fn register_default_schema(ctx: &SessionContext, schema_name: &str) -> crate::NativeResult<()> {
let state = ctx.state();
let default_catalog = state.config().options().catalog.default_catalog.as_str();
let catalog = ctx
.catalog(default_catalog)
.ok_or_else(|| format!("failed to resolve catalog: {default_catalog}"))?;
if catalog.schema(schema_name).is_none() {
catalog.register_schema(schema_name, Arc::new(MemorySchemaProvider::new()))?;
}
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public sealed class ClickHouseDatabaseOptions
{
public required string ConnectionString { get; init; }

public string? SourceName { get; init; }

public string? DatabaseName { get; init; }

public bool IncludeViews { get; init; } = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static void RegisterClickHouse(this SessionContext context, ClickHouseDat

foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews))
{
context.RegisterStreamingTable(table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions
RegisterSourceTable(context, options.SourceName, table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions
{
ConnectionString = options.ConnectionString,
DatabaseName = table.DatabaseName,
Expand Down Expand Up @@ -108,4 +108,15 @@ ORDER BY name
}

private sealed record ClickHouseTable(string DatabaseName, string TableName);

private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider)
{
if (string.IsNullOrWhiteSpace(sourceName))
{
context.RegisterStreamingTable(tableName, provider);
return;
}

context.RegisterStreamingTable(sourceName, tableName, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public sealed class MongoDbDatabaseOptions
{
public required string ConnectionString { get; init; }

public string? SourceName { get; init; }

public string? DatabaseName { get; init; }

public int BatchSize { get; init; } = 1024;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO
IMongoDatabase database = client.GetDatabase(databaseName);
foreach (string collectionName in database.ListCollectionNames().ToEnumerable())
{
context.RegisterStreamingTable(collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions
RegisterSourceTable(context, options.SourceName, collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions
{
ConnectionString = options.ConnectionString,
DatabaseName = databaseName,
Expand Down Expand Up @@ -113,4 +113,15 @@ private static string ResolveDatabaseName(string connectionString, string? datab

throw new ArgumentException("Database name must be provided in MongoDB options or connection string.", nameof(connectionString));
}

private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider)
{
if (string.IsNullOrWhiteSpace(sourceName))
{
context.RegisterStreamingTable(tableName, provider);
return;
}

context.RegisterStreamingTable(sourceName, tableName, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public sealed class MySqlDatabaseOptions
{
public required string ConnectionString { get; init; }

public string? SourceName { get; init; }

public string? DatabaseName { get; init; }

public bool IncludeViews { get; init; } = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptio

foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews))
{
context.RegisterStreamingTable(table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions
RegisterSourceTable(context, options.SourceName, table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions
{
ConnectionString = options.ConnectionString,
DatabaseName = table.DatabaseName,
Expand Down Expand Up @@ -100,4 +100,15 @@ ORDER BY table_name
}

private sealed record MySqlTable(string DatabaseName, string TableName);

private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider)
{
if (string.IsNullOrWhiteSpace(sourceName))
{
context.RegisterStreamingTable(tableName, provider);
return;
}

context.RegisterStreamingTable(sourceName, tableName, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public sealed class PostgreSqlDatabaseOptions
{
public required string ConnectionString { get; init; }

public string? SourceName { get; init; }

public IReadOnlyCollection<string> Schemas { get; init; } = ["public"];

public bool IncludeViews { get; init; } = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDat
foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews))
{
string registrationName = RegistrationName(table, schemas);
context.RegisterStreamingTable(registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions
RegisterSourceTable(context, options.SourceName, registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions
{
ConnectionString = options.ConnectionString,
SchemaName = table.SchemaName,
Expand Down Expand Up @@ -126,4 +126,15 @@ private static string RegistrationName(PostgreSqlTable table, IReadOnlyCollectio
: $"{table.SchemaName}_{table.TableName}";

private sealed record PostgreSqlTable(string SchemaName, string TableName);

private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider)
{
if (string.IsNullOrWhiteSpace(sourceName))
{
context.RegisterStreamingTable(tableName, provider);
return;
}

context.RegisterStreamingTable(sourceName, tableName, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOpt

foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews))
{
context.RegisterStreamingTable(tableName, new SqliteStreamingTableProvider(new SqliteTableOptions
RegisterSourceTable(context, options.SourceName, tableName, new SqliteStreamingTableProvider(new SqliteTableOptions
{
ConnectionString = options.ConnectionString,
TableName = tableName,
Expand Down Expand Up @@ -98,4 +98,15 @@ ORDER BY name
yield return reader.GetString(0);
}
}

private static void RegisterSourceTable(SessionContext context, string? sourceName, string tableName, StreamingTableProvider provider)
{
if (string.IsNullOrWhiteSpace(sourceName))
{
context.RegisterStreamingTable(tableName, provider);
return;
}

context.RegisterStreamingTable(sourceName, tableName, provider);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public sealed class SqliteDatabaseOptions
{
public required string ConnectionString { get; init; }

public string? SourceName { get; init; }

public bool IncludeViews { get; init; } = true;

public int BatchSize { get; init; } = 1024;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,16 @@ internal static extern int df_session_context_register_callback_table(
CallbackTableScan scan,
IntPtr context,
CallbackTableRelease release);

[DllImport(LibraryName, CallingConvention = CallingConvention.Cdecl)]
internal static extern int df_session_context_register_callback_table_in_schema(
IntPtr handle,
IntPtr schemaName,
IntPtr tableName,
IntPtr schemaPtr,
nuint schemaLen,
int supportsPushdown,
CallbackTableScan scan,
IntPtr context,
CallbackTableRelease release);
}
Loading
Loading