From 267a6921e7603538987d95920cca3f276f267c7d Mon Sep 17 00:00:00 2001 From: lqlive Date: Tue, 16 Jun 2026 16:20:14 +0800 Subject: [PATCH] Migrate ClickHouse to official driver; update README Switch ClickHouse provider and sample to ClickHouse.Driver (v1.2.0) from ClickHouse.Client. Update README to document new managed table providers (MySQL, MongoDB, ClickHouse), usage examples, and environment variable setup. Bump .NET SDK prerequisite to 10.0.300+. --- README.md | 56 ++++++++++++++++++- .../Program.cs | 2 +- ...ataFusion.TableProviders.ClickHouse.csproj | 2 +- .../ClickHouseArrowArrayStream.cs | 2 +- .../ClickHouseStreamingTableProvider.cs | 2 +- 5 files changed, 57 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6f7110a..5e57d54 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ and result sets cross as Apache Arrow IPC byte streams. | Path | Contents | | --- | --- | | `src/Apache.DataFusion/` | Managed SDK (public API + interop) | +| `src/Apache.DataFusion.TableProviders.*/` | Managed database table providers | | `native/` | Rust `cdylib` exposing the C ABI | | `proto/` | Protobuf contracts shared by C# and Rust | | `tests/Apache.DataFusion.Tests/` | xUnit test suite | @@ -30,8 +31,9 @@ and result sets cross as Apache Arrow IPC byte streams. ## Prerequisites -- **.NET SDK 9.0.200+** — required to build the `.slnx` solution format - (pinned to `9.0.301` in `global.json`). The libraries still target `net8.0`. +- **.NET SDK 10.0.300+** — required to build the `.slnx` solution format + (pinned to `10.0.300` in `global.json`). The core SDK targets `net8.0`; + database table-provider packages target `net10.0`. - **Rust toolchain** (stable) — to build the native `cdylib`. ## Install @@ -132,6 +134,9 @@ Each scenario is an independent, runnable console project under `examples/`: | `Apache.DataFusion.Sample.Streaming` | Stream Arrow batches and read typed columns | | `Apache.DataFusion.Sample.SessionConfig` | Configure the session builder + observability | | `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query | +| `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider | +| `Apache.DataFusion.Sample.MongoDB` | Query a MongoDB collection through a managed streaming provider | +| `Apache.DataFusion.Sample.ClickHouse` | Query a ClickHouse table through the official `ClickHouse.Driver` | Run any of them with: @@ -156,7 +161,8 @@ dotnet run --project examples/Apache.DataFusion.Sample.Sql - **Table providers**: `SimpleTableProvider` backed by Arrow IPC and a native `MemTable`; `StreamingTableProvider`, a lazy callback-based provider whose `Scan` is invoked by the engine and streamed in over the Arrow C Data Interface (connect any managed source, e.g. - ADO.NET, with no native database driver). + ADO.NET, with no native database driver). Managed provider packages currently include + MySQL, MongoDB, and ClickHouse. - **Scalar UDF**: zero-argument `Int64` managed callbacks. - **Typed exceptions**: `DataFusionException` (base), plus `DataFusionPlanException`, `DataFusionExecutionException`, `DataFusionIoException`, @@ -213,6 +219,50 @@ Console.WriteLine(df.Count()); may run on native worker threads and more than once over the table's lifetime; each call must return a new, independently consumable stream, which the engine disposes once read. +### Managed database providers + +Managed database provider packages are built on `StreamingTableProvider`. They keep database +drivers on the C# side and push simple projection, filter, and limit requests down to the +source query when possible. + +| Provider package | Extension method | Driver | +| --- | --- | --- | +| `Apache.DataFusion.TableProviders.MySql` | `RegisterMySql` | `MySqlConnector` | +| `Apache.DataFusion.TableProviders.MongoDB` | `RegisterMongoDb` | `MongoDB.Driver` | +| `Apache.DataFusion.TableProviders.ClickHouse` | `RegisterClickHouse` | official `ClickHouse.Driver` | + +```csharp +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.ClickHouse; + +using SessionContext context = new(); +context.RegisterClickHouse("audit_log", new ClickHouseTableOptions +{ + ConnectionString = "Host=localhost;Port=8123;Username=default;Password=;Database=default", + Query = """SELECT * FROM audit_log""", +}); + +using DataFrame df = context.Sql(""" + SELECT * + FROM audit_log + LIMIT 10 + """); +df.Show(); +``` + +The registered DataFusion table name is the first argument to the extension method. The +provider's `Query` points at the remote database table or subquery. For example, if you +register `audit_log`, query `FROM audit_log` in DataFusion SQL; if you register `orders`, +query `FROM orders`. + +The sample projects use environment variables for connection strings: + +| Sample | Environment variable | +| --- | --- | +| `Apache.DataFusion.Sample.MySql` | `DATAFUSION_MYSQL_CONNECTION` | +| `Apache.DataFusion.Sample.MongoDB` | `DATAFUSION_MONGODB_CONNECTION` | +| `Apache.DataFusion.Sample.ClickHouse` | `DATAFUSION_CLICKHOUSE_CONNECTION` | + ## Native packaging Native binaries are packaged through RID-specific projects under `packages/`, placing the diff --git a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs index 1fd9cab..b6a04c1 100644 --- a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs +++ b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs @@ -18,7 +18,7 @@ using Apache.DataFusion; using Apache.DataFusion.TableProviders.ClickHouse; -using ClickHouse.Client.ADO; +using ClickHouse.Driver.ADO; string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_CLICKHOUSE_CONNECTION"); if (string.IsNullOrWhiteSpace(connectionString)) diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj index 6ae6abb..8ec5485 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs index af90366..39c201b 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs @@ -22,7 +22,7 @@ using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; -using ClickHouse.Client.ADO; +using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index b2a44ef..0fbea39 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -22,7 +22,7 @@ using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; -using ClickHouse.Client.ADO; +using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse;