From 63760cd7cbde165d7a8950924bf96af2e8460616 Mon Sep 17 00:00:00 2001 From: lqlive Date: Wed, 17 Jun 2026 11:49:24 +0800 Subject: [PATCH 1/4] Refactor usings, adopt C# 12 collections, add .editorconfig Clean up and consolidate using directives across the codebase, removing unnecessary imports. Replace LINQ .ToArray() with C# 12 collection expressions for arrays and use collection expressions for MongoDB document initialization. Clarify namespace declarations for MySql and ClickHouse providers. Add an .editorconfig to enforce import organization and code style. Apply minor formatting improvements for readability. --- .editorconfig | 8 ++++++++ examples/Apache.DataFusion.Sample.MySql/Program.cs | 2 +- .../Apache.DataFusion.TableProviders.ClickHouse.csproj | 2 +- .../BooleanColumnAppender.cs | 1 - .../ClickHouseArrowArrayStream.cs | 1 - .../ClickHouseStreamingTableProvider.cs | 1 - .../ColumnAppender.cs | 1 - .../ColumnPlan.cs | 1 - .../DoubleColumnAppender.cs | 1 - .../FloatColumnAppender.cs | 1 - .../Int32ColumnAppender.cs | 1 - .../Int64ColumnAppender.cs | 1 - .../StringColumnAppender.cs | 1 - .../MongoDbArrowArrayStream.cs | 7 +++---- .../MongoDbColumnAppender.cs | 2 -- .../MongoDbColumnPlan.cs | 6 ++---- .../MongoDbQueryBuilder.cs | 1 - .../MongoDbStreamingTableProvider.cs | 6 ++---- .../BooleanColumnAppender.cs | 2 +- .../ColumnAppender.cs | 3 +-- src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs | 2 +- .../DoubleColumnAppender.cs | 2 +- .../FloatColumnAppender.cs | 2 +- .../Int32ColumnAppender.cs | 2 +- .../Int64ColumnAppender.cs | 2 +- .../MySqlArrowArrayStream.cs | 2 -- .../MySqlStreamingTableProvider.cs | 2 -- .../StreamingTableProviderTests.cs | 2 -- 28 files changed, 24 insertions(+), 41 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..151fa6f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*.{cs,vb}] +# Keep C# using directives in one block when the IDE organizes imports. +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false +dotnet_diagnostic.IDE0055.severity = warning +dotnet_diagnostic.IDE0005.severity = warning diff --git a/examples/Apache.DataFusion.Sample.MySql/Program.cs b/examples/Apache.DataFusion.Sample.MySql/Program.cs index 0a2ce31..1f0e311 100644 --- a/examples/Apache.DataFusion.Sample.MySql/Program.cs +++ b/examples/Apache.DataFusion.Sample.MySql/Program.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using MySqlConnector; using Apache.DataFusion; using Apache.DataFusion.TableProviders.MySql; +using MySqlConnector; string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_MYSQL_CONNECTION"); if (string.IsNullOrWhiteSpace(connectionString)) diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj index 8ec5485..35e0d23 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj @@ -1,4 +1,4 @@ - + net10.0 diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs index 6aa8010..3d6832d 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs index 39c201b..0d00f48 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs @@ -17,7 +17,6 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index 0fbea39..14e4577 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -17,7 +17,6 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs index 746d716..8ddaa2b 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs index 3c4e284..bc8d475 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; - using Apache.Arrow.Types; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs index 5769f28..550a549 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs index bc1f71c..a80f96e 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs index 58f2d32..c47ea6e 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs index 277f9be..a100bf2 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs index 7870b79..e2d3092 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs index 116d64f..a4938d2 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs @@ -17,7 +17,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; - using MongoDB.Bson; using MongoDB.Driver; @@ -73,7 +72,7 @@ public MongoDbArrowArrayStream( return new ValueTask((RecordBatch?)null); } - MongoDbColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + MongoDbColumnAppender[] appenders = [.. columns.Select(static column => column.CreateAppender())]; int rowCount = 0; while (rowCount < batchSize) { @@ -97,7 +96,7 @@ public MongoDbArrowArrayStream( return new ValueTask((RecordBatch?)null); } - IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + IArrowArray[] arrays = [.. appenders.Select(static appender => appender.Build())]; RecordBatch batch = new(schema, arrays, rowCount); return new ValueTask(batch); } @@ -123,7 +122,7 @@ private bool TryReadNextDocument(out BsonDocument document) currentBatch = null; if (!cursor.MoveNext()) { - document = new BsonDocument(); + document = []; return false; } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs index 74bc8a7..39ff62a 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs @@ -16,9 +16,7 @@ // under the License. using System.Globalization; - using Apache.Arrow; - using MongoDB.Bson; namespace Apache.DataFusion.TableProviders.MongoDB; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs index 974d8ff..195c886 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs @@ -16,7 +16,6 @@ // under the License. using Apache.Arrow.Types; - using MongoDB.Bson; namespace Apache.DataFusion.TableProviders.MongoDB; @@ -53,13 +52,12 @@ public static MongoDbColumnPlan[] Infer(IReadOnlyList documents) } } - return columns + return [.. columns .Select(column => new MongoDbColumnPlan( column.Key, GetArrowType(column.Value.Kind), column.Value.Nullable || column.Value.ObservedCount < documents.Count, - column.Value.Kind)) - .ToArray(); + column.Value.Kind))]; } private static IArrowType GetArrowType(MongoDbColumnKind kind) => kind switch diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs index 7111487..0d35989 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs @@ -16,7 +16,6 @@ // under the License. using System.Globalization; - using MongoDB.Bson; using MongoDB.Driver; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs index 9576237..2994a64 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs @@ -17,7 +17,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; - using MongoDB.Bson; using MongoDB.Driver; @@ -97,11 +96,10 @@ private MongoDbColumnPlan[] ProjectColumns(StreamingTableScanRequest request) Dictionary byName = columns.ToDictionary( static column => column.Name, StringComparer.Ordinal); - return request.Projection + return [.. request.Projection .Select(name => byName.TryGetValue(name, out MongoDbColumnPlan? column) ? column - : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'.")) - .ToArray(); + : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'."))]; } private static MongoDbColumnPlan[] InferColumns(IMongoCollection collection, int schemaInferenceLimit) diff --git a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs index 64ebca9..b66fa20 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs index 82f2bc8..114b66f 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; - +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs index 12fd3d7..e48a8e3 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow.Types; using System.Data.Common; +using Apache.Arrow.Types; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs index 87fdd11..2c63817 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs index 69d992e..5f23fc3 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs index 6c66f83..d3e4675 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs index d85c941..d2e8d29 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs index 5ccd36c..2ce32a1 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.MySql.Sql; - using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs index 72f11b3..cb05ec7 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.MySql.Sql; - using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs index 0a32745..2f7a639 100644 --- a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs +++ b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs @@ -19,8 +19,6 @@ using Apache.Arrow.Ipc; using Apache.Arrow.Types; -using Apache.DataFusion; - namespace Apache.DataFusion.Tests; public class StreamingTableProviderTests From 5d8f914e67bb55d900863314b12858367712e6ed Mon Sep 17 00:00:00 2001 From: lqlive Date: Wed, 17 Jun 2026 12:25:37 +0800 Subject: [PATCH 2/4] Add PostgreSQL streaming provider and unify DB APIs - Introduce managed streaming table provider for PostgreSQL with schema/table registration and pushdown support - Add database options classes for ClickHouse, MySQL, and MongoDB; enable schema-wide registration and configuration - Refactor MySQL and ClickHouse table options to use TableName/DatabaseName - Update extension methods and providers for consistent multi-table registration - Add PostgreSQL sample project and update solution/README for new features and usage patterns --- README.md | 43 ++++-- datafusion-csharp.slnx | 2 + .../Program.cs | 8 +- .../Program.cs | 3 +- .../Apache.DataFusion.Sample.MySql/Program.cs | 8 +- ...Apache.DataFusion.Sample.PostgreSql.csproj | 19 +++ .../Program.cs | 63 ++++++++ .../ClickHouseArrowArrayStream.cs | 1 - .../ClickHouseDatabaseOptions.cs | 29 ++++ .../ClickHouseStreamingTableProvider.cs | 34 +++-- .../ClickHouseTableOptions.cs | 4 +- .../SessionContextExtensions.cs | 77 +++++++++- .../MongoDbDatabaseOptions.cs | 29 ++++ .../SessionContextExtensions.cs | 72 ++++++++++ .../MySqlDatabaseOptions.cs | 29 ++++ .../MySqlStreamingTableProvider.cs | 33 +++-- .../MySqlTableOptions.cs | 4 +- .../SessionContextExtensions.cs | 69 ++++++++- ...ataFusion.TableProviders.PostgreSql.csproj | 17 +++ .../BooleanColumnAppender.cs | 40 ++++++ .../ColumnAppender.cs | 43 ++++++ .../ColumnKind.cs | 28 ++++ .../ColumnPlan.cs | 91 ++++++++++++ .../DoubleColumnAppender.cs | 40 ++++++ .../FloatColumnAppender.cs | 40 ++++++ .../Int32ColumnAppender.cs | 40 ++++++ .../Int64ColumnAppender.cs | 40 ++++++ .../PostgreSqlArrowArrayStream.cs | 119 +++++++++++++++ .../PostgreSqlDatabaseOptions.cs | 29 ++++ .../PostgreSqlDialect.cs | 32 +++++ .../PostgreSqlStreamingTableProvider.cs | 136 ++++++++++++++++++ .../PostgreSqlTableOptions.cs | 29 ++++ .../SessionContextExtensions.cs | 129 +++++++++++++++++ .../Sql/PushedQuery.cs | 20 +++ .../Sql/SqlDialect.cs | 36 +++++ .../Sql/SqlQueryBuilder.cs | 79 ++++++++++ .../Sql/SqlQueryParameter.cs | 20 +++ .../Sql/SqlSelectStatement.cs | 48 +++++++ .../StringColumnAppender.cs | 39 +++++ 39 files changed, 1566 insertions(+), 56 deletions(-) create mode 100644 examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj create mode 100644 examples/Apache.DataFusion.Sample.PostgreSql/Program.cs create mode 100644 src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs create mode 100644 src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs diff --git a/README.md b/README.md index 5e57d54..c33ba8a 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ Each scenario is an independent, runnable console project under `examples/`: | `Apache.DataFusion.Sample.SessionConfig` | Configure the session builder + observability | | `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query | | `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider | +| `Apache.DataFusion.Sample.PostgreSql` | Query a PostgreSQL table through a managed streaming provider | | `Apache.DataFusion.Sample.MongoDB` | Query a MongoDB collection through a managed streaming provider | | `Apache.DataFusion.Sample.ClickHouse` | Query a ClickHouse table through the official `ClickHouse.Driver` | @@ -162,7 +163,7 @@ dotnet run --project examples/Apache.DataFusion.Sample.Sql `StreamingTableProvider`, a lazy callback-based provider whose `Scan` is invoked by the engine and streamed in over the Arrow C Data Interface (connect any managed source, e.g. ADO.NET, with no native database driver). Managed provider packages currently include - MySQL, MongoDB, and ClickHouse. + MySQL, PostgreSQL, MongoDB, and ClickHouse. - **Scalar UDF**: zero-argument `Int64` managed callbacks. - **Typed exceptions**: `DataFusionException` (base), plus `DataFusionPlanException`, `DataFusionExecutionException`, `DataFusionIoException`, @@ -222,44 +223,58 @@ must return a new, independently consumable stream, which the engine disposes on ### Managed database providers Managed database provider packages are built on `StreamingTableProvider`. They keep database -drivers on the C# side and push simple projection, filter, and limit requests down to the -source query when possible. +drivers on the C# side, register discovered tables and collections in the current +`SessionContext`, and push simple projection, filter, and limit requests down when possible. | Provider package | Extension method | Driver | | --- | --- | --- | | `Apache.DataFusion.TableProviders.MySql` | `RegisterMySql` | `MySqlConnector` | +| `Apache.DataFusion.TableProviders.PostgreSql` | `RegisterPostgreSql` | `Npgsql` | | `Apache.DataFusion.TableProviders.MongoDB` | `RegisterMongoDb` | `MongoDB.Driver` | | `Apache.DataFusion.TableProviders.ClickHouse` | `RegisterClickHouse` | official `ClickHouse.Driver` | ```csharp using Apache.DataFusion; -using Apache.DataFusion.TableProviders.ClickHouse; +using Apache.DataFusion.TableProviders.PostgreSql; using SessionContext context = new(); -context.RegisterClickHouse("audit_log", new ClickHouseTableOptions -{ - ConnectionString = "Host=localhost;Port=8123;Username=default;Password=;Database=default", - Query = """SELECT * FROM audit_log""", -}); +context.RegisterPostgreSql("Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample"); + +using DataFrame df = context.Sql(""" + SELECT * + FROM datafusion_orders + LIMIT 10 + """); +df.Show(); +``` + +SQL database providers register tables and views from the selected database in one call: + +```csharp +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.MySql; + +using SessionContext context = new(); +context.RegisterMySql("Server=localhost;User ID=root;Password=pass;Database=datafusion_sample"); using DataFrame df = context.Sql(""" SELECT * - FROM audit_log + FROM datafusion_orders LIMIT 10 """); df.Show(); ``` -The registered DataFusion table name is the first argument to the extension method. The -provider's `Query` points at the remote database table or subquery. For example, if you -register `audit_log`, query `FROM audit_log` in DataFusion SQL; if you register `orders`, -query `FROM orders`. +Database registration uses each discovered table or collection name as the DataFusion table +name by default. Single-table registration overloads are also available when you only want +to expose one table. The sample projects use environment variables for connection strings: | Sample | Environment variable | | --- | --- | | `Apache.DataFusion.Sample.MySql` | `DATAFUSION_MYSQL_CONNECTION` | +| `Apache.DataFusion.Sample.PostgreSql` | `DATAFUSION_POSTGRESQL_CONNECTION` | | `Apache.DataFusion.Sample.MongoDB` | `DATAFUSION_MONGODB_CONNECTION` | | `Apache.DataFusion.Sample.ClickHouse` | `DATAFUSION_CLICKHOUSE_CONNECTION` | diff --git a/datafusion-csharp.slnx b/datafusion-csharp.slnx index 8a5a694..2d73a38 100644 --- a/datafusion-csharp.slnx +++ b/datafusion-csharp.slnx @@ -3,12 +3,14 @@ + + diff --git a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs index b6a04c1..02e346b 100644 --- a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs +++ b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs @@ -52,16 +52,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES } using SessionContext context = new(); -context.RegisterClickHouse("orders", new ClickHouseTableOptions -{ - ConnectionString = connectionString, - Query = """SELECT id, customer, total FROM datafusion_orders""", -}); +context.RegisterClickHouse(connectionString); Console.WriteLine("Customer spend from a ClickHouse-backed streaming table:"); using DataFrame df = context.Sql(""" SELECT customer, SUM(total) AS spend - FROM orders + FROM datafusion_orders WHERE total >= 10 GROUP BY customer ORDER BY customer diff --git a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs index 77e2d23..57d434a 100644 --- a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs +++ b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs @@ -68,11 +68,10 @@ ]); using SessionContext context = new(); -context.RegisterMongoDb("orders", new MongoDbTableOptions +context.RegisterMongoDb(new MongoDbDatabaseOptions { ConnectionString = connectionString, DatabaseName = databaseName, - CollectionName = collectionName, }); Console.WriteLine("Customer spend from a MongoDB-backed streaming table:"); diff --git a/examples/Apache.DataFusion.Sample.MySql/Program.cs b/examples/Apache.DataFusion.Sample.MySql/Program.cs index 1f0e311..0a0dfd4 100644 --- a/examples/Apache.DataFusion.Sample.MySql/Program.cs +++ b/examples/Apache.DataFusion.Sample.MySql/Program.cs @@ -51,16 +51,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES } using SessionContext context = new(); -context.RegisterMySql("orders", new MySqlTableOptions -{ - ConnectionString = connectionString, - Query = """SELECT id, customer, total FROM datafusion_orders""", -}); +context.RegisterMySql(connectionString); Console.WriteLine("Customer spend from a MySQL-backed streaming table:"); using DataFrame df = context.Sql(""" SELECT customer, SUM(total) AS spend - FROM orders + FROM datafusion_orders GROUP BY customer ORDER BY customer """); diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj new file mode 100644 index 0000000..1b56c28 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs new file mode 100644 index 0000000..1441ca8 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.PostgreSql; +using Npgsql; + +string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_POSTGRESQL_CONNECTION"); +if (string.IsNullOrWhiteSpace(connectionString)) +{ + Console.WriteLine("Set DATAFUSION_POSTGRESQL_CONNECTION to run this sample."); + Console.WriteLine("Example:"); + Console.WriteLine(" Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample"); + return; +} + +using (NpgsqlConnection connection = new(connectionString)) +{ + connection.Open(); + using NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer VARCHAR(64) NOT NULL, + total DOUBLE PRECISION NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); +} + +using SessionContext context = new(); +context.RegisterPostgreSql(connectionString); + +Console.WriteLine("Customer spend from a PostgreSQL-backed streaming table:"); +using DataFrame df = context.Sql(""" + SELECT customer, SUM(total) AS spend + FROM datafusion_orders + GROUP BY customer + ORDER BY customer + """); +df.Show(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs index 0d00f48..8750e39 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs @@ -20,7 +20,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; - using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs new file mode 100644 index 0000000..4440322 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.ClickHouse; + +public sealed class ClickHouseDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index 14e4577..ca7fca7 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -20,7 +20,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; - using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -28,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.ClickHouse; public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider { private readonly string connectionString; - private readonly string query; + private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; private readonly ClickHouseQueryBuilder queryBuilder = new(ClickHouseDialect.Instance); @@ -41,20 +40,15 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); } - if (string.IsNullOrWhiteSpace(options.Query)) - { - throw new ArgumentException("Query cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } connectionString = options.ConnectionString; - query = options.Query; + sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, query); + columns = FetchColumns(connectionString, sourceSql); Schema = BuildSchema(columns); } @@ -63,13 +57,13 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new ClickHouseArrowArrayStream(connectionString, query, Schema, columns, batchSize); + new ClickHouseArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { ArgumentNullException.ThrowIfNull(request); - PushedQuery pushedQuery = queryBuilder.Build(query, request); + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new ClickHouseArrowArrayStream( @@ -81,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static string BuildSourceSql(ClickHouseTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string? databaseName, string tableName) + { + string quotedTable = ClickHouseDialect.Instance.QuoteIdentifier(tableName); + return string.IsNullOrWhiteSpace(databaseName) + ? quotedTable + : $"{ClickHouseDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; + } + private static ColumnPlan[] FetchColumns(string connectionString, string query) { using ClickHouseConnection connection = new(connectionString); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs index 2114ded..c1471ab 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs @@ -21,7 +21,9 @@ public sealed class ClickHouseTableOptions { public required string ConnectionString { get; init; } - public required string Query { get; init; } + public string? DatabaseName { get; init; } + + public required string TableName { get; init; } public int BatchSize { get; init; } = 1024; } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs index 1afe8ce..740e39b 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs @@ -15,10 +15,47 @@ // specific language governing permissions and limitations // under the License. +using System.Data.Common; +using ClickHouse.Driver.ADO; + namespace Apache.DataFusion.TableProviders.ClickHouse; public static class SessionContextExtensions { + public static void RegisterClickHouse(this SessionContext context, string connectionString) + { + context.RegisterClickHouse(new ClickHouseDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterClickHouse(this SessionContext context, ClickHouseDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + { + context.RegisterStreamingTable(table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = table.DatabaseName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + public static void RegisterClickHouse(this SessionContext context, string name, ClickHouseTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,12 +64,48 @@ public static void RegisterClickHouse(this SessionContext context, string name, context.RegisterStreamingTable(name, new ClickHouseStreamingTableProvider(options)); } - public static void RegisterClickHouse(this SessionContext context, string name, string connectionString, string query) + public static void RegisterClickHouse(this SessionContext context, string name, string connectionString) { context.RegisterClickHouse(name, new ClickHouseTableOptions { ConnectionString = connectionString, - Query = query, + TableName = name, }); } + + private static IEnumerable FetchTables( + string connectionString, + string? databaseName, + bool includeViews) + { + using ClickHouseConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT database, name + FROM system.tables + WHERE database = if({database:String} = '', currentDatabase(), {database:String}) + AND ({include_views:Bool} OR engine NOT IN ('View', 'MaterializedView')) + ORDER BY name + """; + + DbParameter databaseParameter = command.CreateParameter(); + databaseParameter.ParameterName = "database"; + databaseParameter.Value = databaseName ?? string.Empty; + command.Parameters.Add(databaseParameter); + + DbParameter includeViewsParameter = command.CreateParameter(); + includeViewsParameter.ParameterName = "include_views"; + includeViewsParameter.Value = includeViews; + command.Parameters.Add(includeViewsParameter); + + using DbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return new ClickHouseTable(reader.GetString(0), reader.GetString(1)); + } + } + + private sealed record ClickHouseTable(string DatabaseName, string TableName); } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs new file mode 100644 index 0000000..64250bf --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.MongoDB; + +public sealed class MongoDbDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public int BatchSize { get; init; } = 1024; + + public int SchemaInferenceLimit { get; init; } = 100; +} diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs index f023293..643894f 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs @@ -15,10 +15,55 @@ // specific language governing permissions and limitations // under the License. +using MongoDB.Driver; + namespace Apache.DataFusion.TableProviders.MongoDB; public static class SessionContextExtensions { + public static void RegisterMongoDb(this SessionContext context, string connectionString) + { + context.RegisterMongoDb(new MongoDbDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + if (options.SchemaInferenceLimit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero."); + } + + string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName); + MongoClient client = new(options.ConnectionString); + IMongoDatabase database = client.GetDatabase(databaseName); + foreach (string collectionName in database.ListCollectionNames().ToEnumerable()) + { + context.RegisterStreamingTable(collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = databaseName, + CollectionName = collectionName, + BatchSize = options.BatchSize, + SchemaInferenceLimit = options.SchemaInferenceLimit, + })); + } + } + public static void RegisterMongoDb(this SessionContext context, string name, MongoDbTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,6 +72,17 @@ public static void RegisterMongoDb(this SessionContext context, string name, Mon context.RegisterStreamingTable(name, new MongoDbStreamingTableProvider(options)); } + public static void RegisterMongoDb(this SessionContext context, string name, string connectionString) + { + string databaseName = ResolveDatabaseName(connectionString, null); + context.RegisterMongoDb(name, new MongoDbTableOptions + { + ConnectionString = connectionString, + DatabaseName = databaseName, + CollectionName = name, + }); + } + public static void RegisterMongoDb( this SessionContext context, string name, @@ -41,4 +97,20 @@ public static void RegisterMongoDb( CollectionName = collectionName, }); } + + private static string ResolveDatabaseName(string connectionString, string? databaseName) + { + if (!string.IsNullOrWhiteSpace(databaseName)) + { + return databaseName; + } + + string? connectionDatabaseName = MongoUrl.Create(connectionString).DatabaseName; + if (!string.IsNullOrWhiteSpace(connectionDatabaseName)) + { + return connectionDatabaseName; + } + + throw new ArgumentException("Database name must be provided in MongoDB options or connection string.", nameof(connectionString)); + } } diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs new file mode 100644 index 0000000..6ff05bb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.MySql; + +public sealed class MySqlDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs index cb05ec7..2724468 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs @@ -27,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.MySql; public sealed class MySqlStreamingTableProvider : StreamingTableProvider { private readonly string connectionString; - private readonly string query; + private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; private readonly SqlQueryBuilder queryBuilder = new(MySqlDialect.Instance); @@ -40,20 +40,15 @@ public MySqlStreamingTableProvider(MySqlTableOptions options) throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); } - if (string.IsNullOrWhiteSpace(options.Query)) - { - throw new ArgumentException("Query cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } connectionString = options.ConnectionString; - query = options.Query; + sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, query); + columns = FetchColumns(connectionString, sourceSql); Schema = BuildSchema(columns); } @@ -62,13 +57,13 @@ public MySqlStreamingTableProvider(MySqlTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new MySqlArrowArrayStream(connectionString, query, Schema, columns, batchSize); + new MySqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { ArgumentNullException.ThrowIfNull(request); - PushedQuery pushedQuery = queryBuilder.Build(query, request); + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new MySqlArrowArrayStream( @@ -80,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static string BuildSourceSql(MySqlTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string? databaseName, string tableName) + { + string quotedTable = MySqlDialect.Instance.QuoteIdentifier(tableName); + return string.IsNullOrWhiteSpace(databaseName) + ? quotedTable + : $"{MySqlDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; + } + private static ColumnPlan[] FetchColumns(string connectionString, string query) { using MySqlConnection connection = new(connectionString); diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs index 48e4e9b..c91e4ac 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs @@ -21,7 +21,9 @@ public sealed class MySqlTableOptions { public required string ConnectionString { get; init; } - public required string Query { get; init; } + public string? DatabaseName { get; init; } + + public required string TableName { get; init; } public int BatchSize { get; init; } = 1024; } diff --git a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs index 9f4694d..f7e87de 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs @@ -15,10 +15,46 @@ // specific language governing permissions and limitations // under the License. +using MySqlConnector; + namespace Apache.DataFusion.TableProviders.MySql; public static class SessionContextExtensions { + public static void RegisterMySql(this SessionContext context, string connectionString) + { + context.RegisterMySql(new MySqlDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + { + context.RegisterStreamingTable(table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = table.DatabaseName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + public static void RegisterMySql(this SessionContext context, string name, MySqlTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,12 +63,41 @@ public static void RegisterMySql(this SessionContext context, string name, MySql context.RegisterStreamingTable(name, new MySqlStreamingTableProvider(options)); } - public static void RegisterMySql(this SessionContext context, string name, string connectionString, string query) + public static void RegisterMySql(this SessionContext context, string name, string connectionString) { context.RegisterMySql(name, new MySqlTableOptions { ConnectionString = connectionString, - Query = query, + TableName = name, }); } + + private static IEnumerable FetchTables( + string connectionString, + string? databaseName, + bool includeViews) + { + using MySqlConnection connection = new(connectionString); + connection.Open(); + + using MySqlCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = COALESCE(@database_name, DATABASE()) + AND table_type IN ('BASE TABLE', 'VIEW') + AND (@include_views OR table_type = 'BASE TABLE') + ORDER BY table_name + """; + command.Parameters.AddWithValue("@database_name", string.IsNullOrWhiteSpace(databaseName) ? DBNull.Value : databaseName); + command.Parameters.AddWithValue("@include_views", includeViews); + + using MySqlDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return new MySqlTable(reader.GetString(0), reader.GetString(1)); + } + } + + private sealed record MySqlTable(string DatabaseName, string TableName); } diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj new file mode 100644 index 0000000..6874099 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs new file mode 100644 index 0000000..c7bc593 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly BooleanArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs new file mode 100644 index 0000000..bba9402 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal abstract class ColumnAppender(int ordinal) +{ + protected int Ordinal { get; } = ordinal; + + public abstract void Append(DbDataReader reader); + + public abstract IArrowArray Build(); + + protected static string FormatValue(object value) => + value switch + { + byte[] bytes => Convert.ToBase64String(bytes), + DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture), + DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture), + TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture), + DateTimeOffset dateTimeOffset => dateTimeOffset.ToString("O", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs new file mode 100644 index 0000000..492f6eb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal enum ColumnKind +{ + Boolean, + Int32, + Int64, + Float, + Double, + String, +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs new file mode 100644 index 0000000..90fadc0 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow.Types; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed record ColumnPlan( + int Ordinal, + string Name, + IArrowType DataType, + bool Nullable, + ColumnKind Kind) +{ + public static ColumnPlan From(DbColumn column, int ordinal) + { + Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType; + ColumnKind kind = GetKind(type); + return new ColumnPlan( + ordinal, + column.ColumnName ?? $"column_{ordinal}", + GetArrowType(kind), + column.AllowDBNull ?? true, + kind); + } + + public ColumnAppender CreateAppender() => Kind switch + { + ColumnKind.Boolean => new BooleanColumnAppender(Ordinal), + ColumnKind.Int32 => new Int32ColumnAppender(Ordinal), + ColumnKind.Int64 => new Int64ColumnAppender(Ordinal), + ColumnKind.Float => new FloatColumnAppender(Ordinal), + ColumnKind.Double => new DoubleColumnAppender(Ordinal), + _ => new StringColumnAppender(Ordinal), + }; + + private static ColumnKind GetKind(Type? type) + { + if (type == typeof(bool)) + { + return ColumnKind.Boolean; + } + + if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int)) + { + return ColumnKind.Int32; + } + + if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong)) + { + return ColumnKind.Int64; + } + + if (type == typeof(float)) + { + return ColumnKind.Float; + } + + if (type == typeof(double) || type == typeof(decimal)) + { + return ColumnKind.Double; + } + + return ColumnKind.String; + } + + private static IArrowType GetArrowType(ColumnKind kind) => kind switch + { + ColumnKind.Boolean => BooleanType.Default, + ColumnKind.Int32 => Int32Type.Default, + ColumnKind.Int64 => Int64Type.Default, + ColumnKind.Float => FloatType.Default, + ColumnKind.Double => DoubleType.Default, + _ => StringType.Default, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs new file mode 100644 index 0000000..4366532 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly DoubleArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs new file mode 100644 index 0000000..9ed65ce --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly FloatArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs new file mode 100644 index 0000000..de9bad3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int32Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs new file mode 100644 index 0000000..2693520 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int64Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs new file mode 100644 index 0000000..0142ccc --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.PostgreSql.Sql; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class PostgreSqlArrowArrayStream : IArrowArrayStream +{ + private readonly Schema schema; + private readonly ColumnPlan[] columns; + private readonly int batchSize; + private NpgsqlConnection? connection; + private DbCommand? command; + private DbDataReader? reader; + private bool finished; + + public PostgreSqlArrowArrayStream( + string connectionString, + string query, + Schema schema, + ColumnPlan[] columns, + int batchSize, + IReadOnlyList? parameters = null) + { + this.schema = schema; + this.columns = columns; + this.batchSize = batchSize; + + try + { + connection = new NpgsqlConnection(connectionString); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + } + catch + { + Dispose(); + throw; + } + } + + public Schema Schema => schema; + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (finished || reader is null) + { + return new ValueTask((RecordBatch?)null); + } + + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + int rowCount = 0; + while (rowCount < batchSize && reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (ColumnAppender appender in appenders) + { + appender.Append(reader); + } + + rowCount++; + } + + if (rowCount == 0) + { + finished = true; + return new ValueTask((RecordBatch?)null); + } + + IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + RecordBatch batch = new(schema, arrays, rowCount); + return new ValueTask(batch); + } + + public void Dispose() + { + reader?.Dispose(); + command?.Dispose(); + connection?.Dispose(); + reader = null; + command = null; + connection = null; + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs new file mode 100644 index 0000000..aef7378 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public IReadOnlyCollection Schemas { get; init; } = ["public"]; + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs new file mode 100644 index 0000000..9aa837e --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion.TableProviders.PostgreSql.Sql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class PostgreSqlDialect : SqlDialect +{ + public static PostgreSqlDialect Instance { get; } = new(); + + private PostgreSqlDialect() + { + } + + public override string QuoteIdentifier(string identifier) => + $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs new file mode 100644 index 0000000..8e0c5a0 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.PostgreSql.Sql; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlStreamingTableProvider : StreamingTableProvider +{ + private readonly string connectionString; + private readonly string sourceSql; + private readonly int batchSize; + private readonly ColumnPlan[] columns; + private readonly SqlQueryBuilder queryBuilder = new(PostgreSqlDialect.Instance); + + public PostgreSqlStreamingTableProvider(PostgreSqlTableOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + connectionString = options.ConnectionString; + sourceSql = BuildSourceSql(options); + batchSize = options.BatchSize; + columns = FetchColumns(connectionString, sourceSql); + Schema = BuildSchema(columns); + } + + public override Schema Schema { get; } + + public override bool SupportsPushdown => true; + + public override IArrowArrayStream Scan() => + new PostgreSqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + + public override IArrowArrayStream Scan(StreamingTableScanRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); + ColumnPlan[] projectedColumns = ProjectColumns(request); + Schema projectedSchema = BuildSchema(projectedColumns); + return new PostgreSqlArrowArrayStream( + connectionString, + pushedQuery.Sql, + projectedSchema, + projectedColumns, + batchSize, + pushedQuery.Parameters); + } + + private static string BuildSourceSql(PostgreSqlTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + string schemaName = string.IsNullOrWhiteSpace(options.SchemaName) ? "public" : options.SchemaName; + return $"SELECT * FROM {QuoteQualifiedTableName(schemaName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string schemaName, string tableName) => + $"{PostgreSqlDialect.Instance.QuoteIdentifier(schemaName)}.{PostgreSqlDialect.Instance.QuoteIdentifier(tableName)}"; + + private static ColumnPlan[] FetchColumns(string connectionString, string query) + { + using NpgsqlConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = query; + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo); + return reader.GetColumnSchema() + .Select((column, ordinal) => ColumnPlan.From(column, ordinal)) + .ToArray(); + } + + private static Schema BuildSchema(IEnumerable columns) + { + Schema.Builder builder = new(); + foreach (ColumnPlan column in columns) + { + builder.Field(field => field + .Name(column.Name) + .DataType(column.DataType) + .Nullable(column.Nullable)); + } + + return builder.Build(); + } + + private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return columns; + } + + Dictionary byName = columns.ToDictionary( + static column => column.Name, + StringComparer.OrdinalIgnoreCase); + return request.Projection + .Select(name => byName.TryGetValue(name, out ColumnPlan? column) + ? column + : throw new InvalidOperationException($"PostgreSQL table provider does not contain column '{name}'.")) + .ToArray(); + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs new file mode 100644 index 0000000..15c82c8 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlTableOptions +{ + public required string ConnectionString { get; init; } + + public string SchemaName { get; init; } = "public"; + + public required string TableName { get; init; } + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs new file mode 100644 index 0000000..6843e0d --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public static class SessionContextExtensions +{ + public static void RegisterPostgreSql(this SessionContext context, string connectionString) + { + context.RegisterPostgreSql(new PostgreSqlDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + string[] schemas = options.Schemas + .Where(static schema => !string.IsNullOrWhiteSpace(schema)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (schemas.Length == 0) + { + throw new ArgumentException("At least one schema must be provided.", nameof(options)); + } + + foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews)) + { + string registrationName = RegistrationName(table, schemas); + context.RegisterStreamingTable(registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions + { + ConnectionString = options.ConnectionString, + SchemaName = table.SchemaName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + + public static void RegisterPostgreSql(this SessionContext context, string name, PostgreSqlTableOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + + context.RegisterStreamingTable(name, new PostgreSqlStreamingTableProvider(options)); + } + + public static void RegisterPostgreSql(this SessionContext context, string name, string connectionString) + { + context.RegisterPostgreSql(name, new PostgreSqlTableOptions + { + ConnectionString = connectionString, + TableName = name, + }); + } + + private static IEnumerable FetchTables( + string connectionString, + IReadOnlyCollection schemas, + bool includeViews) + { + using NpgsqlConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = ANY(@schemas) + AND table_type = ANY(@table_types) + ORDER BY table_schema, table_name + """; + + DbParameter schemasParameter = command.CreateParameter(); + schemasParameter.ParameterName = "schemas"; + schemasParameter.Value = schemas.ToArray(); + command.Parameters.Add(schemasParameter); + + DbParameter tableTypesParameter = command.CreateParameter(); + tableTypesParameter.ParameterName = "table_types"; + tableTypesParameter.Value = includeViews + ? new[] { "BASE TABLE", "VIEW" } + : new[] { "BASE TABLE" }; + command.Parameters.Add(tableTypesParameter); + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + while (reader.Read()) + { + yield return new PostgreSqlTable(reader.GetString(0), reader.GetString(1)); + } + } + + private static string RegistrationName(PostgreSqlTable table, IReadOnlyCollection schemas) => + schemas.Count == 1 && string.Equals(table.SchemaName, "public", StringComparison.OrdinalIgnoreCase) + ? table.TableName + : $"{table.SchemaName}_{table.TableName}"; + + private sealed record PostgreSqlTable(string SchemaName, string TableName); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs new file mode 100644 index 0000000..0460058 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs new file mode 100644 index 0000000..b38e11b --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal abstract class SqlDialect +{ + public abstract string QuoteIdentifier(string identifier); + + public virtual string ParameterName(int index) => $"@df_filter_{index}"; + + public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch + { + StreamingTableFilterOperator.Equal => "=", + StreamingTableFilterOperator.NotEqual => "<>", + StreamingTableFilterOperator.LessThan => "<", + StreamingTableFilterOperator.LessThanOrEqual => "<=", + StreamingTableFilterOperator.GreaterThan => ">", + StreamingTableFilterOperator.GreaterThanOrEqual => ">=", + _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."), + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs new file mode 100644 index 0000000..36259fa --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed class SqlQueryBuilder(SqlDialect dialect) +{ + public PushedQuery Build(string baseQuery, StreamingTableScanRequest request) + { + ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery); + ArgumentNullException.ThrowIfNull(request); + + List parameters = []; + SqlSelectStatement statement = new( + BuildProjection(request), + TrimTrailingSemicolon(baseQuery), + BuildPredicates(request, parameters), + request.Limit); + + return new PushedQuery(statement.ToSql(), parameters); + } + + private string BuildProjection(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return "*"; + } + + if (request.Projection.Count == 0) + { + return "1"; + } + + return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier)); + } + + private List BuildPredicates( + StreamingTableScanRequest request, + List parameters) + { + List predicates = []; + for (int i = 0; i < request.Filters.Count; i++) + { + StreamingTableFilter filter = request.Filters[i]; + string parameterName = dialect.ParameterName(i); + predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}"); + parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter))); + } + + return predicates; + } + + private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch + { + StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value), + StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.String => filter.Value, + _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."), + }; + + private static string TrimTrailingSemicolon(string sql) => + sql.Trim().TrimEnd(';'); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs new file mode 100644 index 0000000..b172ef3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record SqlQueryParameter(string Name, object Value); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs new file mode 100644 index 0000000..53c8111 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Globalization; + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record SqlSelectStatement( + string Projection, + string Source, + IReadOnlyList Predicates, + ulong? Limit) +{ + public string ToSql() + { + List clauses = + [ + $"SELECT {Projection}", + $"FROM ({Source}) AS df_source", + ]; + + if (Predicates.Count > 0) + { + clauses.Add($"WHERE {string.Join(" AND ", Predicates)}"); + } + + if (Limit.HasValue) + { + clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + return string.Join(" ", clauses); + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs new file mode 100644 index 0000000..b8cdf90 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly StringArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(FormatValue(reader.GetValue(Ordinal))); + } + + public override IArrowArray Build() => builder.Build(); +} From 7e140b0af9fdc1985f5f4949cedb281aaf5b8e3b Mon Sep 17 00:00:00 2001 From: lqlive Date: Wed, 17 Jun 2026 13:34:16 +0800 Subject: [PATCH 3/4] Add managed streaming table provider for SQLite Introduced Apache.DataFusion.TableProviders.Sqlite with full Arrow batch streaming support, schema discovery, and SQL pushdown. Added a sample project demonstrating SQLite integration. Updated solution and documentation to reflect new provider and usage. All code is properly namespaced and licensed. --- README.md | 22 ++- datafusion-csharp.slnx | 2 + .../Apache.DataFusion.Sample.Sqlite.csproj | 15 ++ .../Program.cs | 60 ++++++++ ...he.DataFusion.TableProviders.Sqlite.csproj | 17 +++ .../BooleanColumnAppender.cs | 40 ++++++ .../ColumnAppender.cs | 42 ++++++ .../ColumnKind.cs | 28 ++++ .../ColumnPlan.cs | 91 ++++++++++++ .../DoubleColumnAppender.cs | 40 ++++++ .../FloatColumnAppender.cs | 40 ++++++ .../Int32ColumnAppender.cs | 40 ++++++ .../Int64ColumnAppender.cs | 40 ++++++ .../SessionContextExtensions.cs | 101 ++++++++++++++ .../Sql/PushedQuery.cs | 20 +++ .../Sql/SqlDialect.cs | 36 +++++ .../Sql/SqlQueryBuilder.cs | 79 +++++++++++ .../Sql/SqlQueryParameter.cs | 20 +++ .../Sql/SqlSelectStatement.cs | 48 +++++++ .../SqliteArrowArrayStream.cs | 119 ++++++++++++++++ .../SqliteDatabaseOptions.cs | 27 ++++ .../SqliteDialect.cs | 32 +++++ .../SqliteStreamingTableProvider.cs | 132 ++++++++++++++++++ .../SqliteTableOptions.cs | 27 ++++ .../StringColumnAppender.cs | 39 ++++++ 25 files changed, 1156 insertions(+), 1 deletion(-) create mode 100644 examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj create mode 100644 examples/Apache.DataFusion.Sample.Sqlite/Program.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs create mode 100644 src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs diff --git a/README.md b/README.md index c33ba8a..377bd81 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ Each scenario is an independent, runnable console project under `examples/`: | `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query | | `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider | | `Apache.DataFusion.Sample.PostgreSql` | Query a PostgreSQL table through a managed streaming provider | +| `Apache.DataFusion.Sample.Sqlite` | Query a SQLite table through a managed streaming provider | | `Apache.DataFusion.Sample.MongoDB` | Query a MongoDB collection through a managed streaming provider | | `Apache.DataFusion.Sample.ClickHouse` | Query a ClickHouse table through the official `ClickHouse.Driver` | @@ -163,7 +164,7 @@ dotnet run --project examples/Apache.DataFusion.Sample.Sql `StreamingTableProvider`, a lazy callback-based provider whose `Scan` is invoked by the engine and streamed in over the Arrow C Data Interface (connect any managed source, e.g. ADO.NET, with no native database driver). Managed provider packages currently include - MySQL, PostgreSQL, MongoDB, and ClickHouse. + MySQL, PostgreSQL, SQLite, MongoDB, and ClickHouse. - **Scalar UDF**: zero-argument `Int64` managed callbacks. - **Typed exceptions**: `DataFusionException` (base), plus `DataFusionPlanException`, `DataFusionExecutionException`, `DataFusionIoException`, @@ -230,6 +231,7 @@ drivers on the C# side, register discovered tables and collections in the curren | --- | --- | --- | | `Apache.DataFusion.TableProviders.MySql` | `RegisterMySql` | `MySqlConnector` | | `Apache.DataFusion.TableProviders.PostgreSql` | `RegisterPostgreSql` | `Npgsql` | +| `Apache.DataFusion.TableProviders.Sqlite` | `RegisterSqlite` | `Microsoft.Data.Sqlite` | | `Apache.DataFusion.TableProviders.MongoDB` | `RegisterMongoDb` | `MongoDB.Driver` | | `Apache.DataFusion.TableProviders.ClickHouse` | `RegisterClickHouse` | official `ClickHouse.Driver` | @@ -265,6 +267,23 @@ using DataFrame df = context.Sql(""" df.Show(); ``` +SQLite uses the same registration shape for a local database file: + +```csharp +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; + +using SessionContext context = new(); +context.RegisterSqlite("Data Source=datafusion_sample.sqlite"); + +using DataFrame df = context.Sql(""" + SELECT * + FROM datafusion_orders + LIMIT 10 + """); +df.Show(); +``` + Database registration uses each discovered table or collection name as the DataFusion table name by default. Single-table registration overloads are also available when you only want to expose one table. @@ -275,6 +294,7 @@ The sample projects use environment variables for connection strings: | --- | --- | | `Apache.DataFusion.Sample.MySql` | `DATAFUSION_MYSQL_CONNECTION` | | `Apache.DataFusion.Sample.PostgreSql` | `DATAFUSION_POSTGRESQL_CONNECTION` | +| `Apache.DataFusion.Sample.Sqlite` | `DATAFUSION_SQLITE_CONNECTION` | | `Apache.DataFusion.Sample.MongoDB` | `DATAFUSION_MONGODB_CONNECTION` | | `Apache.DataFusion.Sample.ClickHouse` | `DATAFUSION_CLICKHOUSE_CONNECTION` | diff --git a/datafusion-csharp.slnx b/datafusion-csharp.slnx index 2d73a38..5b5e8d6 100644 --- a/datafusion-csharp.slnx +++ b/datafusion-csharp.slnx @@ -4,6 +4,7 @@ + @@ -11,6 +12,7 @@ + diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj new file mode 100644 index 0000000..0df4e4b --- /dev/null +++ b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Program.cs b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs new file mode 100644 index 0000000..3a7b5cf --- /dev/null +++ b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; +using Microsoft.Data.Sqlite; + +string connectionString = Environment.GetEnvironmentVariable("DATAFUSION_SQLITE_CONNECTION") + ?? new SqliteConnectionStringBuilder + { + DataSource = Path.Combine(Path.GetTempPath(), "datafusion_sample.sqlite"), + }.ToString(); + +using (SqliteConnection connection = new(connectionString)) +{ + connection.Open(); + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer TEXT NOT NULL, + total REAL NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); +} + +using SessionContext context = new(); +context.RegisterSqlite(connectionString); + +Console.WriteLine("Customer spend from a SQLite-backed streaming table:"); +using DataFrame df = context.Sql(""" + SELECT customer, SUM(total) AS spend + FROM datafusion_orders + GROUP BY customer + ORDER BY customer + """); +df.Show(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj new file mode 100644 index 0000000..2e3fc81 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs new file mode 100644 index 0000000..4f290ec --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly BooleanArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs new file mode 100644 index 0000000..9cdc462 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal abstract class ColumnAppender(int ordinal) +{ + protected int Ordinal { get; } = ordinal; + + public abstract void Append(DbDataReader reader); + + public abstract IArrowArray Build(); + + protected static string FormatValue(object value) => + value switch + { + byte[] bytes => Convert.ToBase64String(bytes), + DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture), + DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture), + TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs new file mode 100644 index 0000000..d9e3876 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal enum ColumnKind +{ + Boolean, + Int32, + Int64, + Float, + Double, + String, +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs new file mode 100644 index 0000000..48a411a --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow.Types; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed record ColumnPlan( + int Ordinal, + string Name, + IArrowType DataType, + bool Nullable, + ColumnKind Kind) +{ + public static ColumnPlan From(DbColumn column, int ordinal) + { + Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType; + ColumnKind kind = GetKind(type); + return new ColumnPlan( + ordinal, + column.ColumnName ?? $"column_{ordinal}", + GetArrowType(kind), + column.AllowDBNull ?? true, + kind); + } + + public ColumnAppender CreateAppender() => Kind switch + { + ColumnKind.Boolean => new BooleanColumnAppender(Ordinal), + ColumnKind.Int32 => new Int32ColumnAppender(Ordinal), + ColumnKind.Int64 => new Int64ColumnAppender(Ordinal), + ColumnKind.Float => new FloatColumnAppender(Ordinal), + ColumnKind.Double => new DoubleColumnAppender(Ordinal), + _ => new StringColumnAppender(Ordinal), + }; + + private static ColumnKind GetKind(Type? type) + { + if (type == typeof(bool)) + { + return ColumnKind.Boolean; + } + + if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int)) + { + return ColumnKind.Int32; + } + + if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong)) + { + return ColumnKind.Int64; + } + + if (type == typeof(float)) + { + return ColumnKind.Float; + } + + if (type == typeof(double) || type == typeof(decimal)) + { + return ColumnKind.Double; + } + + return ColumnKind.String; + } + + private static IArrowType GetArrowType(ColumnKind kind) => kind switch + { + ColumnKind.Boolean => BooleanType.Default, + ColumnKind.Int32 => Int32Type.Default, + ColumnKind.Int64 => Int64Type.Default, + ColumnKind.Float => FloatType.Default, + ColumnKind.Double => DoubleType.Default, + _ => StringType.Default, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs new file mode 100644 index 0000000..afcf9d4 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly DoubleArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs new file mode 100644 index 0000000..c25d5fb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly FloatArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs new file mode 100644 index 0000000..f68a3c5 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int32Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs new file mode 100644 index 0000000..1b0a7fc --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int64Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs new file mode 100644 index 0000000..15fa8d7 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public static class SessionContextExtensions +{ + public static void RegisterSqlite(this SessionContext context, string connectionString) + { + context.RegisterSqlite(new SqliteDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews)) + { + context.RegisterStreamingTable(tableName, new SqliteStreamingTableProvider(new SqliteTableOptions + { + ConnectionString = options.ConnectionString, + TableName = tableName, + BatchSize = options.BatchSize, + })); + } + } + + public static void RegisterSqlite(this SessionContext context, string name, SqliteTableOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + + context.RegisterStreamingTable(name, new SqliteStreamingTableProvider(options)); + } + + public static void RegisterSqlite(this SessionContext context, string name, string connectionString) + { + context.RegisterSqlite(name, new SqliteTableOptions + { + ConnectionString = connectionString, + TableName = name, + }); + } + + private static IEnumerable FetchTables(string connectionString, bool includeViews) + { + using SqliteConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT name + FROM sqlite_master + WHERE type IN ('table', 'view') + AND (@include_views OR type = 'table') + AND name NOT LIKE 'sqlite_%' + ORDER BY name + """; + + DbParameter includeViewsParameter = command.CreateParameter(); + includeViewsParameter.ParameterName = "@include_views"; + includeViewsParameter.Value = includeViews; + command.Parameters.Add(includeViewsParameter); + + using DbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return reader.GetString(0); + } + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs new file mode 100644 index 0000000..659f3cf --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs new file mode 100644 index 0000000..9f15e5b --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal abstract class SqlDialect +{ + public abstract string QuoteIdentifier(string identifier); + + public virtual string ParameterName(int index) => $"@df_filter_{index}"; + + public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch + { + StreamingTableFilterOperator.Equal => "=", + StreamingTableFilterOperator.NotEqual => "<>", + StreamingTableFilterOperator.LessThan => "<", + StreamingTableFilterOperator.LessThanOrEqual => "<=", + StreamingTableFilterOperator.GreaterThan => ">", + StreamingTableFilterOperator.GreaterThanOrEqual => ">=", + _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."), + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs new file mode 100644 index 0000000..18ed831 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed class SqlQueryBuilder(SqlDialect dialect) +{ + public PushedQuery Build(string baseQuery, StreamingTableScanRequest request) + { + ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery); + ArgumentNullException.ThrowIfNull(request); + + List parameters = []; + SqlSelectStatement statement = new( + BuildProjection(request), + TrimTrailingSemicolon(baseQuery), + BuildPredicates(request, parameters), + request.Limit); + + return new PushedQuery(statement.ToSql(), parameters); + } + + private string BuildProjection(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return "*"; + } + + if (request.Projection.Count == 0) + { + return "1"; + } + + return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier)); + } + + private List BuildPredicates( + StreamingTableScanRequest request, + List parameters) + { + List predicates = []; + for (int i = 0; i < request.Filters.Count; i++) + { + StreamingTableFilter filter = request.Filters[i]; + string parameterName = dialect.ParameterName(i); + predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}"); + parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter))); + } + + return predicates; + } + + private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch + { + StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value), + StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.String => filter.Value, + _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."), + }; + + private static string TrimTrailingSemicolon(string sql) => + sql.Trim().TrimEnd(';'); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs new file mode 100644 index 0000000..1c78e65 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record SqlQueryParameter(string Name, object Value); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs new file mode 100644 index 0000000..754cb6f --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Globalization; + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record SqlSelectStatement( + string Projection, + string Source, + IReadOnlyList Predicates, + ulong? Limit) +{ + public string ToSql() + { + List clauses = + [ + $"SELECT {Projection}", + $"FROM ({Source}) AS df_source", + ]; + + if (Predicates.Count > 0) + { + clauses.Add($"WHERE {string.Join(" AND ", Predicates)}"); + } + + if (Limit.HasValue) + { + clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + return string.Join(" ", clauses); + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs new file mode 100644 index 0000000..3851ed6 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.Sqlite.Sql; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class SqliteArrowArrayStream : IArrowArrayStream +{ + private readonly Schema schema; + private readonly ColumnPlan[] columns; + private readonly int batchSize; + private SqliteConnection? connection; + private DbCommand? command; + private DbDataReader? reader; + private bool finished; + + public SqliteArrowArrayStream( + string connectionString, + string query, + Schema schema, + ColumnPlan[] columns, + int batchSize, + IReadOnlyList? parameters = null) + { + this.schema = schema; + this.columns = columns; + this.batchSize = batchSize; + + try + { + connection = new SqliteConnection(connectionString); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + } + catch + { + Dispose(); + throw; + } + } + + public Schema Schema => schema; + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (finished || reader is null) + { + return new ValueTask((RecordBatch?)null); + } + + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + int rowCount = 0; + while (rowCount < batchSize && reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (ColumnAppender appender in appenders) + { + appender.Append(reader); + } + + rowCount++; + } + + if (rowCount == 0) + { + finished = true; + return new ValueTask((RecordBatch?)null); + } + + IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + RecordBatch batch = new(schema, arrays, rowCount); + return new ValueTask(batch); + } + + public void Dispose() + { + reader?.Dispose(); + command?.Dispose(); + connection?.Dispose(); + reader = null; + command = null; + connection = null; + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs new file mode 100644 index 0000000..9f7eb32 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs new file mode 100644 index 0000000..2e749b0 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion.TableProviders.Sqlite.Sql; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class SqliteDialect : SqlDialect +{ + public static SqliteDialect Instance { get; } = new(); + + private SqliteDialect() + { + } + + public override string QuoteIdentifier(string identifier) => + $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs new file mode 100644 index 0000000..1136094 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.Sqlite.Sql; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteStreamingTableProvider : StreamingTableProvider +{ + private readonly string connectionString; + private readonly string sourceSql; + private readonly int batchSize; + private readonly ColumnPlan[] columns; + private readonly SqlQueryBuilder queryBuilder = new(SqliteDialect.Instance); + + public SqliteStreamingTableProvider(SqliteTableOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + connectionString = options.ConnectionString; + sourceSql = BuildSourceSql(options); + batchSize = options.BatchSize; + columns = FetchColumns(connectionString, sourceSql); + Schema = BuildSchema(columns); + } + + public override Schema Schema { get; } + + public override bool SupportsPushdown => true; + + public override IArrowArrayStream Scan() => + new SqliteArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + + public override IArrowArrayStream Scan(StreamingTableScanRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); + ColumnPlan[] projectedColumns = ProjectColumns(request); + Schema projectedSchema = BuildSchema(projectedColumns); + return new SqliteArrowArrayStream( + connectionString, + pushedQuery.Sql, + projectedSchema, + projectedColumns, + batchSize, + pushedQuery.Parameters); + } + + private static string BuildSourceSql(SqliteTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {SqliteDialect.Instance.QuoteIdentifier(options.TableName)}"; + } + + private static ColumnPlan[] FetchColumns(string connectionString, string query) + { + using SqliteConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = query; + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo); + return reader.GetColumnSchema() + .Select((column, ordinal) => ColumnPlan.From(column, ordinal)) + .ToArray(); + } + + private static Schema BuildSchema(IEnumerable columns) + { + Schema.Builder builder = new(); + foreach (ColumnPlan column in columns) + { + builder.Field(field => field + .Name(column.Name) + .DataType(column.DataType) + .Nullable(column.Nullable)); + } + + return builder.Build(); + } + + private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return columns; + } + + Dictionary byName = columns.ToDictionary( + static column => column.Name, + StringComparer.OrdinalIgnoreCase); + return request.Projection + .Select(name => byName.TryGetValue(name, out ColumnPlan? column) + ? column + : throw new InvalidOperationException($"SQLite table provider does not contain column '{name}'.")) + .ToArray(); + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs new file mode 100644 index 0000000..28773a3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteTableOptions +{ + public required string ConnectionString { get; init; } + + public required string TableName { get; init; } + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs new file mode 100644 index 0000000..ff4a586 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly StringArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(FormatValue(reader.GetValue(Ordinal))); + } + + public override IArrowArray Build() => builder.Build(); +} From 176d6f222038f1bc179ef67baad06adb523d42b4 Mon Sep 17 00:00:00 2001 From: lqlive Date: Wed, 17 Jun 2026 13:50:50 +0800 Subject: [PATCH 4/4] Add federated query example: SQLite + CSV with DataFusion Introduce Apache.DataFusion.Sample.FederatedQuery, an example project showing how to join data from a SQLite table and a CSV file using DataFusion in C#. Update the solution and README to include this scenario. Refactor column projection logic in several StreamingTableProvider classes for consistency. --- README.md | 1 + datafusion-csharp.slnx | 1 + ...he.DataFusion.Sample.FederatedQuery.csproj | 19 ++++ .../Program.cs | 91 +++++++++++++++++++ .../ClickHouseStreamingTableProvider.cs | 6 +- .../MongoDbStreamingTableProvider.cs | 4 +- .../MySqlStreamingTableProvider.cs | 6 +- .../PostgreSqlStreamingTableProvider.cs | 6 +- .../SqliteStreamingTableProvider.cs | 6 +- 9 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj create mode 100644 examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs diff --git a/README.md b/README.md index 377bd81..6879ae9 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ Each scenario is an independent, runnable console project under `examples/`: | `Apache.DataFusion.Sample.Streaming` | Stream Arrow batches and read typed columns | | `Apache.DataFusion.Sample.SessionConfig` | Configure the session builder + observability | | `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query | +| `Apache.DataFusion.Sample.FederatedQuery` | Run one SQL query across SQLite and CSV sources | | `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider | | `Apache.DataFusion.Sample.PostgreSql` | Query a PostgreSQL table through a managed streaming provider | | `Apache.DataFusion.Sample.Sqlite` | Query a SQLite table through a managed streaming provider | diff --git a/datafusion-csharp.slnx b/datafusion-csharp.slnx index 5b5e8d6..ff9e284 100644 --- a/datafusion-csharp.slnx +++ b/datafusion-csharp.slnx @@ -9,6 +9,7 @@ + diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj new file mode 100644 index 0000000..6922de3 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs new file mode 100644 index 0000000..60f8b8a --- /dev/null +++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; +using Microsoft.Data.Sqlite; + +string sqliteConnectionString = CreateSqliteOrders(); +string customerRegionsPath = WriteCustomerRegionsCsv(); + +using SessionContext context = new(); +context.RegisterSqlite(sqliteConnectionString); +context.RegisterCsv("customer_regions", customerRegionsPath, new CsvReadOptions { HasHeader = true }); + +Console.WriteLine("Federated query joining SQLite orders with CSV customer regions:"); +using DataFrame df = context.Sql(""" + SELECT r.region, o.customer, SUM(o.total) AS spend + FROM datafusion_orders AS o + INNER JOIN customer_regions AS r + ON o.customer = r.customer + GROUP BY r.region, o.customer + ORDER BY r.region, o.customer + """); +df.Show(); + +static string CreateSqliteOrders() +{ + string directory = WorkDirectory(); + string databasePath = Path.Combine(directory, "orders.sqlite"); + string connectionString = new SqliteConnectionStringBuilder + { + DataSource = databasePath, + }.ToString(); + + using SqliteConnection connection = new(connectionString); + connection.Open(); + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer TEXT NOT NULL, + total REAL NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); + + return connectionString; +} + +static string WriteCustomerRegionsCsv() +{ + string path = Path.Combine(WorkDirectory(), "customer_regions.csv"); + File.WriteAllText( + path, + """ + customer,region + alice,west + bob,east + carol,central + """); + return path; +} + +static string WorkDirectory() +{ + string directory = Path.Combine(Path.GetTempPath(), "datafusion-csharp-examples", "federated-query"); + Directory.CreateDirectory(directory); + return directory; +} diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index ca7fca7..6a66879 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -128,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"ClickHouse table provider does not contain column '{name}'.")) .ToArray(); } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs index 2994a64..e4beaf7 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs @@ -93,11 +93,11 @@ private MongoDbColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.Ordinal); return [.. request.Projection - .Select(name => byName.TryGetValue(name, out MongoDbColumnPlan? column) + .Select(name => columnLookup.TryGetValue(name, out MongoDbColumnPlan? column) ? column : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'."))]; } diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs index 2724468..65c395d 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs @@ -128,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"MySQL table provider does not contain column '{name}'.")) .ToArray(); } diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs index 8e0c5a0..6671e8b 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs @@ -124,12 +124,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"PostgreSQL table provider does not contain column '{name}'.")) .ToArray(); } diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs index 1136094..9381fcf 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs @@ -120,12 +120,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"SQLite table provider does not contain column '{name}'.")) .ToArray(); }