diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..151fa6f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*.{cs,vb}] +# Keep C# using directives in one block when the IDE organizes imports. +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false +dotnet_diagnostic.IDE0055.severity = warning +dotnet_diagnostic.IDE0005.severity = warning diff --git a/README.md b/README.md index 5e57d54..6879ae9 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,10 @@ Each scenario is an independent, runnable console project under `examples/`: | `Apache.DataFusion.Sample.Streaming` | Stream Arrow batches and read typed columns | | `Apache.DataFusion.Sample.SessionConfig` | Configure the session builder + observability | | `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query | +| `Apache.DataFusion.Sample.FederatedQuery` | Run one SQL query across SQLite and CSV sources | | `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider | +| `Apache.DataFusion.Sample.PostgreSql` | Query a PostgreSQL table through a managed streaming provider | +| `Apache.DataFusion.Sample.Sqlite` | Query a SQLite table through a managed streaming provider | | `Apache.DataFusion.Sample.MongoDB` | Query a MongoDB collection through a managed streaming provider | | `Apache.DataFusion.Sample.ClickHouse` | Query a ClickHouse table through the official `ClickHouse.Driver` | @@ -162,7 +165,7 @@ dotnet run --project examples/Apache.DataFusion.Sample.Sql `StreamingTableProvider`, a lazy callback-based provider whose `Scan` is invoked by the engine and streamed in over the Arrow C Data Interface (connect any managed source, e.g. ADO.NET, with no native database driver). Managed provider packages currently include - MySQL, MongoDB, and ClickHouse. + MySQL, PostgreSQL, SQLite, MongoDB, and ClickHouse. - **Scalar UDF**: zero-argument `Int64` managed callbacks. - **Typed exceptions**: `DataFusionException` (base), plus `DataFusionPlanException`, `DataFusionExecutionException`, `DataFusionIoException`, @@ -222,44 +225,77 @@ must return a new, independently consumable stream, which the engine disposes on ### Managed database providers Managed database provider packages are built on `StreamingTableProvider`. They keep database -drivers on the C# side and push simple projection, filter, and limit requests down to the -source query when possible. +drivers on the C# side, register discovered tables and collections in the current +`SessionContext`, and push simple projection, filter, and limit requests down when possible. | Provider package | Extension method | Driver | | --- | --- | --- | | `Apache.DataFusion.TableProviders.MySql` | `RegisterMySql` | `MySqlConnector` | +| `Apache.DataFusion.TableProviders.PostgreSql` | `RegisterPostgreSql` | `Npgsql` | +| `Apache.DataFusion.TableProviders.Sqlite` | `RegisterSqlite` | `Microsoft.Data.Sqlite` | | `Apache.DataFusion.TableProviders.MongoDB` | `RegisterMongoDb` | `MongoDB.Driver` | | `Apache.DataFusion.TableProviders.ClickHouse` | `RegisterClickHouse` | official `ClickHouse.Driver` | ```csharp using Apache.DataFusion; -using Apache.DataFusion.TableProviders.ClickHouse; +using Apache.DataFusion.TableProviders.PostgreSql; using SessionContext context = new(); -context.RegisterClickHouse("audit_log", new ClickHouseTableOptions -{ - ConnectionString = "Host=localhost;Port=8123;Username=default;Password=;Database=default", - Query = """SELECT * FROM audit_log""", -}); +context.RegisterPostgreSql("Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample"); + +using DataFrame df = context.Sql(""" + SELECT * + FROM datafusion_orders + LIMIT 10 + """); +df.Show(); +``` + +SQL database providers register tables and views from the selected database in one call: + +```csharp +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.MySql; + +using SessionContext context = new(); +context.RegisterMySql("Server=localhost;User ID=root;Password=pass;Database=datafusion_sample"); + +using DataFrame df = context.Sql(""" + SELECT * + FROM datafusion_orders + LIMIT 10 + """); +df.Show(); +``` + +SQLite uses the same registration shape for a local database file: + +```csharp +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; + +using SessionContext context = new(); +context.RegisterSqlite("Data Source=datafusion_sample.sqlite"); using DataFrame df = context.Sql(""" SELECT * - FROM audit_log + FROM datafusion_orders LIMIT 10 """); df.Show(); ``` -The registered DataFusion table name is the first argument to the extension method. The -provider's `Query` points at the remote database table or subquery. For example, if you -register `audit_log`, query `FROM audit_log` in DataFusion SQL; if you register `orders`, -query `FROM orders`. +Database registration uses each discovered table or collection name as the DataFusion table +name by default. Single-table registration overloads are also available when you only want +to expose one table. The sample projects use environment variables for connection strings: | Sample | Environment variable | | --- | --- | | `Apache.DataFusion.Sample.MySql` | `DATAFUSION_MYSQL_CONNECTION` | +| `Apache.DataFusion.Sample.PostgreSql` | `DATAFUSION_POSTGRESQL_CONNECTION` | +| `Apache.DataFusion.Sample.Sqlite` | `DATAFUSION_SQLITE_CONNECTION` | | `Apache.DataFusion.Sample.MongoDB` | `DATAFUSION_MONGODB_CONNECTION` | | `Apache.DataFusion.Sample.ClickHouse` | `DATAFUSION_CLICKHOUSE_CONNECTION` | diff --git a/datafusion-csharp.slnx b/datafusion-csharp.slnx index 8a5a694..ff9e284 100644 --- a/datafusion-csharp.slnx +++ b/datafusion-csharp.slnx @@ -3,12 +3,17 @@ + + + + + diff --git a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs index b6a04c1..02e346b 100644 --- a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs +++ b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs @@ -52,16 +52,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES } using SessionContext context = new(); -context.RegisterClickHouse("orders", new ClickHouseTableOptions -{ - ConnectionString = connectionString, - Query = """SELECT id, customer, total FROM datafusion_orders""", -}); +context.RegisterClickHouse(connectionString); Console.WriteLine("Customer spend from a ClickHouse-backed streaming table:"); using DataFrame df = context.Sql(""" SELECT customer, SUM(total) AS spend - FROM orders + FROM datafusion_orders WHERE total >= 10 GROUP BY customer ORDER BY customer diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj new file mode 100644 index 0000000..6922de3 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs new file mode 100644 index 0000000..60f8b8a --- /dev/null +++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; +using Microsoft.Data.Sqlite; + +string sqliteConnectionString = CreateSqliteOrders(); +string customerRegionsPath = WriteCustomerRegionsCsv(); + +using SessionContext context = new(); +context.RegisterSqlite(sqliteConnectionString); +context.RegisterCsv("customer_regions", customerRegionsPath, new CsvReadOptions { HasHeader = true }); + +Console.WriteLine("Federated query joining SQLite orders with CSV customer regions:"); +using DataFrame df = context.Sql(""" + SELECT r.region, o.customer, SUM(o.total) AS spend + FROM datafusion_orders AS o + INNER JOIN customer_regions AS r + ON o.customer = r.customer + GROUP BY r.region, o.customer + ORDER BY r.region, o.customer + """); +df.Show(); + +static string CreateSqliteOrders() +{ + string directory = WorkDirectory(); + string databasePath = Path.Combine(directory, "orders.sqlite"); + string connectionString = new SqliteConnectionStringBuilder + { + DataSource = databasePath, + }.ToString(); + + using SqliteConnection connection = new(connectionString); + connection.Open(); + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer TEXT NOT NULL, + total REAL NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); + + return connectionString; +} + +static string WriteCustomerRegionsCsv() +{ + string path = Path.Combine(WorkDirectory(), "customer_regions.csv"); + File.WriteAllText( + path, + """ + customer,region + alice,west + bob,east + carol,central + """); + return path; +} + +static string WorkDirectory() +{ + string directory = Path.Combine(Path.GetTempPath(), "datafusion-csharp-examples", "federated-query"); + Directory.CreateDirectory(directory); + return directory; +} diff --git a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs index 77e2d23..57d434a 100644 --- a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs +++ b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs @@ -68,11 +68,10 @@ ]); using SessionContext context = new(); -context.RegisterMongoDb("orders", new MongoDbTableOptions +context.RegisterMongoDb(new MongoDbDatabaseOptions { ConnectionString = connectionString, DatabaseName = databaseName, - CollectionName = collectionName, }); Console.WriteLine("Customer spend from a MongoDB-backed streaming table:"); diff --git a/examples/Apache.DataFusion.Sample.MySql/Program.cs b/examples/Apache.DataFusion.Sample.MySql/Program.cs index 0a2ce31..0a0dfd4 100644 --- a/examples/Apache.DataFusion.Sample.MySql/Program.cs +++ b/examples/Apache.DataFusion.Sample.MySql/Program.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using MySqlConnector; using Apache.DataFusion; using Apache.DataFusion.TableProviders.MySql; +using MySqlConnector; string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_MYSQL_CONNECTION"); if (string.IsNullOrWhiteSpace(connectionString)) @@ -51,16 +51,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES } using SessionContext context = new(); -context.RegisterMySql("orders", new MySqlTableOptions -{ - ConnectionString = connectionString, - Query = """SELECT id, customer, total FROM datafusion_orders""", -}); +context.RegisterMySql(connectionString); Console.WriteLine("Customer spend from a MySQL-backed streaming table:"); using DataFrame df = context.Sql(""" SELECT customer, SUM(total) AS spend - FROM orders + FROM datafusion_orders GROUP BY customer ORDER BY customer """); diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj new file mode 100644 index 0000000..1b56c28 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs new file mode 100644 index 0000000..1441ca8 --- /dev/null +++ b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.PostgreSql; +using Npgsql; + +string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_POSTGRESQL_CONNECTION"); +if (string.IsNullOrWhiteSpace(connectionString)) +{ + Console.WriteLine("Set DATAFUSION_POSTGRESQL_CONNECTION to run this sample."); + Console.WriteLine("Example:"); + Console.WriteLine(" Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample"); + return; +} + +using (NpgsqlConnection connection = new(connectionString)) +{ + connection.Open(); + using NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer VARCHAR(64) NOT NULL, + total DOUBLE PRECISION NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); +} + +using SessionContext context = new(); +context.RegisterPostgreSql(connectionString); + +Console.WriteLine("Customer spend from a PostgreSQL-backed streaming table:"); +using DataFrame df = context.Sql(""" + SELECT customer, SUM(total) AS spend + FROM datafusion_orders + GROUP BY customer + ORDER BY customer + """); +df.Show(); diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj new file mode 100644 index 0000000..0df4e4b --- /dev/null +++ b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Program.cs b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs new file mode 100644 index 0000000..3a7b5cf --- /dev/null +++ b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion; +using Apache.DataFusion.TableProviders.Sqlite; +using Microsoft.Data.Sqlite; + +string connectionString = Environment.GetEnvironmentVariable("DATAFUSION_SQLITE_CONNECTION") + ?? new SqliteConnectionStringBuilder + { + DataSource = Path.Combine(Path.GetTempPath(), "datafusion_sample.sqlite"), + }.ToString(); + +using (SqliteConnection connection = new(connectionString)) +{ + connection.Open(); + using SqliteCommand command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS datafusion_orders ( + id INTEGER PRIMARY KEY, + customer TEXT NOT NULL, + total REAL NOT NULL + ); + + DELETE FROM datafusion_orders; + + INSERT INTO datafusion_orders (id, customer, total) VALUES + (1, 'alice', 19.99), + (2, 'bob', 7.50), + (3, 'alice', 100.00), + (4, 'carol', 42.25); + """; + command.ExecuteNonQuery(); +} + +using SessionContext context = new(); +context.RegisterSqlite(connectionString); + +Console.WriteLine("Customer spend from a SQLite-backed streaming table:"); +using DataFrame df = context.Sql(""" + SELECT customer, SUM(total) AS spend + FROM datafusion_orders + GROUP BY customer + ORDER BY customer + """); +df.Show(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj index 8ec5485..35e0d23 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj @@ -1,4 +1,4 @@ - + net10.0 diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs index 6aa8010..3d6832d 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs index 39c201b..8750e39 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; - using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs new file mode 100644 index 0000000..4440322 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.ClickHouse; + +public sealed class ClickHouseDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index 0fbea39..6a66879 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.ClickHouse.Sql; - using ClickHouse.Driver.ADO; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -29,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.ClickHouse; public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider { private readonly string connectionString; - private readonly string query; + private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; private readonly ClickHouseQueryBuilder queryBuilder = new(ClickHouseDialect.Instance); @@ -42,20 +40,15 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); } - if (string.IsNullOrWhiteSpace(options.Query)) - { - throw new ArgumentException("Query cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } connectionString = options.ConnectionString; - query = options.Query; + sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, query); + columns = FetchColumns(connectionString, sourceSql); Schema = BuildSchema(columns); } @@ -64,13 +57,13 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new ClickHouseArrowArrayStream(connectionString, query, Schema, columns, batchSize); + new ClickHouseArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { ArgumentNullException.ThrowIfNull(request); - PushedQuery pushedQuery = queryBuilder.Build(query, request); + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new ClickHouseArrowArrayStream( @@ -82,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static string BuildSourceSql(ClickHouseTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string? databaseName, string tableName) + { + string quotedTable = ClickHouseDialect.Instance.QuoteIdentifier(tableName); + return string.IsNullOrWhiteSpace(databaseName) + ? quotedTable + : $"{ClickHouseDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; + } + private static ColumnPlan[] FetchColumns(string connectionString, string query) { using ClickHouseConnection connection = new(connectionString); @@ -117,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"ClickHouse table provider does not contain column '{name}'.")) .ToArray(); } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs index 2114ded..c1471ab 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs @@ -21,7 +21,9 @@ public sealed class ClickHouseTableOptions { public required string ConnectionString { get; init; } - public required string Query { get; init; } + public string? DatabaseName { get; init; } + + public required string TableName { get; init; } public int BatchSize { get; init; } = 1024; } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs index 746d716..8ddaa2b 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs index 3c4e284..bc8d475 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; - using Apache.Arrow.Types; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs index 5769f28..550a549 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs index bc1f71c..a80f96e 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs index 58f2d32..c47ea6e 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs index 277f9be..a100bf2 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs @@ -17,7 +17,6 @@ using System.Data.Common; using System.Globalization; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs index 1afe8ce..740e39b 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs @@ -15,10 +15,47 @@ // specific language governing permissions and limitations // under the License. +using System.Data.Common; +using ClickHouse.Driver.ADO; + namespace Apache.DataFusion.TableProviders.ClickHouse; public static class SessionContextExtensions { + public static void RegisterClickHouse(this SessionContext context, string connectionString) + { + context.RegisterClickHouse(new ClickHouseDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterClickHouse(this SessionContext context, ClickHouseDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + { + context.RegisterStreamingTable(table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = table.DatabaseName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + public static void RegisterClickHouse(this SessionContext context, string name, ClickHouseTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,12 +64,48 @@ public static void RegisterClickHouse(this SessionContext context, string name, context.RegisterStreamingTable(name, new ClickHouseStreamingTableProvider(options)); } - public static void RegisterClickHouse(this SessionContext context, string name, string connectionString, string query) + public static void RegisterClickHouse(this SessionContext context, string name, string connectionString) { context.RegisterClickHouse(name, new ClickHouseTableOptions { ConnectionString = connectionString, - Query = query, + TableName = name, }); } + + private static IEnumerable FetchTables( + string connectionString, + string? databaseName, + bool includeViews) + { + using ClickHouseConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT database, name + FROM system.tables + WHERE database = if({database:String} = '', currentDatabase(), {database:String}) + AND ({include_views:Bool} OR engine NOT IN ('View', 'MaterializedView')) + ORDER BY name + """; + + DbParameter databaseParameter = command.CreateParameter(); + databaseParameter.ParameterName = "database"; + databaseParameter.Value = databaseName ?? string.Empty; + command.Parameters.Add(databaseParameter); + + DbParameter includeViewsParameter = command.CreateParameter(); + includeViewsParameter.ParameterName = "include_views"; + includeViewsParameter.Value = includeViews; + command.Parameters.Add(includeViewsParameter); + + using DbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return new ClickHouseTable(reader.GetString(0), reader.GetString(1)); + } + } + + private sealed record ClickHouseTable(string DatabaseName, string TableName); } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs index 7870b79..e2d3092 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; - using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs index 116d64f..a4938d2 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs @@ -17,7 +17,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; - using MongoDB.Bson; using MongoDB.Driver; @@ -73,7 +72,7 @@ public MongoDbArrowArrayStream( return new ValueTask((RecordBatch?)null); } - MongoDbColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + MongoDbColumnAppender[] appenders = [.. columns.Select(static column => column.CreateAppender())]; int rowCount = 0; while (rowCount < batchSize) { @@ -97,7 +96,7 @@ public MongoDbArrowArrayStream( return new ValueTask((RecordBatch?)null); } - IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + IArrowArray[] arrays = [.. appenders.Select(static appender => appender.Build())]; RecordBatch batch = new(schema, arrays, rowCount); return new ValueTask(batch); } @@ -123,7 +122,7 @@ private bool TryReadNextDocument(out BsonDocument document) currentBatch = null; if (!cursor.MoveNext()) { - document = new BsonDocument(); + document = []; return false; } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs index 74bc8a7..39ff62a 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs @@ -16,9 +16,7 @@ // under the License. using System.Globalization; - using Apache.Arrow; - using MongoDB.Bson; namespace Apache.DataFusion.TableProviders.MongoDB; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs index 974d8ff..195c886 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs @@ -16,7 +16,6 @@ // under the License. using Apache.Arrow.Types; - using MongoDB.Bson; namespace Apache.DataFusion.TableProviders.MongoDB; @@ -53,13 +52,12 @@ public static MongoDbColumnPlan[] Infer(IReadOnlyList documents) } } - return columns + return [.. columns .Select(column => new MongoDbColumnPlan( column.Key, GetArrowType(column.Value.Kind), column.Value.Nullable || column.Value.ObservedCount < documents.Count, - column.Value.Kind)) - .ToArray(); + column.Value.Kind))]; } private static IArrowType GetArrowType(MongoDbColumnKind kind) => kind switch diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs new file mode 100644 index 0000000..64250bf --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.MongoDB; + +public sealed class MongoDbDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public int BatchSize { get; init; } = 1024; + + public int SchemaInferenceLimit { get; init; } = 100; +} diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs index 7111487..0d35989 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs @@ -16,7 +16,6 @@ // under the License. using System.Globalization; - using MongoDB.Bson; using MongoDB.Driver; diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs index 9576237..e4beaf7 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs @@ -17,7 +17,6 @@ using Apache.Arrow; using Apache.Arrow.Ipc; - using MongoDB.Bson; using MongoDB.Driver; @@ -94,14 +93,13 @@ private MongoDbColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.Ordinal); - return request.Projection - .Select(name => byName.TryGetValue(name, out MongoDbColumnPlan? column) + return [.. request.Projection + .Select(name => columnLookup.TryGetValue(name, out MongoDbColumnPlan? column) ? column - : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'.")) - .ToArray(); + : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'."))]; } private static MongoDbColumnPlan[] InferColumns(IMongoCollection collection, int schemaInferenceLimit) diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs index f023293..643894f 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs @@ -15,10 +15,55 @@ // specific language governing permissions and limitations // under the License. +using MongoDB.Driver; + namespace Apache.DataFusion.TableProviders.MongoDB; public static class SessionContextExtensions { + public static void RegisterMongoDb(this SessionContext context, string connectionString) + { + context.RegisterMongoDb(new MongoDbDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + if (options.SchemaInferenceLimit <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero."); + } + + string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName); + MongoClient client = new(options.ConnectionString); + IMongoDatabase database = client.GetDatabase(databaseName); + foreach (string collectionName in database.ListCollectionNames().ToEnumerable()) + { + context.RegisterStreamingTable(collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = databaseName, + CollectionName = collectionName, + BatchSize = options.BatchSize, + SchemaInferenceLimit = options.SchemaInferenceLimit, + })); + } + } + public static void RegisterMongoDb(this SessionContext context, string name, MongoDbTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,6 +72,17 @@ public static void RegisterMongoDb(this SessionContext context, string name, Mon context.RegisterStreamingTable(name, new MongoDbStreamingTableProvider(options)); } + public static void RegisterMongoDb(this SessionContext context, string name, string connectionString) + { + string databaseName = ResolveDatabaseName(connectionString, null); + context.RegisterMongoDb(name, new MongoDbTableOptions + { + ConnectionString = connectionString, + DatabaseName = databaseName, + CollectionName = name, + }); + } + public static void RegisterMongoDb( this SessionContext context, string name, @@ -41,4 +97,20 @@ public static void RegisterMongoDb( CollectionName = collectionName, }); } + + private static string ResolveDatabaseName(string connectionString, string? databaseName) + { + if (!string.IsNullOrWhiteSpace(databaseName)) + { + return databaseName; + } + + string? connectionDatabaseName = MongoUrl.Create(connectionString).DatabaseName; + if (!string.IsNullOrWhiteSpace(connectionDatabaseName)) + { + return connectionDatabaseName; + } + + throw new ArgumentException("Database name must be provided in MongoDB options or connection string.", nameof(connectionString)); + } } diff --git a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs index 64ebca9..b66fa20 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs index 82f2bc8..114b66f 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; - +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs index 12fd3d7..e48a8e3 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow.Types; using System.Data.Common; +using Apache.Arrow.Types; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs index 87fdd11..2c63817 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs index 69d992e..5f23fc3 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs index 6c66f83..d3e4675 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs index d85c941..d2e8d29 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using Apache.Arrow; using System.Data.Common; using System.Globalization; +using Apache.Arrow; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs index 5ccd36c..2ce32a1 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.MySql.Sql; - using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs new file mode 100644 index 0000000..6ff05bb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.MySql; + +public sealed class MySqlDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public string? DatabaseName { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs index 72f11b3..65c395d 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs @@ -17,11 +17,9 @@ using System.Data; using System.Data.Common; - using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.MySql.Sql; - using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -29,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.MySql; public sealed class MySqlStreamingTableProvider : StreamingTableProvider { private readonly string connectionString; - private readonly string query; + private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; private readonly SqlQueryBuilder queryBuilder = new(MySqlDialect.Instance); @@ -42,20 +40,15 @@ public MySqlStreamingTableProvider(MySqlTableOptions options) throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); } - if (string.IsNullOrWhiteSpace(options.Query)) - { - throw new ArgumentException("Query cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } connectionString = options.ConnectionString; - query = options.Query; + sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, query); + columns = FetchColumns(connectionString, sourceSql); Schema = BuildSchema(columns); } @@ -64,13 +57,13 @@ public MySqlStreamingTableProvider(MySqlTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new MySqlArrowArrayStream(connectionString, query, Schema, columns, batchSize); + new MySqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { ArgumentNullException.ThrowIfNull(request); - PushedQuery pushedQuery = queryBuilder.Build(query, request); + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new MySqlArrowArrayStream( @@ -82,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static string BuildSourceSql(MySqlTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string? databaseName, string tableName) + { + string quotedTable = MySqlDialect.Instance.QuoteIdentifier(tableName); + return string.IsNullOrWhiteSpace(databaseName) + ? quotedTable + : $"{MySqlDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; + } + private static ColumnPlan[] FetchColumns(string connectionString, string query) { using MySqlConnection connection = new(connectionString); @@ -117,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) return columns; } - Dictionary byName = columns.ToDictionary( + Dictionary columnLookup = columns.ToDictionary( static column => column.Name, StringComparer.OrdinalIgnoreCase); return request.Projection - .Select(name => byName.TryGetValue(name, out ColumnPlan? column) - ? column + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } : throw new InvalidOperationException($"MySQL table provider does not contain column '{name}'.")) .ToArray(); } diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs index 48e4e9b..c91e4ac 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs @@ -21,7 +21,9 @@ public sealed class MySqlTableOptions { public required string ConnectionString { get; init; } - public required string Query { get; init; } + public string? DatabaseName { get; init; } + + public required string TableName { get; init; } public int BatchSize { get; init; } = 1024; } diff --git a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs index 9f4694d..f7e87de 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs @@ -15,10 +15,46 @@ // specific language governing permissions and limitations // under the License. +using MySqlConnector; + namespace Apache.DataFusion.TableProviders.MySql; public static class SessionContextExtensions { + public static void RegisterMySql(this SessionContext context, string connectionString) + { + context.RegisterMySql(new MySqlDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + { + context.RegisterStreamingTable(table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions + { + ConnectionString = options.ConnectionString, + DatabaseName = table.DatabaseName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + public static void RegisterMySql(this SessionContext context, string name, MySqlTableOptions options) { ArgumentNullException.ThrowIfNull(context); @@ -27,12 +63,41 @@ public static void RegisterMySql(this SessionContext context, string name, MySql context.RegisterStreamingTable(name, new MySqlStreamingTableProvider(options)); } - public static void RegisterMySql(this SessionContext context, string name, string connectionString, string query) + public static void RegisterMySql(this SessionContext context, string name, string connectionString) { context.RegisterMySql(name, new MySqlTableOptions { ConnectionString = connectionString, - Query = query, + TableName = name, }); } + + private static IEnumerable FetchTables( + string connectionString, + string? databaseName, + bool includeViews) + { + using MySqlConnection connection = new(connectionString); + connection.Open(); + + using MySqlCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = COALESCE(@database_name, DATABASE()) + AND table_type IN ('BASE TABLE', 'VIEW') + AND (@include_views OR table_type = 'BASE TABLE') + ORDER BY table_name + """; + command.Parameters.AddWithValue("@database_name", string.IsNullOrWhiteSpace(databaseName) ? DBNull.Value : databaseName); + command.Parameters.AddWithValue("@include_views", includeViews); + + using MySqlDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return new MySqlTable(reader.GetString(0), reader.GetString(1)); + } + } + + private sealed record MySqlTable(string DatabaseName, string TableName); } diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj new file mode 100644 index 0000000..6874099 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs new file mode 100644 index 0000000..c7bc593 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly BooleanArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs new file mode 100644 index 0000000..bba9402 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal abstract class ColumnAppender(int ordinal) +{ + protected int Ordinal { get; } = ordinal; + + public abstract void Append(DbDataReader reader); + + public abstract IArrowArray Build(); + + protected static string FormatValue(object value) => + value switch + { + byte[] bytes => Convert.ToBase64String(bytes), + DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture), + DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture), + TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture), + DateTimeOffset dateTimeOffset => dateTimeOffset.ToString("O", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs new file mode 100644 index 0000000..492f6eb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal enum ColumnKind +{ + Boolean, + Int32, + Int64, + Float, + Double, + String, +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs new file mode 100644 index 0000000..90fadc0 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow.Types; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed record ColumnPlan( + int Ordinal, + string Name, + IArrowType DataType, + bool Nullable, + ColumnKind Kind) +{ + public static ColumnPlan From(DbColumn column, int ordinal) + { + Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType; + ColumnKind kind = GetKind(type); + return new ColumnPlan( + ordinal, + column.ColumnName ?? $"column_{ordinal}", + GetArrowType(kind), + column.AllowDBNull ?? true, + kind); + } + + public ColumnAppender CreateAppender() => Kind switch + { + ColumnKind.Boolean => new BooleanColumnAppender(Ordinal), + ColumnKind.Int32 => new Int32ColumnAppender(Ordinal), + ColumnKind.Int64 => new Int64ColumnAppender(Ordinal), + ColumnKind.Float => new FloatColumnAppender(Ordinal), + ColumnKind.Double => new DoubleColumnAppender(Ordinal), + _ => new StringColumnAppender(Ordinal), + }; + + private static ColumnKind GetKind(Type? type) + { + if (type == typeof(bool)) + { + return ColumnKind.Boolean; + } + + if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int)) + { + return ColumnKind.Int32; + } + + if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong)) + { + return ColumnKind.Int64; + } + + if (type == typeof(float)) + { + return ColumnKind.Float; + } + + if (type == typeof(double) || type == typeof(decimal)) + { + return ColumnKind.Double; + } + + return ColumnKind.String; + } + + private static IArrowType GetArrowType(ColumnKind kind) => kind switch + { + ColumnKind.Boolean => BooleanType.Default, + ColumnKind.Int32 => Int32Type.Default, + ColumnKind.Int64 => Int64Type.Default, + ColumnKind.Float => FloatType.Default, + ColumnKind.Double => DoubleType.Default, + _ => StringType.Default, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs new file mode 100644 index 0000000..4366532 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly DoubleArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs new file mode 100644 index 0000000..9ed65ce --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly FloatArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs new file mode 100644 index 0000000..de9bad3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int32Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs new file mode 100644 index 0000000..2693520 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int64Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs new file mode 100644 index 0000000..0142ccc --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.PostgreSql.Sql; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class PostgreSqlArrowArrayStream : IArrowArrayStream +{ + private readonly Schema schema; + private readonly ColumnPlan[] columns; + private readonly int batchSize; + private NpgsqlConnection? connection; + private DbCommand? command; + private DbDataReader? reader; + private bool finished; + + public PostgreSqlArrowArrayStream( + string connectionString, + string query, + Schema schema, + ColumnPlan[] columns, + int batchSize, + IReadOnlyList? parameters = null) + { + this.schema = schema; + this.columns = columns; + this.batchSize = batchSize; + + try + { + connection = new NpgsqlConnection(connectionString); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + } + catch + { + Dispose(); + throw; + } + } + + public Schema Schema => schema; + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (finished || reader is null) + { + return new ValueTask((RecordBatch?)null); + } + + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + int rowCount = 0; + while (rowCount < batchSize && reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (ColumnAppender appender in appenders) + { + appender.Append(reader); + } + + rowCount++; + } + + if (rowCount == 0) + { + finished = true; + return new ValueTask((RecordBatch?)null); + } + + IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + RecordBatch batch = new(schema, arrays, rowCount); + return new ValueTask(batch); + } + + public void Dispose() + { + reader?.Dispose(); + command?.Dispose(); + connection?.Dispose(); + reader = null; + command = null; + connection = null; + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs new file mode 100644 index 0000000..aef7378 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public IReadOnlyCollection Schemas { get; init; } = ["public"]; + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs new file mode 100644 index 0000000..9aa837e --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion.TableProviders.PostgreSql.Sql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class PostgreSqlDialect : SqlDialect +{ + public static PostgreSqlDialect Instance { get; } = new(); + + private PostgreSqlDialect() + { + } + + public override string QuoteIdentifier(string identifier) => + $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs new file mode 100644 index 0000000..6671e8b --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.PostgreSql.Sql; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlStreamingTableProvider : StreamingTableProvider +{ + private readonly string connectionString; + private readonly string sourceSql; + private readonly int batchSize; + private readonly ColumnPlan[] columns; + private readonly SqlQueryBuilder queryBuilder = new(PostgreSqlDialect.Instance); + + public PostgreSqlStreamingTableProvider(PostgreSqlTableOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + connectionString = options.ConnectionString; + sourceSql = BuildSourceSql(options); + batchSize = options.BatchSize; + columns = FetchColumns(connectionString, sourceSql); + Schema = BuildSchema(columns); + } + + public override Schema Schema { get; } + + public override bool SupportsPushdown => true; + + public override IArrowArrayStream Scan() => + new PostgreSqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + + public override IArrowArrayStream Scan(StreamingTableScanRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); + ColumnPlan[] projectedColumns = ProjectColumns(request); + Schema projectedSchema = BuildSchema(projectedColumns); + return new PostgreSqlArrowArrayStream( + connectionString, + pushedQuery.Sql, + projectedSchema, + projectedColumns, + batchSize, + pushedQuery.Parameters); + } + + private static string BuildSourceSql(PostgreSqlTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + string schemaName = string.IsNullOrWhiteSpace(options.SchemaName) ? "public" : options.SchemaName; + return $"SELECT * FROM {QuoteQualifiedTableName(schemaName, options.TableName)}"; + } + + private static string QuoteQualifiedTableName(string schemaName, string tableName) => + $"{PostgreSqlDialect.Instance.QuoteIdentifier(schemaName)}.{PostgreSqlDialect.Instance.QuoteIdentifier(tableName)}"; + + private static ColumnPlan[] FetchColumns(string connectionString, string query) + { + using NpgsqlConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = query; + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo); + return reader.GetColumnSchema() + .Select((column, ordinal) => ColumnPlan.From(column, ordinal)) + .ToArray(); + } + + private static Schema BuildSchema(IEnumerable columns) + { + Schema.Builder builder = new(); + foreach (ColumnPlan column in columns) + { + builder.Field(field => field + .Name(column.Name) + .DataType(column.DataType) + .Nullable(column.Nullable)); + } + + return builder.Build(); + } + + private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return columns; + } + + Dictionary columnLookup = columns.ToDictionary( + static column => column.Name, + StringComparer.OrdinalIgnoreCase); + return request.Projection + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } + : throw new InvalidOperationException($"PostgreSQL table provider does not contain column '{name}'.")) + .ToArray(); + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs new file mode 100644 index 0000000..15c82c8 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public sealed class PostgreSqlTableOptions +{ + public required string ConnectionString { get; init; } + + public string SchemaName { get; init; } = "public"; + + public required string TableName { get; init; } + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs new file mode 100644 index 0000000..6843e0d --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Npgsql; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +public static class SessionContextExtensions +{ + public static void RegisterPostgreSql(this SessionContext context, string connectionString) + { + context.RegisterPostgreSql(new PostgreSqlDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + string[] schemas = options.Schemas + .Where(static schema => !string.IsNullOrWhiteSpace(schema)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (schemas.Length == 0) + { + throw new ArgumentException("At least one schema must be provided.", nameof(options)); + } + + foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews)) + { + string registrationName = RegistrationName(table, schemas); + context.RegisterStreamingTable(registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions + { + ConnectionString = options.ConnectionString, + SchemaName = table.SchemaName, + TableName = table.TableName, + BatchSize = options.BatchSize, + })); + } + } + + public static void RegisterPostgreSql(this SessionContext context, string name, PostgreSqlTableOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + + context.RegisterStreamingTable(name, new PostgreSqlStreamingTableProvider(options)); + } + + public static void RegisterPostgreSql(this SessionContext context, string name, string connectionString) + { + context.RegisterPostgreSql(name, new PostgreSqlTableOptions + { + ConnectionString = connectionString, + TableName = name, + }); + } + + private static IEnumerable FetchTables( + string connectionString, + IReadOnlyCollection schemas, + bool includeViews) + { + using NpgsqlConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT table_schema, table_name + FROM information_schema.tables + WHERE table_schema = ANY(@schemas) + AND table_type = ANY(@table_types) + ORDER BY table_schema, table_name + """; + + DbParameter schemasParameter = command.CreateParameter(); + schemasParameter.ParameterName = "schemas"; + schemasParameter.Value = schemas.ToArray(); + command.Parameters.Add(schemasParameter); + + DbParameter tableTypesParameter = command.CreateParameter(); + tableTypesParameter.ParameterName = "table_types"; + tableTypesParameter.Value = includeViews + ? new[] { "BASE TABLE", "VIEW" } + : new[] { "BASE TABLE" }; + command.Parameters.Add(tableTypesParameter); + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + while (reader.Read()) + { + yield return new PostgreSqlTable(reader.GetString(0), reader.GetString(1)); + } + } + + private static string RegistrationName(PostgreSqlTable table, IReadOnlyCollection schemas) => + schemas.Count == 1 && string.Equals(table.SchemaName, "public", StringComparison.OrdinalIgnoreCase) + ? table.TableName + : $"{table.SchemaName}_{table.TableName}"; + + private sealed record PostgreSqlTable(string SchemaName, string TableName); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs new file mode 100644 index 0000000..0460058 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs new file mode 100644 index 0000000..b38e11b --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal abstract class SqlDialect +{ + public abstract string QuoteIdentifier(string identifier); + + public virtual string ParameterName(int index) => $"@df_filter_{index}"; + + public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch + { + StreamingTableFilterOperator.Equal => "=", + StreamingTableFilterOperator.NotEqual => "<>", + StreamingTableFilterOperator.LessThan => "<", + StreamingTableFilterOperator.LessThanOrEqual => "<=", + StreamingTableFilterOperator.GreaterThan => ">", + StreamingTableFilterOperator.GreaterThanOrEqual => ">=", + _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."), + }; +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs new file mode 100644 index 0000000..36259fa --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed class SqlQueryBuilder(SqlDialect dialect) +{ + public PushedQuery Build(string baseQuery, StreamingTableScanRequest request) + { + ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery); + ArgumentNullException.ThrowIfNull(request); + + List parameters = []; + SqlSelectStatement statement = new( + BuildProjection(request), + TrimTrailingSemicolon(baseQuery), + BuildPredicates(request, parameters), + request.Limit); + + return new PushedQuery(statement.ToSql(), parameters); + } + + private string BuildProjection(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return "*"; + } + + if (request.Projection.Count == 0) + { + return "1"; + } + + return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier)); + } + + private List BuildPredicates( + StreamingTableScanRequest request, + List parameters) + { + List predicates = []; + for (int i = 0; i < request.Filters.Count; i++) + { + StreamingTableFilter filter = request.Filters[i]; + string parameterName = dialect.ParameterName(i); + predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}"); + parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter))); + } + + return predicates; + } + + private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch + { + StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value), + StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.String => filter.Value, + _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."), + }; + + private static string TrimTrailingSemicolon(string sql) => + sql.Trim().TrimEnd(';'); +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs new file mode 100644 index 0000000..b172ef3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record SqlQueryParameter(string Name, object Value); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs new file mode 100644 index 0000000..53c8111 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Globalization; + +namespace Apache.DataFusion.TableProviders.PostgreSql.Sql; + +internal sealed record SqlSelectStatement( + string Projection, + string Source, + IReadOnlyList Predicates, + ulong? Limit) +{ + public string ToSql() + { + List clauses = + [ + $"SELECT {Projection}", + $"FROM ({Source}) AS df_source", + ]; + + if (Predicates.Count > 0) + { + clauses.Add($"WHERE {string.Join(" AND ", Predicates)}"); + } + + if (Limit.HasValue) + { + clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + return string.Join(" ", clauses); + } +} diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs new file mode 100644 index 0000000..b8cdf90 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.PostgreSql; + +internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly StringArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(FormatValue(reader.GetValue(Ordinal))); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj new file mode 100644 index 0000000..2e3fc81 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs new file mode 100644 index 0000000..4f290ec --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly BooleanArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs new file mode 100644 index 0000000..9cdc462 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal abstract class ColumnAppender(int ordinal) +{ + protected int Ordinal { get; } = ordinal; + + public abstract void Append(DbDataReader reader); + + public abstract IArrowArray Build(); + + protected static string FormatValue(object value) => + value switch + { + byte[] bytes => Convert.ToBase64String(bytes), + DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture), + DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture), + TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs new file mode 100644 index 0000000..d9e3876 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal enum ColumnKind +{ + Boolean, + Int32, + Int64, + Float, + Double, + String, +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs new file mode 100644 index 0000000..48a411a --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow.Types; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed record ColumnPlan( + int Ordinal, + string Name, + IArrowType DataType, + bool Nullable, + ColumnKind Kind) +{ + public static ColumnPlan From(DbColumn column, int ordinal) + { + Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType; + ColumnKind kind = GetKind(type); + return new ColumnPlan( + ordinal, + column.ColumnName ?? $"column_{ordinal}", + GetArrowType(kind), + column.AllowDBNull ?? true, + kind); + } + + public ColumnAppender CreateAppender() => Kind switch + { + ColumnKind.Boolean => new BooleanColumnAppender(Ordinal), + ColumnKind.Int32 => new Int32ColumnAppender(Ordinal), + ColumnKind.Int64 => new Int64ColumnAppender(Ordinal), + ColumnKind.Float => new FloatColumnAppender(Ordinal), + ColumnKind.Double => new DoubleColumnAppender(Ordinal), + _ => new StringColumnAppender(Ordinal), + }; + + private static ColumnKind GetKind(Type? type) + { + if (type == typeof(bool)) + { + return ColumnKind.Boolean; + } + + if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int)) + { + return ColumnKind.Int32; + } + + if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong)) + { + return ColumnKind.Int64; + } + + if (type == typeof(float)) + { + return ColumnKind.Float; + } + + if (type == typeof(double) || type == typeof(decimal)) + { + return ColumnKind.Double; + } + + return ColumnKind.String; + } + + private static IArrowType GetArrowType(ColumnKind kind) => kind switch + { + ColumnKind.Boolean => BooleanType.Default, + ColumnKind.Int32 => Int32Type.Default, + ColumnKind.Int64 => Int64Type.Default, + ColumnKind.Float => FloatType.Default, + ColumnKind.Double => DoubleType.Default, + _ => StringType.Default, + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs new file mode 100644 index 0000000..afcf9d4 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly DoubleArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs new file mode 100644 index 0000000..c25d5fb --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly FloatArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs new file mode 100644 index 0000000..f68a3c5 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int32Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs new file mode 100644 index 0000000..1b0a7fc --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using System.Globalization; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly Int64Array.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs new file mode 100644 index 0000000..15fa8d7 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public static class SessionContextExtensions +{ + public static void RegisterSqlite(this SessionContext context, string connectionString) + { + context.RegisterSqlite(new SqliteDatabaseOptions + { + ConnectionString = connectionString, + }); + } + + public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews)) + { + context.RegisterStreamingTable(tableName, new SqliteStreamingTableProvider(new SqliteTableOptions + { + ConnectionString = options.ConnectionString, + TableName = tableName, + BatchSize = options.BatchSize, + })); + } + } + + public static void RegisterSqlite(this SessionContext context, string name, SqliteTableOptions options) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(options); + + context.RegisterStreamingTable(name, new SqliteStreamingTableProvider(options)); + } + + public static void RegisterSqlite(this SessionContext context, string name, string connectionString) + { + context.RegisterSqlite(name, new SqliteTableOptions + { + ConnectionString = connectionString, + TableName = name, + }); + } + + private static IEnumerable FetchTables(string connectionString, bool includeViews) + { + using SqliteConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = """ + SELECT name + FROM sqlite_master + WHERE type IN ('table', 'view') + AND (@include_views OR type = 'table') + AND name NOT LIKE 'sqlite_%' + ORDER BY name + """; + + DbParameter includeViewsParameter = command.CreateParameter(); + includeViewsParameter.ParameterName = "@include_views"; + includeViewsParameter.Value = includeViews; + command.Parameters.Add(includeViewsParameter); + + using DbDataReader reader = command.ExecuteReader(); + while (reader.Read()) + { + yield return reader.GetString(0); + } + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs new file mode 100644 index 0000000..659f3cf --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs new file mode 100644 index 0000000..9f15e5b --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal abstract class SqlDialect +{ + public abstract string QuoteIdentifier(string identifier); + + public virtual string ParameterName(int index) => $"@df_filter_{index}"; + + public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch + { + StreamingTableFilterOperator.Equal => "=", + StreamingTableFilterOperator.NotEqual => "<>", + StreamingTableFilterOperator.LessThan => "<", + StreamingTableFilterOperator.LessThanOrEqual => "<=", + StreamingTableFilterOperator.GreaterThan => ">", + StreamingTableFilterOperator.GreaterThanOrEqual => ">=", + _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."), + }; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs new file mode 100644 index 0000000..18ed831 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed class SqlQueryBuilder(SqlDialect dialect) +{ + public PushedQuery Build(string baseQuery, StreamingTableScanRequest request) + { + ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery); + ArgumentNullException.ThrowIfNull(request); + + List parameters = []; + SqlSelectStatement statement = new( + BuildProjection(request), + TrimTrailingSemicolon(baseQuery), + BuildPredicates(request, parameters), + request.Limit); + + return new PushedQuery(statement.ToSql(), parameters); + } + + private string BuildProjection(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return "*"; + } + + if (request.Projection.Count == 0) + { + return "1"; + } + + return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier)); + } + + private List BuildPredicates( + StreamingTableScanRequest request, + List parameters) + { + List predicates = []; + for (int i = 0; i < request.Filters.Count; i++) + { + StreamingTableFilter filter = request.Filters[i]; + string parameterName = dialect.ParameterName(i); + predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}"); + parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter))); + } + + return predicates; + } + + private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch + { + StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value), + StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture), + StreamingTableFilterValueKind.String => filter.Value, + _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."), + }; + + private static string TrimTrailingSemicolon(string sql) => + sql.Trim().TrimEnd(';'); +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs new file mode 100644 index 0000000..1c78e65 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record SqlQueryParameter(string Name, object Value); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs new file mode 100644 index 0000000..754cb6f --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Globalization; + +namespace Apache.DataFusion.TableProviders.Sqlite.Sql; + +internal sealed record SqlSelectStatement( + string Projection, + string Source, + IReadOnlyList Predicates, + ulong? Limit) +{ + public string ToSql() + { + List clauses = + [ + $"SELECT {Projection}", + $"FROM ({Source}) AS df_source", + ]; + + if (Predicates.Count > 0) + { + clauses.Add($"WHERE {string.Join(" AND ", Predicates)}"); + } + + if (Limit.HasValue) + { + clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + return string.Join(" ", clauses); + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs new file mode 100644 index 0000000..3851ed6 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.Sqlite.Sql; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class SqliteArrowArrayStream : IArrowArrayStream +{ + private readonly Schema schema; + private readonly ColumnPlan[] columns; + private readonly int batchSize; + private SqliteConnection? connection; + private DbCommand? command; + private DbDataReader? reader; + private bool finished; + + public SqliteArrowArrayStream( + string connectionString, + string query, + Schema schema, + ColumnPlan[] columns, + int batchSize, + IReadOnlyList? parameters = null) + { + this.schema = schema; + this.columns = columns; + this.batchSize = batchSize; + + try + { + connection = new SqliteConnection(connectionString); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + } + catch + { + Dispose(); + throw; + } + } + + public Schema Schema => schema; + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + if (finished || reader is null) + { + return new ValueTask((RecordBatch?)null); + } + + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); + int rowCount = 0; + while (rowCount < batchSize && reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + foreach (ColumnAppender appender in appenders) + { + appender.Append(reader); + } + + rowCount++; + } + + if (rowCount == 0) + { + finished = true; + return new ValueTask((RecordBatch?)null); + } + + IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray(); + RecordBatch batch = new(schema, arrays, rowCount); + return new ValueTask(batch); + } + + public void Dispose() + { + reader?.Dispose(); + command?.Dispose(); + connection?.Dispose(); + reader = null; + command = null; + connection = null; + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs new file mode 100644 index 0000000..9f7eb32 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteDatabaseOptions +{ + public required string ConnectionString { get; init; } + + public bool IncludeViews { get; init; } = true; + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs new file mode 100644 index 0000000..2e749b0 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.DataFusion.TableProviders.Sqlite.Sql; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class SqliteDialect : SqlDialect +{ + public static SqliteDialect Instance { get; } = new(); + + private SqliteDialect() + { + } + + public override string QuoteIdentifier(string identifier) => + $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs new file mode 100644 index 0000000..9381fcf --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data; +using System.Data.Common; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.DataFusion.TableProviders.Sqlite.Sql; +using Microsoft.Data.Sqlite; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteStreamingTableProvider : StreamingTableProvider +{ + private readonly string connectionString; + private readonly string sourceSql; + private readonly int batchSize; + private readonly ColumnPlan[] columns; + private readonly SqlQueryBuilder queryBuilder = new(SqliteDialect.Instance); + + public SqliteStreamingTableProvider(SqliteTableOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); + } + + if (options.BatchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); + } + + connectionString = options.ConnectionString; + sourceSql = BuildSourceSql(options); + batchSize = options.BatchSize; + columns = FetchColumns(connectionString, sourceSql); + Schema = BuildSchema(columns); + } + + public override Schema Schema { get; } + + public override bool SupportsPushdown => true; + + public override IArrowArrayStream Scan() => + new SqliteArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + + public override IArrowArrayStream Scan(StreamingTableScanRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request); + ColumnPlan[] projectedColumns = ProjectColumns(request); + Schema projectedSchema = BuildSchema(projectedColumns); + return new SqliteArrowArrayStream( + connectionString, + pushedQuery.Sql, + projectedSchema, + projectedColumns, + batchSize, + pushedQuery.Parameters); + } + + private static string BuildSourceSql(SqliteTableOptions options) + { + if (string.IsNullOrWhiteSpace(options.TableName)) + { + throw new ArgumentException("Table name must be provided.", nameof(options)); + } + + return $"SELECT * FROM {SqliteDialect.Instance.QuoteIdentifier(options.TableName)}"; + } + + private static ColumnPlan[] FetchColumns(string connectionString, string query) + { + using SqliteConnection connection = new(connectionString); + connection.Open(); + + using DbCommand command = connection.CreateCommand(); + command.CommandText = query; + + using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo); + return reader.GetColumnSchema() + .Select((column, ordinal) => ColumnPlan.From(column, ordinal)) + .ToArray(); + } + + private static Schema BuildSchema(IEnumerable columns) + { + Schema.Builder builder = new(); + foreach (ColumnPlan column in columns) + { + builder.Field(field => field + .Name(column.Name) + .DataType(column.DataType) + .Nullable(column.Nullable)); + } + + return builder.Build(); + } + + private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request) + { + if (!request.HasProjection) + { + return columns; + } + + Dictionary columnLookup = columns.ToDictionary( + static column => column.Name, + StringComparer.OrdinalIgnoreCase); + return request.Projection + .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column) + ? column with { Ordinal = ordinal } + : throw new InvalidOperationException($"SQLite table provider does not contain column '{name}'.")) + .ToArray(); + } +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs new file mode 100644 index 0000000..28773a3 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.DataFusion.TableProviders.Sqlite; + +public sealed class SqliteTableOptions +{ + public required string ConnectionString { get; init; } + + public required string TableName { get; init; } + + public int BatchSize { get; init; } = 1024; +} diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs new file mode 100644 index 0000000..ff4a586 --- /dev/null +++ b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Data.Common; +using Apache.Arrow; + +namespace Apache.DataFusion.TableProviders.Sqlite; + +internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal) +{ + private readonly StringArray.Builder builder = new(); + + public override void Append(DbDataReader reader) + { + if (reader.IsDBNull(Ordinal)) + { + builder.AppendNull(); + return; + } + + builder.Append(FormatValue(reader.GetValue(Ordinal))); + } + + public override IArrowArray Build() => builder.Build(); +} diff --git a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs index 0a32745..2f7a639 100644 --- a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs +++ b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs @@ -19,8 +19,6 @@ using Apache.Arrow.Ipc; using Apache.Arrow.Types; -using Apache.DataFusion; - namespace Apache.DataFusion.Tests; public class StreamingTableProviderTests