diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..151fa6f
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,8 @@
+root = true
+
+[*.{cs,vb}]
+# Keep C# using directives in one block when the IDE organizes imports.
+dotnet_sort_system_directives_first = true
+dotnet_separate_import_directive_groups = false
+dotnet_diagnostic.IDE0055.severity = warning
+dotnet_diagnostic.IDE0005.severity = warning
diff --git a/README.md b/README.md
index 5e57d54..6879ae9 100644
--- a/README.md
+++ b/README.md
@@ -134,7 +134,10 @@ Each scenario is an independent, runnable console project under `examples/`:
| `Apache.DataFusion.Sample.Streaming` | Stream Arrow batches and read typed columns |
| `Apache.DataFusion.Sample.SessionConfig` | Configure the session builder + observability |
| `Apache.DataFusion.Sample.Excel` | Read an `.xlsx` spreadsheet and run a SQL query |
+| `Apache.DataFusion.Sample.FederatedQuery` | Run one SQL query across SQLite and CSV sources |
| `Apache.DataFusion.Sample.MySql` | Query a MySQL table through a managed streaming provider |
+| `Apache.DataFusion.Sample.PostgreSql` | Query a PostgreSQL table through a managed streaming provider |
+| `Apache.DataFusion.Sample.Sqlite` | Query a SQLite table through a managed streaming provider |
| `Apache.DataFusion.Sample.MongoDB` | Query a MongoDB collection through a managed streaming provider |
| `Apache.DataFusion.Sample.ClickHouse` | Query a ClickHouse table through the official `ClickHouse.Driver` |
@@ -162,7 +165,7 @@ dotnet run --project examples/Apache.DataFusion.Sample.Sql
`StreamingTableProvider`, a lazy callback-based provider whose `Scan` is invoked by the
engine and streamed in over the Arrow C Data Interface (connect any managed source, e.g.
ADO.NET, with no native database driver). Managed provider packages currently include
- MySQL, MongoDB, and ClickHouse.
+ MySQL, PostgreSQL, SQLite, MongoDB, and ClickHouse.
- **Scalar UDF**: zero-argument `Int64` managed callbacks.
- **Typed exceptions**: `DataFusionException` (base), plus `DataFusionPlanException`,
`DataFusionExecutionException`, `DataFusionIoException`,
@@ -222,44 +225,77 @@ must return a new, independently consumable stream, which the engine disposes on
### Managed database providers
Managed database provider packages are built on `StreamingTableProvider`. They keep database
-drivers on the C# side and push simple projection, filter, and limit requests down to the
-source query when possible.
+drivers on the C# side, register discovered tables and collections in the current
+`SessionContext`, and push simple projection, filter, and limit requests down when possible.
| Provider package | Extension method | Driver |
| --- | --- | --- |
| `Apache.DataFusion.TableProviders.MySql` | `RegisterMySql` | `MySqlConnector` |
+| `Apache.DataFusion.TableProviders.PostgreSql` | `RegisterPostgreSql` | `Npgsql` |
+| `Apache.DataFusion.TableProviders.Sqlite` | `RegisterSqlite` | `Microsoft.Data.Sqlite` |
| `Apache.DataFusion.TableProviders.MongoDB` | `RegisterMongoDb` | `MongoDB.Driver` |
| `Apache.DataFusion.TableProviders.ClickHouse` | `RegisterClickHouse` | official `ClickHouse.Driver` |
```csharp
using Apache.DataFusion;
-using Apache.DataFusion.TableProviders.ClickHouse;
+using Apache.DataFusion.TableProviders.PostgreSql;
using SessionContext context = new();
-context.RegisterClickHouse("audit_log", new ClickHouseTableOptions
-{
- ConnectionString = "Host=localhost;Port=8123;Username=default;Password=;Database=default",
- Query = """SELECT * FROM audit_log""",
-});
+context.RegisterPostgreSql("Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample");
+
+using DataFrame df = context.Sql("""
+ SELECT *
+ FROM datafusion_orders
+ LIMIT 10
+ """);
+df.Show();
+```
+
+SQL database providers register tables and views from the selected database in one call:
+
+```csharp
+using Apache.DataFusion;
+using Apache.DataFusion.TableProviders.MySql;
+
+using SessionContext context = new();
+context.RegisterMySql("Server=localhost;User ID=root;Password=pass;Database=datafusion_sample");
+
+using DataFrame df = context.Sql("""
+ SELECT *
+ FROM datafusion_orders
+ LIMIT 10
+ """);
+df.Show();
+```
+
+SQLite uses the same registration shape for a local database file:
+
+```csharp
+using Apache.DataFusion;
+using Apache.DataFusion.TableProviders.Sqlite;
+
+using SessionContext context = new();
+context.RegisterSqlite("Data Source=datafusion_sample.sqlite");
using DataFrame df = context.Sql("""
SELECT *
- FROM audit_log
+ FROM datafusion_orders
LIMIT 10
""");
df.Show();
```
-The registered DataFusion table name is the first argument to the extension method. The
-provider's `Query` points at the remote database table or subquery. For example, if you
-register `audit_log`, query `FROM audit_log` in DataFusion SQL; if you register `orders`,
-query `FROM orders`.
+Database registration uses each discovered table or collection name as the DataFusion table
+name by default. Single-table registration overloads are also available when you only want
+to expose one table.
The sample projects use environment variables for connection strings:
| Sample | Environment variable |
| --- | --- |
| `Apache.DataFusion.Sample.MySql` | `DATAFUSION_MYSQL_CONNECTION` |
+| `Apache.DataFusion.Sample.PostgreSql` | `DATAFUSION_POSTGRESQL_CONNECTION` |
+| `Apache.DataFusion.Sample.Sqlite` | `DATAFUSION_SQLITE_CONNECTION` |
| `Apache.DataFusion.Sample.MongoDB` | `DATAFUSION_MONGODB_CONNECTION` |
| `Apache.DataFusion.Sample.ClickHouse` | `DATAFUSION_CLICKHOUSE_CONNECTION` |
diff --git a/datafusion-csharp.slnx b/datafusion-csharp.slnx
index 8a5a694..ff9e284 100644
--- a/datafusion-csharp.slnx
+++ b/datafusion-csharp.slnx
@@ -3,12 +3,17 @@
+
+
+
+
+
diff --git a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs
index b6a04c1..02e346b 100644
--- a/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs
+++ b/examples/Apache.DataFusion.Sample.ClickHouse/Program.cs
@@ -52,16 +52,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES
}
using SessionContext context = new();
-context.RegisterClickHouse("orders", new ClickHouseTableOptions
-{
- ConnectionString = connectionString,
- Query = """SELECT id, customer, total FROM datafusion_orders""",
-});
+context.RegisterClickHouse(connectionString);
Console.WriteLine("Customer spend from a ClickHouse-backed streaming table:");
using DataFrame df = context.Sql("""
SELECT customer, SUM(total) AS spend
- FROM orders
+ FROM datafusion_orders
WHERE total >= 10
GROUP BY customer
ORDER BY customer
diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj
new file mode 100644
index 0000000..6922de3
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Apache.DataFusion.Sample.FederatedQuery.csproj
@@ -0,0 +1,19 @@
+
+
+
+ Exe
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs
new file mode 100644
index 0000000..60f8b8a
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.FederatedQuery/Program.cs
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.DataFusion;
+using Apache.DataFusion.TableProviders.Sqlite;
+using Microsoft.Data.Sqlite;
+
+string sqliteConnectionString = CreateSqliteOrders();
+string customerRegionsPath = WriteCustomerRegionsCsv();
+
+using SessionContext context = new();
+context.RegisterSqlite(sqliteConnectionString);
+context.RegisterCsv("customer_regions", customerRegionsPath, new CsvReadOptions { HasHeader = true });
+
+Console.WriteLine("Federated query joining SQLite orders with CSV customer regions:");
+using DataFrame df = context.Sql("""
+ SELECT r.region, o.customer, SUM(o.total) AS spend
+ FROM datafusion_orders AS o
+ INNER JOIN customer_regions AS r
+ ON o.customer = r.customer
+ GROUP BY r.region, o.customer
+ ORDER BY r.region, o.customer
+ """);
+df.Show();
+
+static string CreateSqliteOrders()
+{
+ string directory = WorkDirectory();
+ string databasePath = Path.Combine(directory, "orders.sqlite");
+ string connectionString = new SqliteConnectionStringBuilder
+ {
+ DataSource = databasePath,
+ }.ToString();
+
+ using SqliteConnection connection = new(connectionString);
+ connection.Open();
+ using SqliteCommand command = connection.CreateCommand();
+ command.CommandText = """
+ CREATE TABLE IF NOT EXISTS datafusion_orders (
+ id INTEGER PRIMARY KEY,
+ customer TEXT NOT NULL,
+ total REAL NOT NULL
+ );
+
+ DELETE FROM datafusion_orders;
+
+ INSERT INTO datafusion_orders (id, customer, total) VALUES
+ (1, 'alice', 19.99),
+ (2, 'bob', 7.50),
+ (3, 'alice', 100.00),
+ (4, 'carol', 42.25);
+ """;
+ command.ExecuteNonQuery();
+
+ return connectionString;
+}
+
+static string WriteCustomerRegionsCsv()
+{
+ string path = Path.Combine(WorkDirectory(), "customer_regions.csv");
+ File.WriteAllText(
+ path,
+ """
+ customer,region
+ alice,west
+ bob,east
+ carol,central
+ """);
+ return path;
+}
+
+static string WorkDirectory()
+{
+ string directory = Path.Combine(Path.GetTempPath(), "datafusion-csharp-examples", "federated-query");
+ Directory.CreateDirectory(directory);
+ return directory;
+}
diff --git a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs
index 77e2d23..57d434a 100644
--- a/examples/Apache.DataFusion.Sample.MongoDB/Program.cs
+++ b/examples/Apache.DataFusion.Sample.MongoDB/Program.cs
@@ -68,11 +68,10 @@
]);
using SessionContext context = new();
-context.RegisterMongoDb("orders", new MongoDbTableOptions
+context.RegisterMongoDb(new MongoDbDatabaseOptions
{
ConnectionString = connectionString,
DatabaseName = databaseName,
- CollectionName = collectionName,
});
Console.WriteLine("Customer spend from a MongoDB-backed streaming table:");
diff --git a/examples/Apache.DataFusion.Sample.MySql/Program.cs b/examples/Apache.DataFusion.Sample.MySql/Program.cs
index 0a2ce31..0a0dfd4 100644
--- a/examples/Apache.DataFusion.Sample.MySql/Program.cs
+++ b/examples/Apache.DataFusion.Sample.MySql/Program.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using MySqlConnector;
using Apache.DataFusion;
using Apache.DataFusion.TableProviders.MySql;
+using MySqlConnector;
string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_MYSQL_CONNECTION");
if (string.IsNullOrWhiteSpace(connectionString))
@@ -51,16 +51,12 @@ INSERT INTO datafusion_orders (id, customer, total) VALUES
}
using SessionContext context = new();
-context.RegisterMySql("orders", new MySqlTableOptions
-{
- ConnectionString = connectionString,
- Query = """SELECT id, customer, total FROM datafusion_orders""",
-});
+context.RegisterMySql(connectionString);
Console.WriteLine("Customer spend from a MySQL-backed streaming table:");
using DataFrame df = context.Sql("""
SELECT customer, SUM(total) AS spend
- FROM orders
+ FROM datafusion_orders
GROUP BY customer
ORDER BY customer
""");
diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj
new file mode 100644
index 0000000..1b56c28
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.PostgreSql/Apache.DataFusion.Sample.PostgreSql.csproj
@@ -0,0 +1,19 @@
+
+
+
+ Exe
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs
new file mode 100644
index 0000000..1441ca8
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.PostgreSql/Program.cs
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.DataFusion;
+using Apache.DataFusion.TableProviders.PostgreSql;
+using Npgsql;
+
+string? connectionString = Environment.GetEnvironmentVariable("DATAFUSION_POSTGRESQL_CONNECTION");
+if (string.IsNullOrWhiteSpace(connectionString))
+{
+ Console.WriteLine("Set DATAFUSION_POSTGRESQL_CONNECTION to run this sample.");
+ Console.WriteLine("Example:");
+ Console.WriteLine(" Host=localhost;Username=postgres;Password=pass;Database=datafusion_sample");
+ return;
+}
+
+using (NpgsqlConnection connection = new(connectionString))
+{
+ connection.Open();
+ using NpgsqlCommand command = connection.CreateCommand();
+ command.CommandText = """
+ CREATE TABLE IF NOT EXISTS datafusion_orders (
+ id INTEGER PRIMARY KEY,
+ customer VARCHAR(64) NOT NULL,
+ total DOUBLE PRECISION NOT NULL
+ );
+
+ DELETE FROM datafusion_orders;
+
+ INSERT INTO datafusion_orders (id, customer, total) VALUES
+ (1, 'alice', 19.99),
+ (2, 'bob', 7.50),
+ (3, 'alice', 100.00),
+ (4, 'carol', 42.25);
+ """;
+ command.ExecuteNonQuery();
+}
+
+using SessionContext context = new();
+context.RegisterPostgreSql(connectionString);
+
+Console.WriteLine("Customer spend from a PostgreSQL-backed streaming table:");
+using DataFrame df = context.Sql("""
+ SELECT customer, SUM(total) AS spend
+ FROM datafusion_orders
+ GROUP BY customer
+ ORDER BY customer
+ """);
+df.Show();
diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj
new file mode 100644
index 0000000..0df4e4b
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.Sqlite/Apache.DataFusion.Sample.Sqlite.csproj
@@ -0,0 +1,15 @@
+
+
+
+ Exe
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
diff --git a/examples/Apache.DataFusion.Sample.Sqlite/Program.cs b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs
new file mode 100644
index 0000000..3a7b5cf
--- /dev/null
+++ b/examples/Apache.DataFusion.Sample.Sqlite/Program.cs
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.DataFusion;
+using Apache.DataFusion.TableProviders.Sqlite;
+using Microsoft.Data.Sqlite;
+
+string connectionString = Environment.GetEnvironmentVariable("DATAFUSION_SQLITE_CONNECTION")
+ ?? new SqliteConnectionStringBuilder
+ {
+ DataSource = Path.Combine(Path.GetTempPath(), "datafusion_sample.sqlite"),
+ }.ToString();
+
+using (SqliteConnection connection = new(connectionString))
+{
+ connection.Open();
+ using SqliteCommand command = connection.CreateCommand();
+ command.CommandText = """
+ CREATE TABLE IF NOT EXISTS datafusion_orders (
+ id INTEGER PRIMARY KEY,
+ customer TEXT NOT NULL,
+ total REAL NOT NULL
+ );
+
+ DELETE FROM datafusion_orders;
+
+ INSERT INTO datafusion_orders (id, customer, total) VALUES
+ (1, 'alice', 19.99),
+ (2, 'bob', 7.50),
+ (3, 'alice', 100.00),
+ (4, 'carol', 42.25);
+ """;
+ command.ExecuteNonQuery();
+}
+
+using SessionContext context = new();
+context.RegisterSqlite(connectionString);
+
+Console.WriteLine("Customer spend from a SQLite-backed streaming table:");
+using DataFrame df = context.Sql("""
+ SELECT customer, SUM(total) AS spend
+ FROM datafusion_orders
+ GROUP BY customer
+ ORDER BY customer
+ """);
+df.Show();
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj
index 8ec5485..35e0d23 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Apache.DataFusion.TableProviders.ClickHouse.csproj
@@ -1,4 +1,4 @@
-
+
net10.0
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs
index 6aa8010..3d6832d 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs
index 39c201b..8750e39 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs
@@ -17,11 +17,9 @@
using System.Data;
using System.Data.Common;
-
using Apache.Arrow;
using Apache.Arrow.Ipc;
using Apache.DataFusion.TableProviders.ClickHouse.Sql;
-
using ClickHouse.Driver.ADO;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs
new file mode 100644
index 0000000..4440322
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.ClickHouse;
+
+public sealed class ClickHouseDatabaseOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public string? DatabaseName { get; init; }
+
+ public bool IncludeViews { get; init; } = true;
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs
index 0fbea39..6a66879 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs
@@ -17,11 +17,9 @@
using System.Data;
using System.Data.Common;
-
using Apache.Arrow;
using Apache.Arrow.Ipc;
using Apache.DataFusion.TableProviders.ClickHouse.Sql;
-
using ClickHouse.Driver.ADO;
namespace Apache.DataFusion.TableProviders.ClickHouse;
@@ -29,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.ClickHouse;
public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider
{
private readonly string connectionString;
- private readonly string query;
+ private readonly string sourceSql;
private readonly int batchSize;
private readonly ColumnPlan[] columns;
private readonly ClickHouseQueryBuilder queryBuilder = new(ClickHouseDialect.Instance);
@@ -42,20 +40,15 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options)
throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
}
- if (string.IsNullOrWhiteSpace(options.Query))
- {
- throw new ArgumentException("Query cannot be null or whitespace.", nameof(options));
- }
-
if (options.BatchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
}
connectionString = options.ConnectionString;
- query = options.Query;
+ sourceSql = BuildSourceSql(options);
batchSize = options.BatchSize;
- columns = FetchColumns(connectionString, query);
+ columns = FetchColumns(connectionString, sourceSql);
Schema = BuildSchema(columns);
}
@@ -64,13 +57,13 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options)
public override bool SupportsPushdown => true;
public override IArrowArrayStream Scan() =>
- new ClickHouseArrowArrayStream(connectionString, query, Schema, columns, batchSize);
+ new ClickHouseArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize);
public override IArrowArrayStream Scan(StreamingTableScanRequest request)
{
ArgumentNullException.ThrowIfNull(request);
- PushedQuery pushedQuery = queryBuilder.Build(query, request);
+ PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request);
ColumnPlan[] projectedColumns = ProjectColumns(request);
Schema projectedSchema = BuildSchema(projectedColumns);
return new ClickHouseArrowArrayStream(
@@ -82,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request)
pushedQuery.Parameters);
}
+ private static string BuildSourceSql(ClickHouseTableOptions options)
+ {
+ if (string.IsNullOrWhiteSpace(options.TableName))
+ {
+ throw new ArgumentException("Table name must be provided.", nameof(options));
+ }
+
+ return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}";
+ }
+
+ private static string QuoteQualifiedTableName(string? databaseName, string tableName)
+ {
+ string quotedTable = ClickHouseDialect.Instance.QuoteIdentifier(tableName);
+ return string.IsNullOrWhiteSpace(databaseName)
+ ? quotedTable
+ : $"{ClickHouseDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}";
+ }
+
private static ColumnPlan[] FetchColumns(string connectionString, string query)
{
using ClickHouseConnection connection = new(connectionString);
@@ -117,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request)
return columns;
}
- Dictionary byName = columns.ToDictionary(
+ Dictionary columnLookup = columns.ToDictionary(
static column => column.Name,
StringComparer.OrdinalIgnoreCase);
return request.Projection
- .Select(name => byName.TryGetValue(name, out ColumnPlan? column)
- ? column
+ .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column)
+ ? column with { Ordinal = ordinal }
: throw new InvalidOperationException($"ClickHouse table provider does not contain column '{name}'."))
.ToArray();
}
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs
index 2114ded..c1471ab 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs
@@ -21,7 +21,9 @@ public sealed class ClickHouseTableOptions
{
public required string ConnectionString { get; init; }
- public required string Query { get; init; }
+ public string? DatabaseName { get; init; }
+
+ public required string TableName { get; init; }
public int BatchSize { get; init; } = 1024;
}
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs
index 746d716..8ddaa2b 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs
index 3c4e284..bc8d475 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ColumnPlan.cs
@@ -16,7 +16,6 @@
// under the License.
using System.Data.Common;
-
using Apache.Arrow.Types;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs
index 5769f28..550a549 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs
index bc1f71c..a80f96e 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs
index 58f2d32..c47ea6e 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs
index 277f9be..a100bf2 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs
@@ -17,7 +17,6 @@
using System.Data.Common;
using System.Globalization;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs
index 1afe8ce..740e39b 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs
@@ -15,10 +15,47 @@
// specific language governing permissions and limitations
// under the License.
+using System.Data.Common;
+using ClickHouse.Driver.ADO;
+
namespace Apache.DataFusion.TableProviders.ClickHouse;
public static class SessionContextExtensions
{
+ public static void RegisterClickHouse(this SessionContext context, string connectionString)
+ {
+ context.RegisterClickHouse(new ClickHouseDatabaseOptions
+ {
+ ConnectionString = connectionString,
+ });
+ }
+
+ public static void RegisterClickHouse(this SessionContext context, ClickHouseDatabaseOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews))
+ {
+ context.RegisterStreamingTable(table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions
+ {
+ ConnectionString = options.ConnectionString,
+ DatabaseName = table.DatabaseName,
+ TableName = table.TableName,
+ BatchSize = options.BatchSize,
+ }));
+ }
+ }
+
public static void RegisterClickHouse(this SessionContext context, string name, ClickHouseTableOptions options)
{
ArgumentNullException.ThrowIfNull(context);
@@ -27,12 +64,48 @@ public static void RegisterClickHouse(this SessionContext context, string name,
context.RegisterStreamingTable(name, new ClickHouseStreamingTableProvider(options));
}
- public static void RegisterClickHouse(this SessionContext context, string name, string connectionString, string query)
+ public static void RegisterClickHouse(this SessionContext context, string name, string connectionString)
{
context.RegisterClickHouse(name, new ClickHouseTableOptions
{
ConnectionString = connectionString,
- Query = query,
+ TableName = name,
});
}
+
+ private static IEnumerable FetchTables(
+ string connectionString,
+ string? databaseName,
+ bool includeViews)
+ {
+ using ClickHouseConnection connection = new(connectionString);
+ connection.Open();
+
+ using DbCommand command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT database, name
+ FROM system.tables
+ WHERE database = if({database:String} = '', currentDatabase(), {database:String})
+ AND ({include_views:Bool} OR engine NOT IN ('View', 'MaterializedView'))
+ ORDER BY name
+ """;
+
+ DbParameter databaseParameter = command.CreateParameter();
+ databaseParameter.ParameterName = "database";
+ databaseParameter.Value = databaseName ?? string.Empty;
+ command.Parameters.Add(databaseParameter);
+
+ DbParameter includeViewsParameter = command.CreateParameter();
+ includeViewsParameter.ParameterName = "include_views";
+ includeViewsParameter.Value = includeViews;
+ command.Parameters.Add(includeViewsParameter);
+
+ using DbDataReader reader = command.ExecuteReader();
+ while (reader.Read())
+ {
+ yield return new ClickHouseTable(reader.GetString(0), reader.GetString(1));
+ }
+ }
+
+ private sealed record ClickHouseTable(string DatabaseName, string TableName);
}
diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs
index 7870b79..e2d3092 100644
--- a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs
@@ -16,7 +16,6 @@
// under the License.
using System.Data.Common;
-
using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.ClickHouse;
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs
index 116d64f..a4938d2 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbArrowArrayStream.cs
@@ -17,7 +17,6 @@
using Apache.Arrow;
using Apache.Arrow.Ipc;
-
using MongoDB.Bson;
using MongoDB.Driver;
@@ -73,7 +72,7 @@ public MongoDbArrowArrayStream(
return new ValueTask((RecordBatch?)null);
}
- MongoDbColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray();
+ MongoDbColumnAppender[] appenders = [.. columns.Select(static column => column.CreateAppender())];
int rowCount = 0;
while (rowCount < batchSize)
{
@@ -97,7 +96,7 @@ public MongoDbArrowArrayStream(
return new ValueTask((RecordBatch?)null);
}
- IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray();
+ IArrowArray[] arrays = [.. appenders.Select(static appender => appender.Build())];
RecordBatch batch = new(schema, arrays, rowCount);
return new ValueTask(batch);
}
@@ -123,7 +122,7 @@ private bool TryReadNextDocument(out BsonDocument document)
currentBatch = null;
if (!cursor.MoveNext())
{
- document = new BsonDocument();
+ document = [];
return false;
}
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs
index 74bc8a7..39ff62a 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnAppender.cs
@@ -16,9 +16,7 @@
// under the License.
using System.Globalization;
-
using Apache.Arrow;
-
using MongoDB.Bson;
namespace Apache.DataFusion.TableProviders.MongoDB;
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs
index 974d8ff..195c886 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbColumnPlan.cs
@@ -16,7 +16,6 @@
// under the License.
using Apache.Arrow.Types;
-
using MongoDB.Bson;
namespace Apache.DataFusion.TableProviders.MongoDB;
@@ -53,13 +52,12 @@ public static MongoDbColumnPlan[] Infer(IReadOnlyList documents)
}
}
- return columns
+ return [.. columns
.Select(column => new MongoDbColumnPlan(
column.Key,
GetArrowType(column.Value.Kind),
column.Value.Nullable || column.Value.ObservedCount < documents.Count,
- column.Value.Kind))
- .ToArray();
+ column.Value.Kind))];
}
private static IArrowType GetArrowType(MongoDbColumnKind kind) => kind switch
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs
new file mode 100644
index 0000000..64250bf
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.MongoDB;
+
+public sealed class MongoDbDatabaseOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public string? DatabaseName { get; init; }
+
+ public int BatchSize { get; init; } = 1024;
+
+ public int SchemaInferenceLimit { get; init; } = 100;
+}
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs
index 7111487..0d35989 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbQueryBuilder.cs
@@ -16,7 +16,6 @@
// under the License.
using System.Globalization;
-
using MongoDB.Bson;
using MongoDB.Driver;
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs
index 9576237..e4beaf7 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs
@@ -17,7 +17,6 @@
using Apache.Arrow;
using Apache.Arrow.Ipc;
-
using MongoDB.Bson;
using MongoDB.Driver;
@@ -94,14 +93,13 @@ private MongoDbColumnPlan[] ProjectColumns(StreamingTableScanRequest request)
return columns;
}
- Dictionary byName = columns.ToDictionary(
+ Dictionary columnLookup = columns.ToDictionary(
static column => column.Name,
StringComparer.Ordinal);
- return request.Projection
- .Select(name => byName.TryGetValue(name, out MongoDbColumnPlan? column)
+ return [.. request.Projection
+ .Select(name => columnLookup.TryGetValue(name, out MongoDbColumnPlan? column)
? column
- : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'."))
- .ToArray();
+ : throw new InvalidOperationException($"MongoDB table provider does not contain field '{name}'."))];
}
private static MongoDbColumnPlan[] InferColumns(IMongoCollection collection, int schemaInferenceLimit)
diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs
index f023293..643894f 100644
--- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs
+++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs
@@ -15,10 +15,55 @@
// specific language governing permissions and limitations
// under the License.
+using MongoDB.Driver;
+
namespace Apache.DataFusion.TableProviders.MongoDB;
public static class SessionContextExtensions
{
+ public static void RegisterMongoDb(this SessionContext context, string connectionString)
+ {
+ context.RegisterMongoDb(new MongoDbDatabaseOptions
+ {
+ ConnectionString = connectionString,
+ });
+ }
+
+ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ if (options.SchemaInferenceLimit <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.SchemaInferenceLimit, "Schema inference limit must be greater than zero.");
+ }
+
+ string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName);
+ MongoClient client = new(options.ConnectionString);
+ IMongoDatabase database = client.GetDatabase(databaseName);
+ foreach (string collectionName in database.ListCollectionNames().ToEnumerable())
+ {
+ context.RegisterStreamingTable(collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions
+ {
+ ConnectionString = options.ConnectionString,
+ DatabaseName = databaseName,
+ CollectionName = collectionName,
+ BatchSize = options.BatchSize,
+ SchemaInferenceLimit = options.SchemaInferenceLimit,
+ }));
+ }
+ }
+
public static void RegisterMongoDb(this SessionContext context, string name, MongoDbTableOptions options)
{
ArgumentNullException.ThrowIfNull(context);
@@ -27,6 +72,17 @@ public static void RegisterMongoDb(this SessionContext context, string name, Mon
context.RegisterStreamingTable(name, new MongoDbStreamingTableProvider(options));
}
+ public static void RegisterMongoDb(this SessionContext context, string name, string connectionString)
+ {
+ string databaseName = ResolveDatabaseName(connectionString, null);
+ context.RegisterMongoDb(name, new MongoDbTableOptions
+ {
+ ConnectionString = connectionString,
+ DatabaseName = databaseName,
+ CollectionName = name,
+ });
+ }
+
public static void RegisterMongoDb(
this SessionContext context,
string name,
@@ -41,4 +97,20 @@ public static void RegisterMongoDb(
CollectionName = collectionName,
});
}
+
+ private static string ResolveDatabaseName(string connectionString, string? databaseName)
+ {
+ if (!string.IsNullOrWhiteSpace(databaseName))
+ {
+ return databaseName;
+ }
+
+ string? connectionDatabaseName = MongoUrl.Create(connectionString).DatabaseName;
+ if (!string.IsNullOrWhiteSpace(connectionDatabaseName))
+ {
+ return connectionDatabaseName;
+ }
+
+ throw new ArgumentException("Database name must be provided in MongoDB options or connection string.", nameof(connectionString));
+ }
}
diff --git a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs
index 64ebca9..b66fa20 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs
index 82f2bc8..114b66f 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
-
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs
index 12fd3d7..e48a8e3 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnPlan.cs
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow.Types;
using System.Data.Common;
+using Apache.Arrow.Types;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs
index 87fdd11..2c63817 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs
index 69d992e..5f23fc3 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs
index 6c66f83..d3e4675 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs
index d85c941..d2e8d29 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-using Apache.Arrow;
using System.Data.Common;
using System.Globalization;
+using Apache.Arrow;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs
index 5ccd36c..2ce32a1 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs
@@ -17,11 +17,9 @@
using System.Data;
using System.Data.Common;
-
using Apache.Arrow;
using Apache.Arrow.Ipc;
using Apache.DataFusion.TableProviders.MySql.Sql;
-
using MySqlConnector;
namespace Apache.DataFusion.TableProviders.MySql;
diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs
new file mode 100644
index 0000000..6ff05bb
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.MySql;
+
+public sealed class MySqlDatabaseOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public string? DatabaseName { get; init; }
+
+ public bool IncludeViews { get; init; } = true;
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs
index 72f11b3..65c395d 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs
@@ -17,11 +17,9 @@
using System.Data;
using System.Data.Common;
-
using Apache.Arrow;
using Apache.Arrow.Ipc;
using Apache.DataFusion.TableProviders.MySql.Sql;
-
using MySqlConnector;
namespace Apache.DataFusion.TableProviders.MySql;
@@ -29,7 +27,7 @@ namespace Apache.DataFusion.TableProviders.MySql;
public sealed class MySqlStreamingTableProvider : StreamingTableProvider
{
private readonly string connectionString;
- private readonly string query;
+ private readonly string sourceSql;
private readonly int batchSize;
private readonly ColumnPlan[] columns;
private readonly SqlQueryBuilder queryBuilder = new(MySqlDialect.Instance);
@@ -42,20 +40,15 @@ public MySqlStreamingTableProvider(MySqlTableOptions options)
throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
}
- if (string.IsNullOrWhiteSpace(options.Query))
- {
- throw new ArgumentException("Query cannot be null or whitespace.", nameof(options));
- }
-
if (options.BatchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
}
connectionString = options.ConnectionString;
- query = options.Query;
+ sourceSql = BuildSourceSql(options);
batchSize = options.BatchSize;
- columns = FetchColumns(connectionString, query);
+ columns = FetchColumns(connectionString, sourceSql);
Schema = BuildSchema(columns);
}
@@ -64,13 +57,13 @@ public MySqlStreamingTableProvider(MySqlTableOptions options)
public override bool SupportsPushdown => true;
public override IArrowArrayStream Scan() =>
- new MySqlArrowArrayStream(connectionString, query, Schema, columns, batchSize);
+ new MySqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize);
public override IArrowArrayStream Scan(StreamingTableScanRequest request)
{
ArgumentNullException.ThrowIfNull(request);
- PushedQuery pushedQuery = queryBuilder.Build(query, request);
+ PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request);
ColumnPlan[] projectedColumns = ProjectColumns(request);
Schema projectedSchema = BuildSchema(projectedColumns);
return new MySqlArrowArrayStream(
@@ -82,6 +75,24 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request)
pushedQuery.Parameters);
}
+ private static string BuildSourceSql(MySqlTableOptions options)
+ {
+ if (string.IsNullOrWhiteSpace(options.TableName))
+ {
+ throw new ArgumentException("Table name must be provided.", nameof(options));
+ }
+
+ return $"SELECT * FROM {QuoteQualifiedTableName(options.DatabaseName, options.TableName)}";
+ }
+
+ private static string QuoteQualifiedTableName(string? databaseName, string tableName)
+ {
+ string quotedTable = MySqlDialect.Instance.QuoteIdentifier(tableName);
+ return string.IsNullOrWhiteSpace(databaseName)
+ ? quotedTable
+ : $"{MySqlDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}";
+ }
+
private static ColumnPlan[] FetchColumns(string connectionString, string query)
{
using MySqlConnection connection = new(connectionString);
@@ -117,12 +128,12 @@ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request)
return columns;
}
- Dictionary byName = columns.ToDictionary(
+ Dictionary columnLookup = columns.ToDictionary(
static column => column.Name,
StringComparer.OrdinalIgnoreCase);
return request.Projection
- .Select(name => byName.TryGetValue(name, out ColumnPlan? column)
- ? column
+ .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column)
+ ? column with { Ordinal = ordinal }
: throw new InvalidOperationException($"MySQL table provider does not contain column '{name}'."))
.ToArray();
}
diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs
index 48e4e9b..c91e4ac 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs
@@ -21,7 +21,9 @@ public sealed class MySqlTableOptions
{
public required string ConnectionString { get; init; }
- public required string Query { get; init; }
+ public string? DatabaseName { get; init; }
+
+ public required string TableName { get; init; }
public int BatchSize { get; init; } = 1024;
}
diff --git a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs
index 9f4694d..f7e87de 100644
--- a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs
+++ b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs
@@ -15,10 +15,46 @@
// specific language governing permissions and limitations
// under the License.
+using MySqlConnector;
+
namespace Apache.DataFusion.TableProviders.MySql;
public static class SessionContextExtensions
{
+ public static void RegisterMySql(this SessionContext context, string connectionString)
+ {
+ context.RegisterMySql(new MySqlDatabaseOptions
+ {
+ ConnectionString = connectionString,
+ });
+ }
+
+ public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews))
+ {
+ context.RegisterStreamingTable(table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions
+ {
+ ConnectionString = options.ConnectionString,
+ DatabaseName = table.DatabaseName,
+ TableName = table.TableName,
+ BatchSize = options.BatchSize,
+ }));
+ }
+ }
+
public static void RegisterMySql(this SessionContext context, string name, MySqlTableOptions options)
{
ArgumentNullException.ThrowIfNull(context);
@@ -27,12 +63,41 @@ public static void RegisterMySql(this SessionContext context, string name, MySql
context.RegisterStreamingTable(name, new MySqlStreamingTableProvider(options));
}
- public static void RegisterMySql(this SessionContext context, string name, string connectionString, string query)
+ public static void RegisterMySql(this SessionContext context, string name, string connectionString)
{
context.RegisterMySql(name, new MySqlTableOptions
{
ConnectionString = connectionString,
- Query = query,
+ TableName = name,
});
}
+
+ private static IEnumerable FetchTables(
+ string connectionString,
+ string? databaseName,
+ bool includeViews)
+ {
+ using MySqlConnection connection = new(connectionString);
+ connection.Open();
+
+ using MySqlCommand command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT table_schema, table_name
+ FROM information_schema.tables
+ WHERE table_schema = COALESCE(@database_name, DATABASE())
+ AND table_type IN ('BASE TABLE', 'VIEW')
+ AND (@include_views OR table_type = 'BASE TABLE')
+ ORDER BY table_name
+ """;
+ command.Parameters.AddWithValue("@database_name", string.IsNullOrWhiteSpace(databaseName) ? DBNull.Value : databaseName);
+ command.Parameters.AddWithValue("@include_views", includeViews);
+
+ using MySqlDataReader reader = command.ExecuteReader();
+ while (reader.Read())
+ {
+ yield return new MySqlTable(reader.GetString(0), reader.GetString(1));
+ }
+ }
+
+ private sealed record MySqlTable(string DatabaseName, string TableName);
}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj
new file mode 100644
index 0000000..6874099
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Apache.DataFusion.TableProviders.PostgreSql.csproj
@@ -0,0 +1,17 @@
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs
new file mode 100644
index 0000000..c7bc593
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly BooleanArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs
new file mode 100644
index 0000000..bba9402
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal abstract class ColumnAppender(int ordinal)
+{
+ protected int Ordinal { get; } = ordinal;
+
+ public abstract void Append(DbDataReader reader);
+
+ public abstract IArrowArray Build();
+
+ protected static string FormatValue(object value) =>
+ value switch
+ {
+ byte[] bytes => Convert.ToBase64String(bytes),
+ DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture),
+ DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture),
+ TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture),
+ DateTimeOffset dateTimeOffset => dateTimeOffset.ToString("O", CultureInfo.InvariantCulture),
+ IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture),
+ _ => value.ToString() ?? string.Empty,
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs
new file mode 100644
index 0000000..492f6eb
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnKind.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal enum ColumnKind
+{
+ Boolean,
+ Int32,
+ Int64,
+ Float,
+ Double,
+ String,
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs
new file mode 100644
index 0000000..90fadc0
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnPlan.cs
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using Apache.Arrow.Types;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed record ColumnPlan(
+ int Ordinal,
+ string Name,
+ IArrowType DataType,
+ bool Nullable,
+ ColumnKind Kind)
+{
+ public static ColumnPlan From(DbColumn column, int ordinal)
+ {
+ Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType;
+ ColumnKind kind = GetKind(type);
+ return new ColumnPlan(
+ ordinal,
+ column.ColumnName ?? $"column_{ordinal}",
+ GetArrowType(kind),
+ column.AllowDBNull ?? true,
+ kind);
+ }
+
+ public ColumnAppender CreateAppender() => Kind switch
+ {
+ ColumnKind.Boolean => new BooleanColumnAppender(Ordinal),
+ ColumnKind.Int32 => new Int32ColumnAppender(Ordinal),
+ ColumnKind.Int64 => new Int64ColumnAppender(Ordinal),
+ ColumnKind.Float => new FloatColumnAppender(Ordinal),
+ ColumnKind.Double => new DoubleColumnAppender(Ordinal),
+ _ => new StringColumnAppender(Ordinal),
+ };
+
+ private static ColumnKind GetKind(Type? type)
+ {
+ if (type == typeof(bool))
+ {
+ return ColumnKind.Boolean;
+ }
+
+ if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int))
+ {
+ return ColumnKind.Int32;
+ }
+
+ if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong))
+ {
+ return ColumnKind.Int64;
+ }
+
+ if (type == typeof(float))
+ {
+ return ColumnKind.Float;
+ }
+
+ if (type == typeof(double) || type == typeof(decimal))
+ {
+ return ColumnKind.Double;
+ }
+
+ return ColumnKind.String;
+ }
+
+ private static IArrowType GetArrowType(ColumnKind kind) => kind switch
+ {
+ ColumnKind.Boolean => BooleanType.Default,
+ ColumnKind.Int32 => Int32Type.Default,
+ ColumnKind.Int64 => Int64Type.Default,
+ ColumnKind.Float => FloatType.Default,
+ ColumnKind.Double => DoubleType.Default,
+ _ => StringType.Default,
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs
new file mode 100644
index 0000000..4366532
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly DoubleArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs
new file mode 100644
index 0000000..9ed65ce
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly FloatArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs
new file mode 100644
index 0000000..de9bad3
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly Int32Array.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs
new file mode 100644
index 0000000..2693520
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly Int64Array.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs
new file mode 100644
index 0000000..0142ccc
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data;
+using System.Data.Common;
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.DataFusion.TableProviders.PostgreSql.Sql;
+using Npgsql;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class PostgreSqlArrowArrayStream : IArrowArrayStream
+{
+ private readonly Schema schema;
+ private readonly ColumnPlan[] columns;
+ private readonly int batchSize;
+ private NpgsqlConnection? connection;
+ private DbCommand? command;
+ private DbDataReader? reader;
+ private bool finished;
+
+ public PostgreSqlArrowArrayStream(
+ string connectionString,
+ string query,
+ Schema schema,
+ ColumnPlan[] columns,
+ int batchSize,
+ IReadOnlyList? parameters = null)
+ {
+ this.schema = schema;
+ this.columns = columns;
+ this.batchSize = batchSize;
+
+ try
+ {
+ connection = new NpgsqlConnection(connectionString);
+ connection.Open();
+
+ command = connection.CreateCommand();
+ command.CommandText = query;
+ if (parameters is not null)
+ {
+ foreach (SqlQueryParameter parameter in parameters)
+ {
+ DbParameter dbParameter = command.CreateParameter();
+ dbParameter.ParameterName = parameter.Name;
+ dbParameter.Value = parameter.Value;
+ command.Parameters.Add(dbParameter);
+ }
+ }
+
+ reader = command.ExecuteReader(CommandBehavior.SequentialAccess);
+ }
+ catch
+ {
+ Dispose();
+ throw;
+ }
+ }
+
+ public Schema Schema => schema;
+
+ public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (finished || reader is null)
+ {
+ return new ValueTask((RecordBatch?)null);
+ }
+
+ ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray();
+ int rowCount = 0;
+ while (rowCount < batchSize && reader.Read())
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ foreach (ColumnAppender appender in appenders)
+ {
+ appender.Append(reader);
+ }
+
+ rowCount++;
+ }
+
+ if (rowCount == 0)
+ {
+ finished = true;
+ return new ValueTask((RecordBatch?)null);
+ }
+
+ IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray();
+ RecordBatch batch = new(schema, arrays, rowCount);
+ return new ValueTask(batch);
+ }
+
+ public void Dispose()
+ {
+ reader?.Dispose();
+ command?.Dispose();
+ connection?.Dispose();
+ reader = null;
+ command = null;
+ connection = null;
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs
new file mode 100644
index 0000000..aef7378
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+public sealed class PostgreSqlDatabaseOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public IReadOnlyCollection Schemas { get; init; } = ["public"];
+
+ public bool IncludeViews { get; init; } = true;
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs
new file mode 100644
index 0000000..9aa837e
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDialect.cs
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class PostgreSqlDialect : SqlDialect
+{
+ public static PostgreSqlDialect Instance { get; } = new();
+
+ private PostgreSqlDialect()
+ {
+ }
+
+ public override string QuoteIdentifier(string identifier) =>
+ $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\"";
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs
new file mode 100644
index 0000000..6671e8b
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data;
+using System.Data.Common;
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.DataFusion.TableProviders.PostgreSql.Sql;
+using Npgsql;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+public sealed class PostgreSqlStreamingTableProvider : StreamingTableProvider
+{
+ private readonly string connectionString;
+ private readonly string sourceSql;
+ private readonly int batchSize;
+ private readonly ColumnPlan[] columns;
+ private readonly SqlQueryBuilder queryBuilder = new(PostgreSqlDialect.Instance);
+
+ public PostgreSqlStreamingTableProvider(PostgreSqlTableOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ connectionString = options.ConnectionString;
+ sourceSql = BuildSourceSql(options);
+ batchSize = options.BatchSize;
+ columns = FetchColumns(connectionString, sourceSql);
+ Schema = BuildSchema(columns);
+ }
+
+ public override Schema Schema { get; }
+
+ public override bool SupportsPushdown => true;
+
+ public override IArrowArrayStream Scan() =>
+ new PostgreSqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize);
+
+ public override IArrowArrayStream Scan(StreamingTableScanRequest request)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+
+ PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request);
+ ColumnPlan[] projectedColumns = ProjectColumns(request);
+ Schema projectedSchema = BuildSchema(projectedColumns);
+ return new PostgreSqlArrowArrayStream(
+ connectionString,
+ pushedQuery.Sql,
+ projectedSchema,
+ projectedColumns,
+ batchSize,
+ pushedQuery.Parameters);
+ }
+
+ private static string BuildSourceSql(PostgreSqlTableOptions options)
+ {
+ if (string.IsNullOrWhiteSpace(options.TableName))
+ {
+ throw new ArgumentException("Table name must be provided.", nameof(options));
+ }
+
+ string schemaName = string.IsNullOrWhiteSpace(options.SchemaName) ? "public" : options.SchemaName;
+ return $"SELECT * FROM {QuoteQualifiedTableName(schemaName, options.TableName)}";
+ }
+
+ private static string QuoteQualifiedTableName(string schemaName, string tableName) =>
+ $"{PostgreSqlDialect.Instance.QuoteIdentifier(schemaName)}.{PostgreSqlDialect.Instance.QuoteIdentifier(tableName)}";
+
+ private static ColumnPlan[] FetchColumns(string connectionString, string query)
+ {
+ using NpgsqlConnection connection = new(connectionString);
+ connection.Open();
+
+ using DbCommand command = connection.CreateCommand();
+ command.CommandText = query;
+
+ using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo);
+ return reader.GetColumnSchema()
+ .Select((column, ordinal) => ColumnPlan.From(column, ordinal))
+ .ToArray();
+ }
+
+ private static Schema BuildSchema(IEnumerable columns)
+ {
+ Schema.Builder builder = new();
+ foreach (ColumnPlan column in columns)
+ {
+ builder.Field(field => field
+ .Name(column.Name)
+ .DataType(column.DataType)
+ .Nullable(column.Nullable));
+ }
+
+ return builder.Build();
+ }
+
+ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request)
+ {
+ if (!request.HasProjection)
+ {
+ return columns;
+ }
+
+ Dictionary columnLookup = columns.ToDictionary(
+ static column => column.Name,
+ StringComparer.OrdinalIgnoreCase);
+ return request.Projection
+ .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column)
+ ? column with { Ordinal = ordinal }
+ : throw new InvalidOperationException($"PostgreSQL table provider does not contain column '{name}'."))
+ .ToArray();
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs
new file mode 100644
index 0000000..15c82c8
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+public sealed class PostgreSqlTableOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public string SchemaName { get; init; } = "public";
+
+ public required string TableName { get; init; }
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs
new file mode 100644
index 0000000..6843e0d
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data;
+using System.Data.Common;
+using Npgsql;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+public static class SessionContextExtensions
+{
+ public static void RegisterPostgreSql(this SessionContext context, string connectionString)
+ {
+ context.RegisterPostgreSql(new PostgreSqlDatabaseOptions
+ {
+ ConnectionString = connectionString,
+ });
+ }
+
+ public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDatabaseOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ string[] schemas = options.Schemas
+ .Where(static schema => !string.IsNullOrWhiteSpace(schema))
+ .Distinct(StringComparer.OrdinalIgnoreCase)
+ .ToArray();
+ if (schemas.Length == 0)
+ {
+ throw new ArgumentException("At least one schema must be provided.", nameof(options));
+ }
+
+ foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews))
+ {
+ string registrationName = RegistrationName(table, schemas);
+ context.RegisterStreamingTable(registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions
+ {
+ ConnectionString = options.ConnectionString,
+ SchemaName = table.SchemaName,
+ TableName = table.TableName,
+ BatchSize = options.BatchSize,
+ }));
+ }
+ }
+
+ public static void RegisterPostgreSql(this SessionContext context, string name, PostgreSqlTableOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+
+ context.RegisterStreamingTable(name, new PostgreSqlStreamingTableProvider(options));
+ }
+
+ public static void RegisterPostgreSql(this SessionContext context, string name, string connectionString)
+ {
+ context.RegisterPostgreSql(name, new PostgreSqlTableOptions
+ {
+ ConnectionString = connectionString,
+ TableName = name,
+ });
+ }
+
+ private static IEnumerable FetchTables(
+ string connectionString,
+ IReadOnlyCollection schemas,
+ bool includeViews)
+ {
+ using NpgsqlConnection connection = new(connectionString);
+ connection.Open();
+
+ using DbCommand command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT table_schema, table_name
+ FROM information_schema.tables
+ WHERE table_schema = ANY(@schemas)
+ AND table_type = ANY(@table_types)
+ ORDER BY table_schema, table_name
+ """;
+
+ DbParameter schemasParameter = command.CreateParameter();
+ schemasParameter.ParameterName = "schemas";
+ schemasParameter.Value = schemas.ToArray();
+ command.Parameters.Add(schemasParameter);
+
+ DbParameter tableTypesParameter = command.CreateParameter();
+ tableTypesParameter.ParameterName = "table_types";
+ tableTypesParameter.Value = includeViews
+ ? new[] { "BASE TABLE", "VIEW" }
+ : new[] { "BASE TABLE" };
+ command.Parameters.Add(tableTypesParameter);
+
+ using DbDataReader reader = command.ExecuteReader(CommandBehavior.SequentialAccess);
+ while (reader.Read())
+ {
+ yield return new PostgreSqlTable(reader.GetString(0), reader.GetString(1));
+ }
+ }
+
+ private static string RegistrationName(PostgreSqlTable table, IReadOnlyCollection schemas) =>
+ schemas.Count == 1 && string.Equals(table.SchemaName, "public", StringComparison.OrdinalIgnoreCase)
+ ? table.TableName
+ : $"{table.SchemaName}_{table.TableName}";
+
+ private sealed record PostgreSqlTable(string SchemaName, string TableName);
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs
new file mode 100644
index 0000000..0460058
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/PushedQuery.cs
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters);
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs
new file mode 100644
index 0000000..b38e11b
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlDialect.cs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+internal abstract class SqlDialect
+{
+ public abstract string QuoteIdentifier(string identifier);
+
+ public virtual string ParameterName(int index) => $"@df_filter_{index}";
+
+ public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch
+ {
+ StreamingTableFilterOperator.Equal => "=",
+ StreamingTableFilterOperator.NotEqual => "<>",
+ StreamingTableFilterOperator.LessThan => "<",
+ StreamingTableFilterOperator.LessThanOrEqual => "<=",
+ StreamingTableFilterOperator.GreaterThan => ">",
+ StreamingTableFilterOperator.GreaterThanOrEqual => ">=",
+ _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."),
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs
new file mode 100644
index 0000000..36259fa
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryBuilder.cs
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+internal sealed class SqlQueryBuilder(SqlDialect dialect)
+{
+ public PushedQuery Build(string baseQuery, StreamingTableScanRequest request)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery);
+ ArgumentNullException.ThrowIfNull(request);
+
+ List parameters = [];
+ SqlSelectStatement statement = new(
+ BuildProjection(request),
+ TrimTrailingSemicolon(baseQuery),
+ BuildPredicates(request, parameters),
+ request.Limit);
+
+ return new PushedQuery(statement.ToSql(), parameters);
+ }
+
+ private string BuildProjection(StreamingTableScanRequest request)
+ {
+ if (!request.HasProjection)
+ {
+ return "*";
+ }
+
+ if (request.Projection.Count == 0)
+ {
+ return "1";
+ }
+
+ return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier));
+ }
+
+ private List BuildPredicates(
+ StreamingTableScanRequest request,
+ List parameters)
+ {
+ List predicates = [];
+ for (int i = 0; i < request.Filters.Count; i++)
+ {
+ StreamingTableFilter filter = request.Filters[i];
+ string parameterName = dialect.ParameterName(i);
+ predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}");
+ parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter)));
+ }
+
+ return predicates;
+ }
+
+ private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch
+ {
+ StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value),
+ StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture),
+ StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture),
+ StreamingTableFilterValueKind.String => filter.Value,
+ _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."),
+ };
+
+ private static string TrimTrailingSemicolon(string sql) =>
+ sql.Trim().TrimEnd(';');
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs
new file mode 100644
index 0000000..b172ef3
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlQueryParameter.cs
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+internal sealed record SqlQueryParameter(string Name, object Value);
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs
new file mode 100644
index 0000000..53c8111
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Sql/SqlSelectStatement.cs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Globalization;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql.Sql;
+
+internal sealed record SqlSelectStatement(
+ string Projection,
+ string Source,
+ IReadOnlyList Predicates,
+ ulong? Limit)
+{
+ public string ToSql()
+ {
+ List clauses =
+ [
+ $"SELECT {Projection}",
+ $"FROM ({Source}) AS df_source",
+ ];
+
+ if (Predicates.Count > 0)
+ {
+ clauses.Add($"WHERE {string.Join(" AND ", Predicates)}");
+ }
+
+ if (Limit.HasValue)
+ {
+ clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}");
+ }
+
+ return string.Join(" ", clauses);
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs
new file mode 100644
index 0000000..b8cdf90
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.PostgreSql;
+
+internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly StringArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(FormatValue(reader.GetValue(Ordinal)));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj
new file mode 100644
index 0000000..2e3fc81
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Apache.DataFusion.TableProviders.Sqlite.csproj
@@ -0,0 +1,17 @@
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs
new file mode 100644
index 0000000..4f290ec
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly BooleanArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs
new file mode 100644
index 0000000..9cdc462
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal abstract class ColumnAppender(int ordinal)
+{
+ protected int Ordinal { get; } = ordinal;
+
+ public abstract void Append(DbDataReader reader);
+
+ public abstract IArrowArray Build();
+
+ protected static string FormatValue(object value) =>
+ value switch
+ {
+ byte[] bytes => Convert.ToBase64String(bytes),
+ DateTime dateTime => dateTime.ToString("O", CultureInfo.InvariantCulture),
+ DateOnly dateOnly => dateOnly.ToString("O", CultureInfo.InvariantCulture),
+ TimeOnly timeOnly => timeOnly.ToString("O", CultureInfo.InvariantCulture),
+ IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture),
+ _ => value.ToString() ?? string.Empty,
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs
new file mode 100644
index 0000000..d9e3876
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnKind.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal enum ColumnKind
+{
+ Boolean,
+ Int32,
+ Int64,
+ Float,
+ Double,
+ String,
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs
new file mode 100644
index 0000000..48a411a
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnPlan.cs
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using Apache.Arrow.Types;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed record ColumnPlan(
+ int Ordinal,
+ string Name,
+ IArrowType DataType,
+ bool Nullable,
+ ColumnKind Kind)
+{
+ public static ColumnPlan From(DbColumn column, int ordinal)
+ {
+ Type? type = System.Nullable.GetUnderlyingType(column.DataType ?? typeof(string)) ?? column.DataType;
+ ColumnKind kind = GetKind(type);
+ return new ColumnPlan(
+ ordinal,
+ column.ColumnName ?? $"column_{ordinal}",
+ GetArrowType(kind),
+ column.AllowDBNull ?? true,
+ kind);
+ }
+
+ public ColumnAppender CreateAppender() => Kind switch
+ {
+ ColumnKind.Boolean => new BooleanColumnAppender(Ordinal),
+ ColumnKind.Int32 => new Int32ColumnAppender(Ordinal),
+ ColumnKind.Int64 => new Int64ColumnAppender(Ordinal),
+ ColumnKind.Float => new FloatColumnAppender(Ordinal),
+ ColumnKind.Double => new DoubleColumnAppender(Ordinal),
+ _ => new StringColumnAppender(Ordinal),
+ };
+
+ private static ColumnKind GetKind(Type? type)
+ {
+ if (type == typeof(bool))
+ {
+ return ColumnKind.Boolean;
+ }
+
+ if (type == typeof(byte) || type == typeof(sbyte) || type == typeof(short) || type == typeof(ushort) || type == typeof(int))
+ {
+ return ColumnKind.Int32;
+ }
+
+ if (type == typeof(uint) || type == typeof(long) || type == typeof(ulong))
+ {
+ return ColumnKind.Int64;
+ }
+
+ if (type == typeof(float))
+ {
+ return ColumnKind.Float;
+ }
+
+ if (type == typeof(double) || type == typeof(decimal))
+ {
+ return ColumnKind.Double;
+ }
+
+ return ColumnKind.String;
+ }
+
+ private static IArrowType GetArrowType(ColumnKind kind) => kind switch
+ {
+ ColumnKind.Boolean => BooleanType.Default,
+ ColumnKind.Int32 => Int32Type.Default,
+ ColumnKind.Int64 => Int64Type.Default,
+ ColumnKind.Float => FloatType.Default,
+ ColumnKind.Double => DoubleType.Default,
+ _ => StringType.Default,
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs
new file mode 100644
index 0000000..afcf9d4
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly DoubleArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs
new file mode 100644
index 0000000..c25d5fb
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly FloatArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs
new file mode 100644
index 0000000..f68a3c5
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly Int32Array.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs
new file mode 100644
index 0000000..1b0a7fc
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using System.Globalization;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly Int64Array.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs
new file mode 100644
index 0000000..15fa8d7
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs
@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using Microsoft.Data.Sqlite;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+public static class SessionContextExtensions
+{
+ public static void RegisterSqlite(this SessionContext context, string connectionString)
+ {
+ context.RegisterSqlite(new SqliteDatabaseOptions
+ {
+ ConnectionString = connectionString,
+ });
+ }
+
+ public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews))
+ {
+ context.RegisterStreamingTable(tableName, new SqliteStreamingTableProvider(new SqliteTableOptions
+ {
+ ConnectionString = options.ConnectionString,
+ TableName = tableName,
+ BatchSize = options.BatchSize,
+ }));
+ }
+ }
+
+ public static void RegisterSqlite(this SessionContext context, string name, SqliteTableOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(context);
+ ArgumentNullException.ThrowIfNull(options);
+
+ context.RegisterStreamingTable(name, new SqliteStreamingTableProvider(options));
+ }
+
+ public static void RegisterSqlite(this SessionContext context, string name, string connectionString)
+ {
+ context.RegisterSqlite(name, new SqliteTableOptions
+ {
+ ConnectionString = connectionString,
+ TableName = name,
+ });
+ }
+
+ private static IEnumerable FetchTables(string connectionString, bool includeViews)
+ {
+ using SqliteConnection connection = new(connectionString);
+ connection.Open();
+
+ using DbCommand command = connection.CreateCommand();
+ command.CommandText = """
+ SELECT name
+ FROM sqlite_master
+ WHERE type IN ('table', 'view')
+ AND (@include_views OR type = 'table')
+ AND name NOT LIKE 'sqlite_%'
+ ORDER BY name
+ """;
+
+ DbParameter includeViewsParameter = command.CreateParameter();
+ includeViewsParameter.ParameterName = "@include_views";
+ includeViewsParameter.Value = includeViews;
+ command.Parameters.Add(includeViewsParameter);
+
+ using DbDataReader reader = command.ExecuteReader();
+ while (reader.Read())
+ {
+ yield return reader.GetString(0);
+ }
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs
new file mode 100644
index 0000000..659f3cf
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/PushedQuery.cs
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+internal sealed record PushedQuery(string Sql, IReadOnlyList Parameters);
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs
new file mode 100644
index 0000000..9f15e5b
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlDialect.cs
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+internal abstract class SqlDialect
+{
+ public abstract string QuoteIdentifier(string identifier);
+
+ public virtual string ParameterName(int index) => $"@df_filter_{index}";
+
+ public virtual string OperatorSql(StreamingTableFilterOperator op) => op switch
+ {
+ StreamingTableFilterOperator.Equal => "=",
+ StreamingTableFilterOperator.NotEqual => "<>",
+ StreamingTableFilterOperator.LessThan => "<",
+ StreamingTableFilterOperator.LessThanOrEqual => "<=",
+ StreamingTableFilterOperator.GreaterThan => ">",
+ StreamingTableFilterOperator.GreaterThanOrEqual => ">=",
+ _ => throw new NotSupportedException($"Unsupported filter operator '{op}'."),
+ };
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs
new file mode 100644
index 0000000..18ed831
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryBuilder.cs
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+internal sealed class SqlQueryBuilder(SqlDialect dialect)
+{
+ public PushedQuery Build(string baseQuery, StreamingTableScanRequest request)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(baseQuery);
+ ArgumentNullException.ThrowIfNull(request);
+
+ List parameters = [];
+ SqlSelectStatement statement = new(
+ BuildProjection(request),
+ TrimTrailingSemicolon(baseQuery),
+ BuildPredicates(request, parameters),
+ request.Limit);
+
+ return new PushedQuery(statement.ToSql(), parameters);
+ }
+
+ private string BuildProjection(StreamingTableScanRequest request)
+ {
+ if (!request.HasProjection)
+ {
+ return "*";
+ }
+
+ if (request.Projection.Count == 0)
+ {
+ return "1";
+ }
+
+ return string.Join(", ", request.Projection.Select(dialect.QuoteIdentifier));
+ }
+
+ private List BuildPredicates(
+ StreamingTableScanRequest request,
+ List parameters)
+ {
+ List predicates = [];
+ for (int i = 0; i < request.Filters.Count; i++)
+ {
+ StreamingTableFilter filter = request.Filters[i];
+ string parameterName = dialect.ParameterName(i);
+ predicates.Add($"{dialect.QuoteIdentifier(filter.Column)} {dialect.OperatorSql(filter.Operator)} {parameterName}");
+ parameters.Add(new SqlQueryParameter(parameterName, ConvertFilterValue(filter)));
+ }
+
+ return predicates;
+ }
+
+ private static object ConvertFilterValue(StreamingTableFilter filter) => filter.ValueKind switch
+ {
+ StreamingTableFilterValueKind.Boolean => bool.Parse(filter.Value),
+ StreamingTableFilterValueKind.Integer => long.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture),
+ StreamingTableFilterValueKind.FloatingPoint => double.Parse(filter.Value, System.Globalization.CultureInfo.InvariantCulture),
+ StreamingTableFilterValueKind.String => filter.Value,
+ _ => throw new NotSupportedException($"Unsupported filter value kind '{filter.ValueKind}'."),
+ };
+
+ private static string TrimTrailingSemicolon(string sql) =>
+ sql.Trim().TrimEnd(';');
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs
new file mode 100644
index 0000000..1c78e65
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlQueryParameter.cs
@@ -0,0 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+internal sealed record SqlQueryParameter(string Name, object Value);
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs
new file mode 100644
index 0000000..754cb6f
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/Sql/SqlSelectStatement.cs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Globalization;
+
+namespace Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+internal sealed record SqlSelectStatement(
+ string Projection,
+ string Source,
+ IReadOnlyList Predicates,
+ ulong? Limit)
+{
+ public string ToSql()
+ {
+ List clauses =
+ [
+ $"SELECT {Projection}",
+ $"FROM ({Source}) AS df_source",
+ ];
+
+ if (Predicates.Count > 0)
+ {
+ clauses.Add($"WHERE {string.Join(" AND ", Predicates)}");
+ }
+
+ if (Limit.HasValue)
+ {
+ clauses.Add($"LIMIT {Limit.Value.ToString(CultureInfo.InvariantCulture)}");
+ }
+
+ return string.Join(" ", clauses);
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs
new file mode 100644
index 0000000..3851ed6
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data;
+using System.Data.Common;
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.DataFusion.TableProviders.Sqlite.Sql;
+using Microsoft.Data.Sqlite;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class SqliteArrowArrayStream : IArrowArrayStream
+{
+ private readonly Schema schema;
+ private readonly ColumnPlan[] columns;
+ private readonly int batchSize;
+ private SqliteConnection? connection;
+ private DbCommand? command;
+ private DbDataReader? reader;
+ private bool finished;
+
+ public SqliteArrowArrayStream(
+ string connectionString,
+ string query,
+ Schema schema,
+ ColumnPlan[] columns,
+ int batchSize,
+ IReadOnlyList? parameters = null)
+ {
+ this.schema = schema;
+ this.columns = columns;
+ this.batchSize = batchSize;
+
+ try
+ {
+ connection = new SqliteConnection(connectionString);
+ connection.Open();
+
+ command = connection.CreateCommand();
+ command.CommandText = query;
+ if (parameters is not null)
+ {
+ foreach (SqlQueryParameter parameter in parameters)
+ {
+ DbParameter dbParameter = command.CreateParameter();
+ dbParameter.ParameterName = parameter.Name;
+ dbParameter.Value = parameter.Value;
+ command.Parameters.Add(dbParameter);
+ }
+ }
+
+ reader = command.ExecuteReader(CommandBehavior.SequentialAccess);
+ }
+ catch
+ {
+ Dispose();
+ throw;
+ }
+ }
+
+ public Schema Schema => schema;
+
+ public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (finished || reader is null)
+ {
+ return new ValueTask((RecordBatch?)null);
+ }
+
+ ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray();
+ int rowCount = 0;
+ while (rowCount < batchSize && reader.Read())
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ foreach (ColumnAppender appender in appenders)
+ {
+ appender.Append(reader);
+ }
+
+ rowCount++;
+ }
+
+ if (rowCount == 0)
+ {
+ finished = true;
+ return new ValueTask((RecordBatch?)null);
+ }
+
+ IArrowArray[] arrays = appenders.Select(static appender => appender.Build()).ToArray();
+ RecordBatch batch = new(schema, arrays, rowCount);
+ return new ValueTask(batch);
+ }
+
+ public void Dispose()
+ {
+ reader?.Dispose();
+ command?.Dispose();
+ connection?.Dispose();
+ reader = null;
+ command = null;
+ connection = null;
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs
new file mode 100644
index 0000000..9f7eb32
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+public sealed class SqliteDatabaseOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public bool IncludeViews { get; init; } = true;
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs
new file mode 100644
index 0000000..2e749b0
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDialect.cs
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Apache.DataFusion.TableProviders.Sqlite.Sql;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class SqliteDialect : SqlDialect
+{
+ public static SqliteDialect Instance { get; } = new();
+
+ private SqliteDialect()
+ {
+ }
+
+ public override string QuoteIdentifier(string identifier) =>
+ $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\"";
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs
new file mode 100644
index 0000000..9381fcf
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data;
+using System.Data.Common;
+using Apache.Arrow;
+using Apache.Arrow.Ipc;
+using Apache.DataFusion.TableProviders.Sqlite.Sql;
+using Microsoft.Data.Sqlite;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+public sealed class SqliteStreamingTableProvider : StreamingTableProvider
+{
+ private readonly string connectionString;
+ private readonly string sourceSql;
+ private readonly int batchSize;
+ private readonly ColumnPlan[] columns;
+ private readonly SqlQueryBuilder queryBuilder = new(SqliteDialect.Instance);
+
+ public SqliteStreamingTableProvider(SqliteTableOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
+ }
+
+ if (options.BatchSize <= 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
+ }
+
+ connectionString = options.ConnectionString;
+ sourceSql = BuildSourceSql(options);
+ batchSize = options.BatchSize;
+ columns = FetchColumns(connectionString, sourceSql);
+ Schema = BuildSchema(columns);
+ }
+
+ public override Schema Schema { get; }
+
+ public override bool SupportsPushdown => true;
+
+ public override IArrowArrayStream Scan() =>
+ new SqliteArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize);
+
+ public override IArrowArrayStream Scan(StreamingTableScanRequest request)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+
+ PushedQuery pushedQuery = queryBuilder.Build(sourceSql, request);
+ ColumnPlan[] projectedColumns = ProjectColumns(request);
+ Schema projectedSchema = BuildSchema(projectedColumns);
+ return new SqliteArrowArrayStream(
+ connectionString,
+ pushedQuery.Sql,
+ projectedSchema,
+ projectedColumns,
+ batchSize,
+ pushedQuery.Parameters);
+ }
+
+ private static string BuildSourceSql(SqliteTableOptions options)
+ {
+ if (string.IsNullOrWhiteSpace(options.TableName))
+ {
+ throw new ArgumentException("Table name must be provided.", nameof(options));
+ }
+
+ return $"SELECT * FROM {SqliteDialect.Instance.QuoteIdentifier(options.TableName)}";
+ }
+
+ private static ColumnPlan[] FetchColumns(string connectionString, string query)
+ {
+ using SqliteConnection connection = new(connectionString);
+ connection.Open();
+
+ using DbCommand command = connection.CreateCommand();
+ command.CommandText = query;
+
+ using DbDataReader reader = command.ExecuteReader(CommandBehavior.SchemaOnly | CommandBehavior.KeyInfo);
+ return reader.GetColumnSchema()
+ .Select((column, ordinal) => ColumnPlan.From(column, ordinal))
+ .ToArray();
+ }
+
+ private static Schema BuildSchema(IEnumerable columns)
+ {
+ Schema.Builder builder = new();
+ foreach (ColumnPlan column in columns)
+ {
+ builder.Field(field => field
+ .Name(column.Name)
+ .DataType(column.DataType)
+ .Nullable(column.Nullable));
+ }
+
+ return builder.Build();
+ }
+
+ private ColumnPlan[] ProjectColumns(StreamingTableScanRequest request)
+ {
+ if (!request.HasProjection)
+ {
+ return columns;
+ }
+
+ Dictionary columnLookup = columns.ToDictionary(
+ static column => column.Name,
+ StringComparer.OrdinalIgnoreCase);
+ return request.Projection
+ .Select((name, ordinal) => columnLookup.TryGetValue(name, out ColumnPlan? column)
+ ? column with { Ordinal = ordinal }
+ : throw new InvalidOperationException($"SQLite table provider does not contain column '{name}'."))
+ .ToArray();
+ }
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs
new file mode 100644
index 0000000..28773a3
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+public sealed class SqliteTableOptions
+{
+ public required string ConnectionString { get; init; }
+
+ public required string TableName { get; init; }
+
+ public int BatchSize { get; init; } = 1024;
+}
diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs
new file mode 100644
index 0000000..ff4a586
--- /dev/null
+++ b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Data.Common;
+using Apache.Arrow;
+
+namespace Apache.DataFusion.TableProviders.Sqlite;
+
+internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal)
+{
+ private readonly StringArray.Builder builder = new();
+
+ public override void Append(DbDataReader reader)
+ {
+ if (reader.IsDBNull(Ordinal))
+ {
+ builder.AppendNull();
+ return;
+ }
+
+ builder.Append(FormatValue(reader.GetValue(Ordinal)));
+ }
+
+ public override IArrowArray Build() => builder.Build();
+}
diff --git a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs
index 0a32745..2f7a639 100644
--- a/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs
+++ b/tests/Apache.DataFusion.Tests/StreamingTableProviderTests.cs
@@ -19,8 +19,6 @@
using Apache.Arrow.Ipc;
using Apache.Arrow.Types;
-using Apache.DataFusion;
-
namespace Apache.DataFusion.Tests;
public class StreamingTableProviderTests