diff --git a/native/src/lib.rs b/native/src/lib.rs index 7fef138..f0abf3a 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -29,7 +29,7 @@ use std::os::raw::{c_char, c_int, c_uchar}; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::{Arc, OnceLock}; -use ::arrow::array::{RecordBatch, RecordBatchIterator}; +use ::arrow::array::{RecordBatch, RecordBatchIterator, RecordBatchReader}; use ::arrow::datatypes::SchemaRef; use ::arrow::error::ArrowError; use ::arrow::ffi_stream::FFI_ArrowArrayStream; @@ -43,6 +43,7 @@ use datafusion::error::DataFusionError; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::SendableRecordBatchStream; use datafusion::logical_expr::{col, Expr, Partitioning, SortExpr}; use datafusion::prelude::SessionConfig; use futures::StreamExt; @@ -823,6 +824,52 @@ fn export_batches( Ok(()) } +/// Adapts a DataFusion `SendableRecordBatchStream` into a synchronous +/// `RecordBatchReader` so it can be handed to an `FFI_ArrowArrayStream` and +/// pulled lazily from the managed side. Each managed pull drives a single +/// `stream.next()` on the shared Tokio runtime, so native memory stays bounded +/// by one in-flight batch instead of the entire result set. +struct BlockingStreamReader { + schema: SchemaRef, + stream: SendableRecordBatchStream, +} + +impl Iterator for BlockingStreamReader { + type Item = Result; + + fn next(&mut self) -> Option { + runtime() + .block_on(self.stream.next()) + .map(|result| result.map_err(|err| ArrowError::ExternalError(Box::new(err)))) + } +} + +impl RecordBatchReader for BlockingStreamReader { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } +} + +/// Export a streaming `SendableRecordBatchStream` through the caller-provided +/// Arrow C Data Interface pointer. Unlike `export_batches`, the result set is +/// never collected up front: batches are produced one at a time as the managed +/// reader advances, keeping native memory bounded to a single in-flight batch. +fn export_stream( + out: *mut FFI_ArrowArrayStream, + schema: SchemaRef, + stream: SendableRecordBatchStream, +) -> NativeResult<()> { + if out.is_null() { + return Err("output stream pointer is null".into()); + } + let reader = BlockingStreamReader { schema, stream }; + let ffi = FFI_ArrowArrayStream::new(Box::new(reader)); + unsafe { + std::ptr::write(out, ffi); + } + Ok(()) +} + #[no_mangle] pub extern "C" fn df_dataframe_collect_cdata( handle: *mut DataFrame, @@ -848,15 +895,11 @@ pub extern "C" fn df_dataframe_execute_stream_cdata( let df = unsafe { *Box::from_raw(require_ptr(handle, "DataFrame handle")?) }; let token = resolve_token(token_handle); let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone()); - let batches = runtime().block_on(run_cancellable(&token, async move { - let mut stream = df.execute_stream().await?; - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push(batch?); - } - Ok::<_, DataFusionError>(batches) - }))?; - export_batches(out, schema, batches) + // Only establish the stream here (honouring a pre-cancelled token); the + // batches themselves are pulled lazily by the managed reader so the full + // result set is never buffered in native memory at once. + let stream = runtime().block_on(run_cancellable(&token, df.execute_stream()))?; + export_stream(out, schema, stream) }) } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs index 8750e39..b298def 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseArrowArrayStream.cs @@ -29,48 +29,33 @@ internal sealed class ClickHouseArrowArrayStream : IArrowArrayStream private readonly Schema schema; private readonly ColumnPlan[] columns; private readonly int batchSize; + private readonly Func connectionFactory; + private readonly string query; + private readonly IReadOnlyList? parameters; private ClickHouseConnection? connection; private DbCommand? command; private DbDataReader? reader; private bool finished; public ClickHouseArrowArrayStream( - string connectionString, + Func connectionFactory, string query, Schema schema, ColumnPlan[] columns, int batchSize, IReadOnlyList? parameters = null) { + // Defer opening the connection and issuing the query until the first + // batch is pulled. The schema is already known from registration, so a + // consumer that only inspects the schema never pays for a round trip, + // and the query cost shifts out of plan establishment into the first + // read. this.schema = schema; this.columns = columns; this.batchSize = batchSize; - - try - { - connection = new ClickHouseConnection(connectionString); - connection.Open(); - - command = connection.CreateCommand(); - command.CommandText = query; - if (parameters is not null) - { - foreach (ClickHouseQueryParameter parameter in parameters) - { - DbParameter dbParameter = command.CreateParameter(); - dbParameter.ParameterName = parameter.Name; - dbParameter.Value = parameter.Value; - command.Parameters.Add(dbParameter); - } - } - - reader = command.ExecuteReader(CommandBehavior.SequentialAccess); - } - catch - { - Dispose(); - throw; - } + this.connectionFactory = connectionFactory; + this.query = query; + this.parameters = parameters; } public Schema Schema => schema; @@ -78,19 +63,21 @@ public ClickHouseArrowArrayStream( public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (finished || reader is null) + if (finished) { return new ValueTask((RecordBatch?)null); } + DbDataReader activeReader = EnsureReader(); + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; - while (rowCount < batchSize && reader.Read()) + while (rowCount < batchSize && activeReader.Read()) { cancellationToken.ThrowIfCancellationRequested(); foreach (ColumnAppender appender in appenders) { - appender.Append(reader); + appender.Append(activeReader); } rowCount++; @@ -107,6 +94,41 @@ public ClickHouseArrowArrayStream( return new ValueTask(batch); } + private DbDataReader EnsureReader() + { + if (reader is not null) + { + return reader; + } + + try + { + connection = connectionFactory(); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (ClickHouseQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + return reader; + } + catch + { + Dispose(); + throw; + } + } + public void Dispose() { reader?.Dispose(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs index 2866949..cffeb95 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseDatabaseOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using ClickHouse.Driver.ADO; + namespace Apache.DataFusion.TableProviders.ClickHouse; public sealed class ClickHouseDatabaseOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public Func? ConnectionFactory { get; init; } public string? SourceName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs index 6a66879..9ca4e23 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseStreamingTableProvider.cs @@ -26,7 +26,7 @@ namespace Apache.DataFusion.TableProviders.ClickHouse; public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider { - private readonly string connectionString; + private readonly Func connectionFactory; private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; @@ -35,20 +35,15 @@ public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) { ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - connectionString = options.ConnectionString; + connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options)); sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, sourceSql); + columns = FetchColumns(connectionFactory, sourceSql); Schema = BuildSchema(columns); } @@ -57,7 +52,7 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new ClickHouseArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + new ClickHouseArrowArrayStream(connectionFactory, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { @@ -67,7 +62,7 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new ClickHouseArrowArrayStream( - connectionString, + connectionFactory, pushedQuery.Sql, projectedSchema, projectedColumns, @@ -75,6 +70,21 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static Func CreateConnectionFactory(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName); + } + + // A ClickHouseDataSource owns a single underlying client, so every + // connection it hands out shares the same HTTP connection pool. This + // avoids a fresh TLS handshake on each scan, unlike constructing a + // standalone ClickHouseConnection per call. + ClickHouseDataSource dataSource = new(connectionString); + return dataSource.CreateConnection; + } + private static string BuildSourceSql(ClickHouseTableOptions options) { if (string.IsNullOrWhiteSpace(options.TableName)) @@ -93,9 +103,9 @@ private static string QuoteQualifiedTableName(string? databaseName, string table : $"{ClickHouseDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; } - private static ColumnPlan[] FetchColumns(string connectionString, string query) + private static ColumnPlan[] FetchColumns(Func connectionFactory, string query) { - using ClickHouseConnection connection = new(connectionString); + using ClickHouseConnection connection = connectionFactory(); connection.Open(); using DbCommand command = connection.CreateCommand(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs index c1471ab..b3e4868 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/ClickHouseTableOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using ClickHouse.Driver.ADO; + namespace Apache.DataFusion.TableProviders.ClickHouse; public sealed class ClickHouseTableOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public Func? ConnectionFactory { get; init; } public string? DatabaseName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs index 15185d6..37b997f 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/SessionContextExtensions.cs @@ -34,21 +34,20 @@ public static void RegisterClickHouse(this SessionContext context, ClickHouseDat { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + Func connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options)); + + foreach (ClickHouseTable table in FetchTables(connectionFactory, options.DatabaseName, options.IncludeViews)) { RegisterSourceTable(context, options.SourceName, table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions { ConnectionString = options.ConnectionString, + ConnectionFactory = connectionFactory, DatabaseName = table.DatabaseName, TableName = table.TableName, BatchSize = options.BatchSize, @@ -73,12 +72,26 @@ public static void RegisterClickHouse(this SessionContext context, string name, }); } + private static Func CreateConnectionFactory(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName); + } + + // Share one ClickHouseDataSource across every table registered from this + // database so all scans reuse a single HTTP connection pool instead of + // re-establishing a TLS connection on each scan. + ClickHouseDataSource dataSource = new(connectionString); + return dataSource.CreateConnection; + } + private static IEnumerable FetchTables( - string connectionString, + Func connectionFactory, string? databaseName, bool includeViews) { - using ClickHouseConnection connection = new(connectionString); + using ClickHouseConnection connection = connectionFactory(); connection.Open(); using DbCommand command = connection.CreateCommand(); diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs index 0b7a8ed..a96f860 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbDatabaseOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using MongoDB.Driver; + namespace Apache.DataFusion.TableProviders.MongoDB; public sealed class MongoDbDatabaseOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public IMongoClient? Client { get; init; } public string? SourceName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs index e4beaf7..052774e 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbStreamingTableProvider.cs @@ -32,11 +32,6 @@ public sealed class MongoDbStreamingTableProvider : StreamingTableProvider public MongoDbStreamingTableProvider(MongoDbTableOptions options) { ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } - if (string.IsNullOrWhiteSpace(options.DatabaseName)) { throw new ArgumentException("Database name cannot be null or whitespace.", nameof(options)); @@ -58,7 +53,7 @@ public MongoDbStreamingTableProvider(MongoDbTableOptions options) } batchSize = options.BatchSize; - MongoClient client = new(options.ConnectionString); + IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options)); collection = client.GetDatabase(options.DatabaseName).GetCollection(options.CollectionName); columns = InferColumns(collection, options.SchemaInferenceLimit); Schema = BuildSchema(columns); @@ -86,6 +81,16 @@ private MongoDbArrowArrayStream CreateStream(StreamingTableScanRequest request) return new MongoDbArrowArrayStream(collection, query, projectedSchema, projectedColumns, batchSize); } + private static IMongoClient CreateClient(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a MongoDB client is not provided.", parameterName); + } + + return new MongoClient(connectionString); + } + private MongoDbColumnPlan[] ProjectColumns(StreamingTableScanRequest request) { if (!request.HasProjection) diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs index ba1517b..a3f6d73 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/MongoDbTableOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using MongoDB.Driver; + namespace Apache.DataFusion.TableProviders.MongoDB; public sealed class MongoDbTableOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public IMongoClient? Client { get; init; } public required string DatabaseName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs index 86467d3..a9bb5a1 100644 --- a/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MongoDB/SessionContextExtensions.cs @@ -33,10 +33,6 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } if (options.BatchSize <= 0) { @@ -49,13 +45,14 @@ public static void RegisterMongoDb(this SessionContext context, MongoDbDatabaseO } string databaseName = ResolveDatabaseName(options.ConnectionString, options.DatabaseName); - MongoClient client = new(options.ConnectionString); + IMongoClient client = options.Client ?? CreateClient(options.ConnectionString, nameof(options)); IMongoDatabase database = client.GetDatabase(databaseName); foreach (string collectionName in database.ListCollectionNames().ToEnumerable()) { RegisterSourceTable(context, options.SourceName, collectionName, new MongoDbStreamingTableProvider(new MongoDbTableOptions { ConnectionString = options.ConnectionString, + Client = client, DatabaseName = databaseName, CollectionName = collectionName, BatchSize = options.BatchSize, @@ -98,13 +95,28 @@ public static void RegisterMongoDb( }); } - private static string ResolveDatabaseName(string connectionString, string? databaseName) + private static IMongoClient CreateClient(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a MongoDB client is not provided.", parameterName); + } + + return new MongoClient(connectionString); + } + + private static string ResolveDatabaseName(string? connectionString, string? databaseName) { if (!string.IsNullOrWhiteSpace(databaseName)) { return databaseName; } + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Database name must be provided in MongoDB options when a connection string is not provided.", nameof(databaseName)); + } + string? connectionDatabaseName = MongoUrl.Create(connectionString).DatabaseName; if (!string.IsNullOrWhiteSpace(connectionDatabaseName)) { diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs index 2ce32a1..3e974ea 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs @@ -29,45 +29,31 @@ internal sealed class MySqlArrowArrayStream : IArrowArrayStream private readonly Schema schema; private readonly ColumnPlan[] columns; private readonly int batchSize; + private readonly MySqlDataSource dataSource; + private readonly string query; + private readonly IReadOnlyList? parameters; private MySqlConnection? connection; private MySqlCommand? command; private DbDataReader? reader; private bool finished; public MySqlArrowArrayStream( - string connectionString, + MySqlDataSource dataSource, string query, Schema schema, ColumnPlan[] columns, int batchSize, IReadOnlyList? parameters = null) { + // Defer opening the connection and issuing the query until the first + // batch is pulled. The schema is already known from registration, so the + // query cost shifts out of plan establishment into the first read. this.schema = schema; this.columns = columns; this.batchSize = batchSize; - - try - { - connection = new MySqlConnection(connectionString); - connection.Open(); - - command = connection.CreateCommand(); - command.CommandText = query; - if (parameters is not null) - { - foreach (SqlQueryParameter parameter in parameters) - { - command.Parameters.AddWithValue(parameter.Name, parameter.Value); - } - } - - reader = command.ExecuteReader(CommandBehavior.SequentialAccess); - } - catch - { - Dispose(); - throw; - } + this.dataSource = dataSource; + this.query = query; + this.parameters = parameters; } public Schema Schema => schema; @@ -75,19 +61,21 @@ public MySqlArrowArrayStream( public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (finished || reader is null) + if (finished) { return new ValueTask((RecordBatch?)null); } + DbDataReader activeReader = EnsureReader(); + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; - while (rowCount < batchSize && reader.Read()) + while (rowCount < batchSize && activeReader.Read()) { cancellationToken.ThrowIfCancellationRequested(); foreach (ColumnAppender appender in appenders) { - appender.Append(reader); + appender.Append(activeReader); } rowCount++; @@ -104,6 +92,37 @@ public MySqlArrowArrayStream( return new ValueTask(batch); } + private DbDataReader EnsureReader() + { + if (reader is not null) + { + return reader; + } + + try + { + connection = dataSource.OpenConnection(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + command.Parameters.AddWithValue(parameter.Name, parameter.Value); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + return reader; + } + catch + { + Dispose(); + throw; + } + } + public void Dispose() { reader?.Dispose(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs index f3f0053..61531f4 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlDatabaseOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using MySqlConnector; + namespace Apache.DataFusion.TableProviders.MySql; public sealed class MySqlDatabaseOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public MySqlDataSource? DataSource { get; init; } public string? SourceName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs index 65c395d..8c44a48 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlStreamingTableProvider.cs @@ -26,7 +26,7 @@ namespace Apache.DataFusion.TableProviders.MySql; public sealed class MySqlStreamingTableProvider : StreamingTableProvider { - private readonly string connectionString; + private readonly MySqlDataSource dataSource; private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; @@ -35,20 +35,15 @@ public sealed class MySqlStreamingTableProvider : StreamingTableProvider public MySqlStreamingTableProvider(MySqlTableOptions options) { ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - connectionString = options.ConnectionString; + dataSource = options.DataSource ?? CreateDataSource(options.ConnectionString, nameof(options)); sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, sourceSql); + columns = FetchColumns(dataSource, sourceSql); Schema = BuildSchema(columns); } @@ -57,7 +52,7 @@ public MySqlStreamingTableProvider(MySqlTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new MySqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + new MySqlArrowArrayStream(dataSource, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { @@ -67,7 +62,7 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new MySqlArrowArrayStream( - connectionString, + dataSource, pushedQuery.Sql, projectedSchema, projectedColumns, @@ -75,6 +70,16 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static MySqlDataSource CreateDataSource(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a data source is not provided.", parameterName); + } + + return new MySqlDataSourceBuilder(connectionString).Build(); + } + private static string BuildSourceSql(MySqlTableOptions options) { if (string.IsNullOrWhiteSpace(options.TableName)) @@ -93,10 +98,9 @@ private static string QuoteQualifiedTableName(string? databaseName, string table : $"{MySqlDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}"; } - private static ColumnPlan[] FetchColumns(string connectionString, string query) + private static ColumnPlan[] FetchColumns(MySqlDataSource dataSource, string query) { - using MySqlConnection connection = new(connectionString); - connection.Open(); + using MySqlConnection connection = dataSource.OpenConnection(); using DbCommand command = connection.CreateCommand(); command.CommandText = query; diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs index c91e4ac..bf97b78 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlTableOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using MySqlConnector; + namespace Apache.DataFusion.TableProviders.MySql; public sealed class MySqlTableOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public MySqlDataSource? DataSource { get; init; } public string? DatabaseName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs index 2f39496..b70cd3f 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/SessionContextExtensions.cs @@ -33,21 +33,20 @@ public static void RegisterMySql(this SessionContext context, MySqlDatabaseOptio { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - foreach (MySqlTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews)) + MySqlDataSource dataSource = options.DataSource ?? CreateDataSource(options.ConnectionString, nameof(options)); + + foreach (MySqlTable table in FetchTables(dataSource, options.DatabaseName, options.IncludeViews)) { RegisterSourceTable(context, options.SourceName, table.TableName, new MySqlStreamingTableProvider(new MySqlTableOptions { ConnectionString = options.ConnectionString, + DataSource = dataSource, DatabaseName = table.DatabaseName, TableName = table.TableName, BatchSize = options.BatchSize, @@ -72,13 +71,22 @@ public static void RegisterMySql(this SessionContext context, string name, strin }); } + private static MySqlDataSource CreateDataSource(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a data source is not provided.", parameterName); + } + + return new MySqlDataSourceBuilder(connectionString).Build(); + } + private static IEnumerable FetchTables( - string connectionString, + MySqlDataSource dataSource, string? databaseName, bool includeViews) { - using MySqlConnection connection = new(connectionString); - connection.Open(); + using MySqlConnection connection = dataSource.OpenConnection(); using MySqlCommand command = connection.CreateCommand(); command.CommandText = """ diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs index 0142ccc..d232126 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs @@ -29,48 +29,31 @@ internal sealed class PostgreSqlArrowArrayStream : IArrowArrayStream private readonly Schema schema; private readonly ColumnPlan[] columns; private readonly int batchSize; + private readonly NpgsqlDataSource dataSource; + private readonly string query; + private readonly IReadOnlyList? parameters; private NpgsqlConnection? connection; private DbCommand? command; private DbDataReader? reader; private bool finished; public PostgreSqlArrowArrayStream( - string connectionString, + NpgsqlDataSource dataSource, string query, Schema schema, ColumnPlan[] columns, int batchSize, IReadOnlyList? parameters = null) { + // Defer opening the connection and issuing the query until the first + // batch is pulled. The schema is already known from registration, so the + // query cost shifts out of plan establishment into the first read. this.schema = schema; this.columns = columns; this.batchSize = batchSize; - - try - { - connection = new NpgsqlConnection(connectionString); - connection.Open(); - - command = connection.CreateCommand(); - command.CommandText = query; - if (parameters is not null) - { - foreach (SqlQueryParameter parameter in parameters) - { - DbParameter dbParameter = command.CreateParameter(); - dbParameter.ParameterName = parameter.Name; - dbParameter.Value = parameter.Value; - command.Parameters.Add(dbParameter); - } - } - - reader = command.ExecuteReader(CommandBehavior.SequentialAccess); - } - catch - { - Dispose(); - throw; - } + this.dataSource = dataSource; + this.query = query; + this.parameters = parameters; } public Schema Schema => schema; @@ -78,19 +61,21 @@ public PostgreSqlArrowArrayStream( public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (finished || reader is null) + if (finished) { return new ValueTask((RecordBatch?)null); } + DbDataReader activeReader = EnsureReader(); + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; - while (rowCount < batchSize && reader.Read()) + while (rowCount < batchSize && activeReader.Read()) { cancellationToken.ThrowIfCancellationRequested(); foreach (ColumnAppender appender in appenders) { - appender.Append(reader); + appender.Append(activeReader); } rowCount++; @@ -107,6 +92,40 @@ public PostgreSqlArrowArrayStream( return new ValueTask(batch); } + private DbDataReader EnsureReader() + { + if (reader is not null) + { + return reader; + } + + try + { + connection = dataSource.OpenConnection(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + return reader; + } + catch + { + Dispose(); + throw; + } + } + public void Dispose() { reader?.Dispose(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs index 4e53a06..047c0a7 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlDatabaseOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using Npgsql; + namespace Apache.DataFusion.TableProviders.PostgreSql; public sealed class PostgreSqlDatabaseOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public NpgsqlDataSource? DataSource { get; init; } public string? SourceName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs index 6671e8b..7c1ecf3 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlStreamingTableProvider.cs @@ -26,7 +26,7 @@ namespace Apache.DataFusion.TableProviders.PostgreSql; public sealed class PostgreSqlStreamingTableProvider : StreamingTableProvider { - private readonly string connectionString; + private readonly NpgsqlDataSource dataSource; private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; @@ -35,20 +35,15 @@ public sealed class PostgreSqlStreamingTableProvider : StreamingTableProvider public PostgreSqlStreamingTableProvider(PostgreSqlTableOptions options) { ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - connectionString = options.ConnectionString; + dataSource = options.DataSource ?? CreateDataSource(options.ConnectionString, nameof(options)); sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, sourceSql); + columns = FetchColumns(dataSource, sourceSql); Schema = BuildSchema(columns); } @@ -57,7 +52,7 @@ public PostgreSqlStreamingTableProvider(PostgreSqlTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new PostgreSqlArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + new PostgreSqlArrowArrayStream(dataSource, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { @@ -67,7 +62,7 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new PostgreSqlArrowArrayStream( - connectionString, + dataSource, pushedQuery.Sql, projectedSchema, projectedColumns, @@ -75,6 +70,16 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static NpgsqlDataSource CreateDataSource(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a data source is not provided.", parameterName); + } + + return NpgsqlDataSource.Create(connectionString); + } + private static string BuildSourceSql(PostgreSqlTableOptions options) { if (string.IsNullOrWhiteSpace(options.TableName)) @@ -89,10 +94,9 @@ private static string BuildSourceSql(PostgreSqlTableOptions options) private static string QuoteQualifiedTableName(string schemaName, string tableName) => $"{PostgreSqlDialect.Instance.QuoteIdentifier(schemaName)}.{PostgreSqlDialect.Instance.QuoteIdentifier(tableName)}"; - private static ColumnPlan[] FetchColumns(string connectionString, string query) + private static ColumnPlan[] FetchColumns(NpgsqlDataSource dataSource, string query) { - using NpgsqlConnection connection = new(connectionString); - connection.Open(); + using NpgsqlConnection connection = dataSource.OpenConnection(); using DbCommand command = connection.CreateCommand(); command.CommandText = query; diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs index 15c82c8..5997aa3 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlTableOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using Npgsql; + namespace Apache.DataFusion.TableProviders.PostgreSql; public sealed class PostgreSqlTableOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public NpgsqlDataSource? DataSource { get; init; } public string SchemaName { get; init; } = "public"; diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs index 68ad98b..001af26 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/SessionContextExtensions.cs @@ -35,10 +35,6 @@ public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDat { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } if (options.BatchSize <= 0) { @@ -54,12 +50,15 @@ public static void RegisterPostgreSql(this SessionContext context, PostgreSqlDat throw new ArgumentException("At least one schema must be provided.", nameof(options)); } - foreach (PostgreSqlTable table in FetchTables(options.ConnectionString, schemas, options.IncludeViews)) + NpgsqlDataSource dataSource = options.DataSource ?? CreateDataSource(options.ConnectionString, nameof(options)); + + foreach (PostgreSqlTable table in FetchTables(dataSource, schemas, options.IncludeViews)) { string registrationName = RegistrationName(table, schemas); RegisterSourceTable(context, options.SourceName, registrationName, new PostgreSqlStreamingTableProvider(new PostgreSqlTableOptions { ConnectionString = options.ConnectionString, + DataSource = dataSource, SchemaName = table.SchemaName, TableName = table.TableName, BatchSize = options.BatchSize, @@ -84,13 +83,22 @@ public static void RegisterPostgreSql(this SessionContext context, string name, }); } + private static NpgsqlDataSource CreateDataSource(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a data source is not provided.", parameterName); + } + + return NpgsqlDataSource.Create(connectionString); + } + private static IEnumerable FetchTables( - string connectionString, + NpgsqlDataSource dataSource, IReadOnlyCollection schemas, bool includeViews) { - using NpgsqlConnection connection = new(connectionString); - connection.Open(); + using NpgsqlConnection connection = dataSource.OpenConnection(); using DbCommand command = connection.CreateCommand(); command.CommandText = """ diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs index 7a5406b..020043a 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SessionContextExtensions.cs @@ -34,21 +34,20 @@ public static void RegisterSqlite(this SessionContext context, SqliteDatabaseOpt { ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - foreach (string tableName in FetchTables(options.ConnectionString, options.IncludeViews)) + Func connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options)); + + foreach (string tableName in FetchTables(connectionFactory, options.IncludeViews)) { RegisterSourceTable(context, options.SourceName, tableName, new SqliteStreamingTableProvider(new SqliteTableOptions { ConnectionString = options.ConnectionString, + ConnectionFactory = connectionFactory, TableName = tableName, BatchSize = options.BatchSize, })); @@ -72,9 +71,19 @@ public static void RegisterSqlite(this SessionContext context, string name, stri }); } - private static IEnumerable FetchTables(string connectionString, bool includeViews) + private static Func CreateConnectionFactory(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName); + } + + return () => new SqliteConnection(connectionString); + } + + private static IEnumerable FetchTables(Func connectionFactory, bool includeViews) { - using SqliteConnection connection = new(connectionString); + using SqliteConnection connection = connectionFactory(); connection.Open(); using DbCommand command = connection.CreateCommand(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs index 3851ed6..2fbaecb 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs @@ -29,48 +29,31 @@ internal sealed class SqliteArrowArrayStream : IArrowArrayStream private readonly Schema schema; private readonly ColumnPlan[] columns; private readonly int batchSize; + private readonly Func connectionFactory; + private readonly string query; + private readonly IReadOnlyList? parameters; private SqliteConnection? connection; private DbCommand? command; private DbDataReader? reader; private bool finished; public SqliteArrowArrayStream( - string connectionString, + Func connectionFactory, string query, Schema schema, ColumnPlan[] columns, int batchSize, IReadOnlyList? parameters = null) { + // Defer opening the connection and issuing the query until the first + // batch is pulled. The schema is already known from registration, so the + // query cost shifts out of plan establishment into the first read. this.schema = schema; this.columns = columns; this.batchSize = batchSize; - - try - { - connection = new SqliteConnection(connectionString); - connection.Open(); - - command = connection.CreateCommand(); - command.CommandText = query; - if (parameters is not null) - { - foreach (SqlQueryParameter parameter in parameters) - { - DbParameter dbParameter = command.CreateParameter(); - dbParameter.ParameterName = parameter.Name; - dbParameter.Value = parameter.Value; - command.Parameters.Add(dbParameter); - } - } - - reader = command.ExecuteReader(CommandBehavior.SequentialAccess); - } - catch - { - Dispose(); - throw; - } + this.connectionFactory = connectionFactory; + this.query = query; + this.parameters = parameters; } public Schema Schema => schema; @@ -78,19 +61,21 @@ public SqliteArrowArrayStream( public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - if (finished || reader is null) + if (finished) { return new ValueTask((RecordBatch?)null); } + DbDataReader activeReader = EnsureReader(); + ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; - while (rowCount < batchSize && reader.Read()) + while (rowCount < batchSize && activeReader.Read()) { cancellationToken.ThrowIfCancellationRequested(); foreach (ColumnAppender appender in appenders) { - appender.Append(reader); + appender.Append(activeReader); } rowCount++; @@ -107,6 +92,41 @@ public SqliteArrowArrayStream( return new ValueTask(batch); } + private DbDataReader EnsureReader() + { + if (reader is not null) + { + return reader; + } + + try + { + connection = connectionFactory(); + connection.Open(); + + command = connection.CreateCommand(); + command.CommandText = query; + if (parameters is not null) + { + foreach (SqlQueryParameter parameter in parameters) + { + DbParameter dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dbParameter.Value = parameter.Value; + command.Parameters.Add(dbParameter); + } + } + + reader = command.ExecuteReader(CommandBehavior.SequentialAccess); + return reader; + } + catch + { + Dispose(); + throw; + } + } + public void Dispose() { reader?.Dispose(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs index c9a696f..1416e7a 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteDatabaseOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using Microsoft.Data.Sqlite; + namespace Apache.DataFusion.TableProviders.Sqlite; public sealed class SqliteDatabaseOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public Func? ConnectionFactory { get; init; } public string? SourceName { get; init; } diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs index 9381fcf..6cff6f5 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteStreamingTableProvider.cs @@ -26,7 +26,7 @@ namespace Apache.DataFusion.TableProviders.Sqlite; public sealed class SqliteStreamingTableProvider : StreamingTableProvider { - private readonly string connectionString; + private readonly Func connectionFactory; private readonly string sourceSql; private readonly int batchSize; private readonly ColumnPlan[] columns; @@ -35,20 +35,15 @@ public sealed class SqliteStreamingTableProvider : StreamingTableProvider public SqliteStreamingTableProvider(SqliteTableOptions options) { ArgumentNullException.ThrowIfNull(options); - if (string.IsNullOrWhiteSpace(options.ConnectionString)) - { - throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options)); - } - if (options.BatchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero."); } - connectionString = options.ConnectionString; + connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options)); sourceSql = BuildSourceSql(options); batchSize = options.BatchSize; - columns = FetchColumns(connectionString, sourceSql); + columns = FetchColumns(connectionFactory, sourceSql); Schema = BuildSchema(columns); } @@ -57,7 +52,7 @@ public SqliteStreamingTableProvider(SqliteTableOptions options) public override bool SupportsPushdown => true; public override IArrowArrayStream Scan() => - new SqliteArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize); + new SqliteArrowArrayStream(connectionFactory, sourceSql, Schema, columns, batchSize); public override IArrowArrayStream Scan(StreamingTableScanRequest request) { @@ -67,7 +62,7 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) ColumnPlan[] projectedColumns = ProjectColumns(request); Schema projectedSchema = BuildSchema(projectedColumns); return new SqliteArrowArrayStream( - connectionString, + connectionFactory, pushedQuery.Sql, projectedSchema, projectedColumns, @@ -75,6 +70,16 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request) pushedQuery.Parameters); } + private static Func CreateConnectionFactory(string? connectionString, string parameterName) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName); + } + + return () => new SqliteConnection(connectionString); + } + private static string BuildSourceSql(SqliteTableOptions options) { if (string.IsNullOrWhiteSpace(options.TableName)) @@ -85,9 +90,9 @@ private static string BuildSourceSql(SqliteTableOptions options) return $"SELECT * FROM {SqliteDialect.Instance.QuoteIdentifier(options.TableName)}"; } - private static ColumnPlan[] FetchColumns(string connectionString, string query) + private static ColumnPlan[] FetchColumns(Func connectionFactory, string query) { - using SqliteConnection connection = new(connectionString); + using SqliteConnection connection = connectionFactory(); connection.Open(); using DbCommand command = connection.CreateCommand(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs index 28773a3..c65ea7a 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteTableOptions.cs @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. +using Microsoft.Data.Sqlite; + namespace Apache.DataFusion.TableProviders.Sqlite; public sealed class SqliteTableOptions { - public required string ConnectionString { get; init; } + public string? ConnectionString { get; init; } + + public Func? ConnectionFactory { get; init; } public required string TableName { get; init; } diff --git a/src/Apache.DataFusion/DataFrame/DataFrame.Execution.cs b/src/Apache.DataFusion/DataFrame/DataFrame.Execution.cs index cf4c157..9416815 100644 --- a/src/Apache.DataFusion/DataFrame/DataFrame.Execution.cs +++ b/src/Apache.DataFusion/DataFrame/DataFrame.Execution.cs @@ -62,12 +62,21 @@ internal byte[] CollectIpcBytes(CancellationToken cancellationToken) public ArrowBatchReader ExecuteStream() => ExecuteStream(CancellationToken.None); /// - /// Execute the plan and materialize the result with cooperative - /// cancellation. Firing from another - /// thread aborts the in-flight query at its next poll point and throws - /// . A token that cannot be - /// cancelled (e.g. ) behaves exactly - /// like the parameterless overload. + /// Execute the plan and return its record batches as a streaming reader. + /// Unlike , the full result set is + /// not buffered in native memory: each call to + /// drives one batch + /// on the native side, so memory stays bounded to a single in-flight batch. + /// + /// is honoured while the stream is + /// being established: a token that is already cancelled aborts with + /// before any work runs. Once the + /// reader is returned, cancellation of subsequent batch reads is driven by + /// the token passed to + /// . A token that + /// cannot be cancelled (e.g. ) behaves + /// exactly like the parameterless overload. + /// /// public ArrowBatchReader ExecuteStream(CancellationToken cancellationToken) {