Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 53 additions & 10 deletions native/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use std::os::raw::{c_char, c_int, c_uchar};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::{Arc, OnceLock};

use ::arrow::array::{RecordBatch, RecordBatchIterator};
use ::arrow::array::{RecordBatch, RecordBatchIterator, RecordBatchReader};
use ::arrow::datatypes::SchemaRef;
use ::arrow::error::ArrowError;
use ::arrow::ffi_stream::FFI_ArrowArrayStream;
Expand All @@ -43,6 +43,7 @@ use datafusion::error::DataFusionError;
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool};
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::SendableRecordBatchStream;
use datafusion::logical_expr::{col, Expr, Partitioning, SortExpr};
use datafusion::prelude::SessionConfig;
use futures::StreamExt;
Expand Down Expand Up @@ -823,6 +824,52 @@ fn export_batches(
Ok(())
}

/// Adapts a DataFusion `SendableRecordBatchStream` into a synchronous
/// `RecordBatchReader` so it can be handed to an `FFI_ArrowArrayStream` and
/// pulled lazily from the managed side. Each managed pull drives a single
/// `stream.next()` on the shared Tokio runtime, so native memory stays bounded
/// by one in-flight batch instead of the entire result set.
struct BlockingStreamReader {
schema: SchemaRef,
stream: SendableRecordBatchStream,
}

impl Iterator for BlockingStreamReader {
type Item = Result<RecordBatch, ArrowError>;

fn next(&mut self) -> Option<Self::Item> {
runtime()
.block_on(self.stream.next())
.map(|result| result.map_err(|err| ArrowError::ExternalError(Box::new(err))))
}
}

impl RecordBatchReader for BlockingStreamReader {
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
}

/// Export a streaming `SendableRecordBatchStream` through the caller-provided
/// Arrow C Data Interface pointer. Unlike `export_batches`, the result set is
/// never collected up front: batches are produced one at a time as the managed
/// reader advances, keeping native memory bounded to a single in-flight batch.
fn export_stream(
out: *mut FFI_ArrowArrayStream,
schema: SchemaRef,
stream: SendableRecordBatchStream,
) -> NativeResult<()> {
if out.is_null() {
return Err("output stream pointer is null".into());
}
let reader = BlockingStreamReader { schema, stream };
let ffi = FFI_ArrowArrayStream::new(Box::new(reader));
unsafe {
std::ptr::write(out, ffi);
}
Ok(())
}

#[no_mangle]
pub extern "C" fn df_dataframe_collect_cdata(
handle: *mut DataFrame,
Expand All @@ -848,15 +895,11 @@ pub extern "C" fn df_dataframe_execute_stream_cdata(
let df = unsafe { *Box::from_raw(require_ptr(handle, "DataFrame handle")?) };
let token = resolve_token(token_handle);
let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone());
let batches = runtime().block_on(run_cancellable(&token, async move {
let mut stream = df.execute_stream().await?;
let mut batches = Vec::new();
while let Some(batch) = stream.next().await {
batches.push(batch?);
}
Ok::<_, DataFusionError>(batches)
}))?;
export_batches(out, schema, batches)
// Only establish the stream here (honouring a pre-cancelled token); the
// batches themselves are pulled lazily by the managed reader so the full
// result set is never buffered in native memory at once.
let stream = runtime().block_on(run_cancellable(&token, df.execute_stream()))?;
export_stream(out, schema, stream)
})
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,68 +29,55 @@ internal sealed class ClickHouseArrowArrayStream : IArrowArrayStream
private readonly Schema schema;
private readonly ColumnPlan[] columns;
private readonly int batchSize;
private readonly Func<ClickHouseConnection> connectionFactory;
private readonly string query;
private readonly IReadOnlyList<ClickHouseQueryParameter>? parameters;
private ClickHouseConnection? connection;
private DbCommand? command;
private DbDataReader? reader;
private bool finished;

public ClickHouseArrowArrayStream(
string connectionString,
Func<ClickHouseConnection> connectionFactory,
string query,
Schema schema,
ColumnPlan[] columns,
int batchSize,
IReadOnlyList<ClickHouseQueryParameter>? parameters = null)
{
// Defer opening the connection and issuing the query until the first
// batch is pulled. The schema is already known from registration, so a
// consumer that only inspects the schema never pays for a round trip,
// and the query cost shifts out of plan establishment into the first
// read.
this.schema = schema;
this.columns = columns;
this.batchSize = batchSize;

try
{
connection = new ClickHouseConnection(connectionString);
connection.Open();

command = connection.CreateCommand();
command.CommandText = query;
if (parameters is not null)
{
foreach (ClickHouseQueryParameter parameter in parameters)
{
DbParameter dbParameter = command.CreateParameter();
dbParameter.ParameterName = parameter.Name;
dbParameter.Value = parameter.Value;
command.Parameters.Add(dbParameter);
}
}

reader = command.ExecuteReader(CommandBehavior.SequentialAccess);
}
catch
{
Dispose();
throw;
}
this.connectionFactory = connectionFactory;
this.query = query;
this.parameters = parameters;
}

public Schema Schema => schema;

public ValueTask<RecordBatch?> ReadNextRecordBatchAsync(CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
if (finished || reader is null)
if (finished)
{
return new ValueTask<RecordBatch?>((RecordBatch?)null);
}

DbDataReader activeReader = EnsureReader();

ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray();
int rowCount = 0;
while (rowCount < batchSize && reader.Read())
while (rowCount < batchSize && activeReader.Read())
{
cancellationToken.ThrowIfCancellationRequested();
foreach (ColumnAppender appender in appenders)
{
appender.Append(reader);
appender.Append(activeReader);
}

rowCount++;
Expand All @@ -107,6 +94,41 @@ public ClickHouseArrowArrayStream(
return new ValueTask<RecordBatch?>(batch);
}

private DbDataReader EnsureReader()
{
if (reader is not null)
{
return reader;
}

try
{
connection = connectionFactory();
connection.Open();

command = connection.CreateCommand();
command.CommandText = query;
if (parameters is not null)
{
foreach (ClickHouseQueryParameter parameter in parameters)
{
DbParameter dbParameter = command.CreateParameter();
dbParameter.ParameterName = parameter.Name;
dbParameter.Value = parameter.Value;
command.Parameters.Add(dbParameter);
}
}

reader = command.ExecuteReader(CommandBehavior.SequentialAccess);
return reader;
}
catch
{
Dispose();
throw;
}
}

public void Dispose()
{
reader?.Dispose();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@
// specific language governing permissions and limitations
// under the License.

using ClickHouse.Driver.ADO;

namespace Apache.DataFusion.TableProviders.ClickHouse;

public sealed class ClickHouseDatabaseOptions
{
public required string ConnectionString { get; init; }
public string? ConnectionString { get; init; }

public Func<ClickHouseConnection>? ConnectionFactory { get; init; }

public string? SourceName { get; init; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace Apache.DataFusion.TableProviders.ClickHouse;

public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider
{
private readonly string connectionString;
private readonly Func<ClickHouseConnection> connectionFactory;
private readonly string sourceSql;
private readonly int batchSize;
private readonly ColumnPlan[] columns;
Expand All @@ -35,20 +35,15 @@ public sealed class ClickHouseStreamingTableProvider : StreamingTableProvider
public ClickHouseStreamingTableProvider(ClickHouseTableOptions options)
{
ArgumentNullException.ThrowIfNull(options);
if (string.IsNullOrWhiteSpace(options.ConnectionString))
{
throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
}

if (options.BatchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
}

connectionString = options.ConnectionString;
connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options));
sourceSql = BuildSourceSql(options);
batchSize = options.BatchSize;
columns = FetchColumns(connectionString, sourceSql);
columns = FetchColumns(connectionFactory, sourceSql);
Schema = BuildSchema(columns);
}

Expand All @@ -57,7 +52,7 @@ public ClickHouseStreamingTableProvider(ClickHouseTableOptions options)
public override bool SupportsPushdown => true;

public override IArrowArrayStream Scan() =>
new ClickHouseArrowArrayStream(connectionString, sourceSql, Schema, columns, batchSize);
new ClickHouseArrowArrayStream(connectionFactory, sourceSql, Schema, columns, batchSize);

public override IArrowArrayStream Scan(StreamingTableScanRequest request)
{
Expand All @@ -67,14 +62,29 @@ public override IArrowArrayStream Scan(StreamingTableScanRequest request)
ColumnPlan[] projectedColumns = ProjectColumns(request);
Schema projectedSchema = BuildSchema(projectedColumns);
return new ClickHouseArrowArrayStream(
connectionString,
connectionFactory,
pushedQuery.Sql,
projectedSchema,
projectedColumns,
batchSize,
pushedQuery.Parameters);
}

private static Func<ClickHouseConnection> CreateConnectionFactory(string? connectionString, string parameterName)
{
if (string.IsNullOrWhiteSpace(connectionString))
{
throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName);
}

// A ClickHouseDataSource owns a single underlying client, so every
// connection it hands out shares the same HTTP connection pool. This
// avoids a fresh TLS handshake on each scan, unlike constructing a
// standalone ClickHouseConnection per call.
ClickHouseDataSource dataSource = new(connectionString);
return dataSource.CreateConnection;
}

private static string BuildSourceSql(ClickHouseTableOptions options)
{
if (string.IsNullOrWhiteSpace(options.TableName))
Expand All @@ -93,9 +103,9 @@ private static string QuoteQualifiedTableName(string? databaseName, string table
: $"{ClickHouseDialect.Instance.QuoteIdentifier(databaseName)}.{quotedTable}";
}

private static ColumnPlan[] FetchColumns(string connectionString, string query)
private static ColumnPlan[] FetchColumns(Func<ClickHouseConnection> connectionFactory, string query)
{
using ClickHouseConnection connection = new(connectionString);
using ClickHouseConnection connection = connectionFactory();
connection.Open();

using DbCommand command = connection.CreateCommand();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@
// specific language governing permissions and limitations
// under the License.

using ClickHouse.Driver.ADO;

namespace Apache.DataFusion.TableProviders.ClickHouse;

public sealed class ClickHouseTableOptions
{
public required string ConnectionString { get; init; }
public string? ConnectionString { get; init; }

public Func<ClickHouseConnection>? ConnectionFactory { get; init; }

public string? DatabaseName { get; init; }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,21 +34,20 @@ public static void RegisterClickHouse(this SessionContext context, ClickHouseDat
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(options);
if (string.IsNullOrWhiteSpace(options.ConnectionString))
{
throw new ArgumentException("Connection string cannot be null or whitespace.", nameof(options));
}

if (options.BatchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(options), options.BatchSize, "Batch size must be greater than zero.");
}

foreach (ClickHouseTable table in FetchTables(options.ConnectionString, options.DatabaseName, options.IncludeViews))
Func<ClickHouseConnection> connectionFactory = options.ConnectionFactory ?? CreateConnectionFactory(options.ConnectionString, nameof(options));

foreach (ClickHouseTable table in FetchTables(connectionFactory, options.DatabaseName, options.IncludeViews))
{
RegisterSourceTable(context, options.SourceName, table.TableName, new ClickHouseStreamingTableProvider(new ClickHouseTableOptions
{
ConnectionString = options.ConnectionString,
ConnectionFactory = connectionFactory,
DatabaseName = table.DatabaseName,
TableName = table.TableName,
BatchSize = options.BatchSize,
Expand All @@ -73,12 +72,26 @@ public static void RegisterClickHouse(this SessionContext context, string name,
});
}

private static Func<ClickHouseConnection> CreateConnectionFactory(string? connectionString, string parameterName)
{
if (string.IsNullOrWhiteSpace(connectionString))
{
throw new ArgumentException("Connection string cannot be null or whitespace when a connection factory is not provided.", parameterName);
}

// Share one ClickHouseDataSource across every table registered from this
// database so all scans reuse a single HTTP connection pool instead of
// re-establishing a TLS connection on each scan.
ClickHouseDataSource dataSource = new(connectionString);
return dataSource.CreateConnection;
}

private static IEnumerable<ClickHouseTable> FetchTables(
string connectionString,
Func<ClickHouseConnection> connectionFactory,
string? databaseName,
bool includeViews)
{
using ClickHouseConnection connection = new(connectionString);
using ClickHouseConnection connection = connectionFactory();
connection.Open();

using DbCommand command = connection.CreateCommand();
Expand Down
Loading
Loading