From c890c1ea78f3afed5ba7b74bd423a12d0efbc47b Mon Sep 17 00:00:00 2001 From: lqlive Date: Thu, 18 Jun 2026 15:29:49 +0800 Subject: [PATCH] Use typed readers in provider appenders Avoid generic DbDataReader.GetValue conversion in database table providers so row-to-Arrow conversion follows each driver's typed reader path. Co-authored-by: Cursor --- .../BooleanColumnAppender.cs | 3 +-- .../DoubleColumnAppender.cs | 3 +-- .../FloatColumnAppender.cs | 3 +-- .../Int32ColumnAppender.cs | 3 +-- .../Int64ColumnAppender.cs | 3 +-- .../StringColumnAppender.cs | 2 +- .../BooleanColumnAppender.cs | 7 +++---- .../ColumnAppender.cs | 4 ++-- .../DoubleColumnAppender.cs | 7 +++---- .../FloatColumnAppender.cs | 7 +++---- .../Int32ColumnAppender.cs | 7 +++---- .../Int64ColumnAppender.cs | 7 +++---- .../MySqlArrowArrayStream.cs | 7 +++---- .../StringColumnAppender.cs | 6 +++--- .../BooleanColumnAppender.cs | 7 +++---- .../ColumnAppender.cs | 4 ++-- .../DoubleColumnAppender.cs | 7 +++---- .../FloatColumnAppender.cs | 7 +++---- .../Int32ColumnAppender.cs | 7 +++---- .../Int64ColumnAppender.cs | 7 +++---- .../PostgreSqlArrowArrayStream.cs | 8 ++++---- .../StringColumnAppender.cs | 6 +++--- .../BooleanColumnAppender.cs | 7 +++---- .../ColumnAppender.cs | 4 ++-- .../DoubleColumnAppender.cs | 7 +++---- .../FloatColumnAppender.cs | 7 +++---- .../Int32ColumnAppender.cs | 7 +++---- .../Int64ColumnAppender.cs | 7 +++---- .../SqliteArrowArrayStream.cs | 9 ++++----- .../StringColumnAppender.cs | 6 +++--- 30 files changed, 77 insertions(+), 99 deletions(-) diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs index 3d6832d..b8c2081 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/BooleanColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; -using System.Globalization; using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetBoolean(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs index 550a549..1a86458 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/DoubleColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; -using System.Globalization; using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetDouble(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs index a80f96e..0c50021 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/FloatColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; -using System.Globalization; using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetFloat(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs index c47ea6e..eb00e39 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int32ColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; -using System.Globalization; using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt32(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs index a100bf2..d05a296 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/Int64ColumnAppender.cs @@ -16,7 +16,6 @@ // under the License. using System.Data.Common; -using System.Globalization; using Apache.Arrow; namespace Apache.DataFusion.TableProviders.ClickHouse; @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt64(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs index e2d3092..6677fc2 100644 --- a/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.ClickHouse/StringColumnAppender.cs @@ -32,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(FormatValue(reader.GetValue(Ordinal))); + builder.Append(reader.GetString(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs index b66fa20..1ac132a 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/BooleanColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +24,7 @@ internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordina { private readonly BooleanArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetBoolean(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs index 114b66f..bb7b02c 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +25,7 @@ internal abstract class ColumnAppender(int ordinal) { protected int Ordinal { get; } = ordinal; - public abstract void Append(DbDataReader reader); + public abstract void Append(MySqlDataReader reader); public abstract IArrowArray Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs index 2c63817..c50bc34 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/DoubleColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +24,7 @@ internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly DoubleArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetDouble(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs index 5f23fc3..083c5d8 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/FloatColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +24,7 @@ internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly FloatArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetFloat(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs index d3e4675..cba0c94 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int32ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +24,7 @@ internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int32Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt32(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs index d2e8d29..be82b1f 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/Int64ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -25,7 +24,7 @@ internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int64Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt64(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs index 3e974ea..c45515a 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/MySqlArrowArrayStream.cs @@ -16,7 +16,6 @@ // under the License. using System.Data; -using System.Data.Common; using Apache.Arrow; using Apache.Arrow.Ipc; using Apache.DataFusion.TableProviders.MySql.Sql; @@ -34,7 +33,7 @@ internal sealed class MySqlArrowArrayStream : IArrowArrayStream private readonly IReadOnlyList? parameters; private MySqlConnection? connection; private MySqlCommand? command; - private DbDataReader? reader; + private MySqlDataReader? reader; private bool finished; public MySqlArrowArrayStream( @@ -66,7 +65,7 @@ public MySqlArrowArrayStream( return new ValueTask((RecordBatch?)null); } - DbDataReader activeReader = EnsureReader(); + MySqlDataReader activeReader = EnsureReader(); ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; @@ -92,7 +91,7 @@ public MySqlArrowArrayStream( return new ValueTask(batch); } - private DbDataReader EnsureReader() + private MySqlDataReader EnsureReader() { if (reader is not null) { diff --git a/src/Apache.DataFusion.TableProviders.MySql/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.MySql/StringColumnAppender.cs index 4fa7971..80fc256 100644 --- a/src/Apache.DataFusion.TableProviders.MySql/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.MySql/StringColumnAppender.cs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using Apache.Arrow; +using MySqlConnector; namespace Apache.DataFusion.TableProviders.MySql; @@ -24,7 +24,7 @@ internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly StringArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(MySqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -32,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(FormatValue(reader.GetValue(Ordinal))); + builder.Append(reader.GetString(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs index c7bc593..903f3e7 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/BooleanColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +24,7 @@ internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordina { private readonly BooleanArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetBoolean(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs index bba9402..a6694aa 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +25,7 @@ internal abstract class ColumnAppender(int ordinal) { protected int Ordinal { get; } = ordinal; - public abstract void Append(DbDataReader reader); + public abstract void Append(NpgsqlDataReader reader); public abstract IArrowArray Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs index 4366532..e0f53d9 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/DoubleColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +24,7 @@ internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly DoubleArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetDouble(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs index 9ed65ce..95f3542 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/FloatColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +24,7 @@ internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly FloatArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetFloat(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs index de9bad3..b77cdaf 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int32ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +24,7 @@ internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int32Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt32(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs index 2693520..692a5c4 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/Int64ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -25,7 +24,7 @@ internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int64Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt64(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs index d232126..f6eaf25 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/PostgreSqlArrowArrayStream.cs @@ -33,8 +33,8 @@ internal sealed class PostgreSqlArrowArrayStream : IArrowArrayStream private readonly string query; private readonly IReadOnlyList? parameters; private NpgsqlConnection? connection; - private DbCommand? command; - private DbDataReader? reader; + private NpgsqlCommand? command; + private NpgsqlDataReader? reader; private bool finished; public PostgreSqlArrowArrayStream( @@ -66,7 +66,7 @@ public PostgreSqlArrowArrayStream( return new ValueTask((RecordBatch?)null); } - DbDataReader activeReader = EnsureReader(); + NpgsqlDataReader activeReader = EnsureReader(); ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; @@ -92,7 +92,7 @@ public PostgreSqlArrowArrayStream( return new ValueTask(batch); } - private DbDataReader EnsureReader() + private NpgsqlDataReader EnsureReader() { if (reader is not null) { diff --git a/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs index b8cdf90..4cdaa62 100644 --- a/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.PostgreSql/StringColumnAppender.cs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using Apache.Arrow; +using Npgsql; namespace Apache.DataFusion.TableProviders.PostgreSql; @@ -24,7 +24,7 @@ internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly StringArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(NpgsqlDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -32,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(FormatValue(reader.GetValue(Ordinal))); + builder.Append(reader.GetString(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs index 4f290ec..cae6879 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/BooleanColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +24,7 @@ internal sealed class BooleanColumnAppender(int ordinal) : ColumnAppender(ordina { private readonly BooleanArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToBoolean(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetBoolean(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs index 9cdc462..e69a682 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/ColumnAppender.cs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +25,7 @@ internal abstract class ColumnAppender(int ordinal) { protected int Ordinal { get; } = ordinal; - public abstract void Append(DbDataReader reader); + public abstract void Append(SqliteDataReader reader); public abstract IArrowArray Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs index afcf9d4..d4427f7 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/DoubleColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +24,7 @@ internal sealed class DoubleColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly DoubleArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToDouble(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetDouble(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs index c25d5fb..55ca3da 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/FloatColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +24,7 @@ internal sealed class FloatColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly FloatArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToSingle(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetFloat(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs index f68a3c5..ac9fd47 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int32ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +24,7 @@ internal sealed class Int32ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int32Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt32(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt32(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs index 1b0a7fc..ff8ef06 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/Int64ColumnAppender.cs @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; -using System.Globalization; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -25,7 +24,7 @@ internal sealed class Int64ColumnAppender(int ordinal) : ColumnAppender(ordinal) { private readonly Int64Array.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -33,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(Convert.ToInt64(reader.GetValue(Ordinal), CultureInfo.InvariantCulture)); + builder.Append(reader.GetInt64(Ordinal)); } public override IArrowArray Build() => builder.Build(); diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs index 2fbaecb..4435a40 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/SqliteArrowArrayStream.cs @@ -33,8 +33,8 @@ internal sealed class SqliteArrowArrayStream : IArrowArrayStream private readonly string query; private readonly IReadOnlyList? parameters; private SqliteConnection? connection; - private DbCommand? command; - private DbDataReader? reader; + private SqliteCommand? command; + private SqliteDataReader? reader; private bool finished; public SqliteArrowArrayStream( @@ -66,8 +66,7 @@ public SqliteArrowArrayStream( return new ValueTask((RecordBatch?)null); } - DbDataReader activeReader = EnsureReader(); - + SqliteDataReader activeReader = EnsureReader(); ColumnAppender[] appenders = columns.Select(static column => column.CreateAppender()).ToArray(); int rowCount = 0; while (rowCount < batchSize && activeReader.Read()) @@ -92,7 +91,7 @@ public SqliteArrowArrayStream( return new ValueTask(batch); } - private DbDataReader EnsureReader() + private SqliteDataReader EnsureReader() { if (reader is not null) { diff --git a/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs index ff4a586..0907da7 100644 --- a/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs +++ b/src/Apache.DataFusion.TableProviders.Sqlite/StringColumnAppender.cs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -using System.Data.Common; using Apache.Arrow; +using Microsoft.Data.Sqlite; namespace Apache.DataFusion.TableProviders.Sqlite; @@ -24,7 +24,7 @@ internal sealed class StringColumnAppender(int ordinal) : ColumnAppender(ordinal { private readonly StringArray.Builder builder = new(); - public override void Append(DbDataReader reader) + public override void Append(SqliteDataReader reader) { if (reader.IsDBNull(Ordinal)) { @@ -32,7 +32,7 @@ public override void Append(DbDataReader reader) return; } - builder.Append(FormatValue(reader.GetValue(Ordinal))); + builder.Append(reader.GetString(Ordinal)); } public override IArrowArray Build() => builder.Build();