From 933d752fc692ae846982d3cbfb31845f0fff9291 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Tue, 21 Apr 2026 14:48:48 +0200 Subject: [PATCH 01/43] modifiche per statistics --- DuckDB.NET.Bindings/DuckDBWrapperObjects.cs | 35 ++- .../NativeMethods.ProfilingInfo.cs | 31 +++ .../NativeMethods/NativeMethods.Value.cs | 25 +++ DuckDB.NET.Data/Common/TimerUtils.cs | 22 ++ DuckDB.NET.Data/Connection/SqlStatistics.cs | 115 ++++++++++ DuckDB.NET.Data/DuckDBCommand.cs | 4 +- DuckDB.NET.Data/DuckDBConnection.cs | 93 +++++++- .../PreparedStatement/ClrToDuckDBConverter.cs | 24 ++ .../PreparedStatement/PreparedStatement.cs | 16 +- .../Profiling/DuckDBProfilingInfoWrapper.cs | 63 ++++++ DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 77 +++++++ DuckDB.NET.Samples/Program.cs | 128 ++++++++++- .../Parameters/MapParameterTests.cs | 205 ++++++++++++++++++ 13 files changed, 823 insertions(+), 15 deletions(-) create mode 100644 DuckDB.NET.Bindings/NativeMethods/NativeMethods.ProfilingInfo.cs create mode 100644 DuckDB.NET.Data/Common/TimerUtils.cs create mode 100644 DuckDB.NET.Data/Connection/SqlStatistics.cs create mode 100644 DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs create mode 100644 DuckDB.NET.Data/Profiling/ProfilingInfo.cs create mode 100644 DuckDB.NET.Test/Parameters/MapParameterTests.cs diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index e42f90c6..b4e0e809 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -1,4 +1,6 @@ -namespace DuckDB.NET.Native; +using System.Collections.Generic; + +namespace DuckDB.NET.Native; public class DuckDBDatabase() : SafeHandleZeroOrMinusOneIsInvalid(true) { @@ -59,6 +61,17 @@ protected override bool ReleaseHandle() } } +public class DuckDBProfilingInfo() : SafeHandleZeroOrMinusOneIsInvalid(true) +{ + // No explicit destroy/free function for duckdb_profiling_info in the C API as of now. + // If DuckDB adds a destroy function in the future, call it here. + protected override bool ReleaseHandle() + { + // No-op: DuckDB does not require explicit destruction of profiling info handles. + return true; + } +} + public class DuckDBLogicalType() : SafeHandleZeroOrMinusOneIsInvalid(true) { protected override bool ReleaseHandle() @@ -160,6 +173,25 @@ public T GetValue() [MethodImpl(MethodImplOptions.AggressiveInlining)] static T Cast(TSource value) => Unsafe.As(ref value); } + public Dictionary GetMapValue() where TKey : notnull + { + var result = new Dictionary(); + + ulong entryCount = NativeMethods.Value.DuckDBGetMapSize(this); + for (ulong i = 0; i < entryCount; i++) + { + using var key = NativeMethods.Value.DuckDBGetMapKey(this, i); + using var value = NativeMethods.Value.DuckDBGetMapValue(this, i); + + var keyValue = key.GetValue(); + var valueValue = value.GetValue(); + + result.Add(keyValue, valueValue); + } + + return result; + + } private DateTime GetTimestampValue(DuckDBTimestampStruct timestampStruct, DuckDBType duckDBType) { @@ -194,6 +226,7 @@ private DateTimeOffset GetTimeTzValue() var timeTz = NativeMethods.DateTimeHelpers.DuckDBFromTimeTz(timeTzStruct); return new DateTimeOffset(timeTz.Time.ToDateTime(), TimeSpan.FromSeconds(timeTz.Offset)); } + } public class DuckDBClientContext() : SafeHandleZeroOrMinusOneIsInvalid(true) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ProfilingInfo.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ProfilingInfo.cs new file mode 100644 index 00000000..b170dc9b --- /dev/null +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ProfilingInfo.cs @@ -0,0 +1,31 @@ + +namespace DuckDB.NET.Native; + +public partial class NativeMethods +{ + //https://duckdb.org/docs/current/clients/c/api#profiling-info + public static partial class ProfilingInfo + { + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_get_profiling_info")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBProfilingInfo DuckDBGetProfilingInfo(DuckDBNativeConnection connection); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_profiling_info_get_value")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBValue DuckDBProfilingInfoGetValue(DuckDBProfilingInfo info, [MarshalAs(UnmanagedType.LPStr)] string key); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_profiling_info_get_metrics")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBValue DuckDBProfilingInfoGetMetrics(DuckDBProfilingInfo info); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_profiling_info_get_child_count")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial ulong DuckDBProfilingInfoGetChildCount(DuckDBProfilingInfo info); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_profiling_info_get_child")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBProfilingInfo DuckDBProfilingInfoGetChild(DuckDBProfilingInfo info, ulong index); + + } +} \ No newline at end of file diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs index 518fe61f..25b9cda5 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs @@ -215,6 +215,18 @@ public static partial class Value [return: MarshalUsing(typeof(DuckDBCallerOwnedStringMarshaller))] public static partial string DuckDBGetVarchar(DuckDBValue value); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_get_map_size")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial ulong DuckDBGetMapSize(DuckDBValue value); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_get_map_key")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBValue DuckDBGetMapKey(DuckDBValue value, ulong index); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_get_map_value")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBValue DuckDBGetMapValue(DuckDBValue value, ulong index); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_create_list_value")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBValue DuckDBCreateListValue(DuckDBLogicalType logicalType, IntPtr[] values, long count); @@ -223,6 +235,10 @@ public static partial class Value [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBValue DuckDBCreateArrayValue(DuckDBLogicalType logicalType, IntPtr[] values, long count); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_create_map_value")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBValue DuckDBCreateMapValue(DuckDBLogicalType logicalType, IntPtr[] keys, IntPtr[] values, long count); + // Maybe [SuppressGCTransition]: new Value — one small allocation [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_create_null_value")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] @@ -251,5 +267,14 @@ public static DuckDBValue DuckDBCreateArrayValue(DuckDBLogicalType logicalType, return duckDBValue; } + + public static DuckDBValue DuckDBCreateMapValue(DuckDBLogicalType logicalType, DuckDBValue[] keys, DuckDBValue[] values, int count) + { + var duckDBValue = DuckDBCreateMapValue(logicalType, keys.Select(item => item.DangerousGetHandle()).ToArray(), values.Select(item => item.DangerousGetHandle()).ToArray(), count); + + duckDBValue.SetChildValues(values); + + return duckDBValue; + } } } diff --git a/DuckDB.NET.Data/Common/TimerUtils.cs b/DuckDB.NET.Data/Common/TimerUtils.cs new file mode 100644 index 00000000..285db04a --- /dev/null +++ b/DuckDB.NET.Data/Common/TimerUtils.cs @@ -0,0 +1,22 @@ +namespace DuckDB.NET.Data.Common +{ + internal sealed class TimerUtils + { + + internal static long TimerCurrent() => DateTimeOffset.UtcNow.UtcTicks; + + internal static DateTimeOffset Now() => DateTimeOffset.UtcNow; + + internal static uint CalculateTickCountElapsed(long startTick, long endTick) + { + + return (uint)(endTick - startTick); + } + + internal static long TimerToMilliseconds(long timerValue) + { + long result = timerValue / TimeSpan.TicksPerMillisecond; + return result; + } + } +} diff --git a/DuckDB.NET.Data/Connection/SqlStatistics.cs b/DuckDB.NET.Data/Connection/SqlStatistics.cs new file mode 100644 index 00000000..ee5902c9 --- /dev/null +++ b/DuckDB.NET.Data/Connection/SqlStatistics.cs @@ -0,0 +1,115 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; + +namespace DuckDB.NET.Data.Connection +{ + internal sealed class SqlStatistics + { + + // internal values that are not exposed through properties + internal long closeTimestamp; + internal long openTimestamp; + internal long? startExecutionTimestamp; + + // internal values that are exposed through properties + internal long executionTime; + internal long connectionTime; + internal DateTimeOffset startExecutionTime; + internal DateTimeOffset endExecutionTime; + internal Dictionary> metrics = []; + + internal SqlStatistics() + { + } + + internal static SqlStatistics StartTimer(SqlStatistics statistics) + { + if (statistics != null && !statistics.RequestExecutionTimer()) + { + // we're re-entrant -- don't bother. + statistics = null; + } + return statistics; + } + + internal static void StopTimer(SqlStatistics statistics) + { + if (statistics != null) + { + statistics.ReleaseAndUpdateExecutionTimer(); + } + } + + internal bool RequestExecutionTimer() + { + if (!startExecutionTimestamp.HasValue) + { + startExecutionTimestamp = TimerUtils.TimerCurrent(); + startExecutionTime = TimerUtils.Now(); + return true; + } + return false; + } + + internal void ReleaseAndUpdateExecutionTimer() + { + if (startExecutionTimestamp.HasValue) + { + uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + executionTime += elapsed; + endExecutionTime = startExecutionTime.AddTicks(elapsed); + + startExecutionTimestamp = null; + } + } + + internal void UpdateStatistics() + { + // update connection time + if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) + { + connectionTime = closeTimestamp - openTimestamp; + } + else + { + connectionTime = long.MaxValue; + } + } + + internal void ReadMetrics(DuckDBNativeConnection connection, int index) + { + var profile = new ProfilingInfo(connection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + metrics[index] = curMetrics; + } + } + + internal IDictionary GetDictionary() + { + const int Count = 18; + var dictionary = new Dictionary(Count) + { + { "StartTime", startExecutionTime }, + { "EndTime", endExecutionTime }, + { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, + { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, + { "MetricsCount", metrics.Count }, + { "Metrics", metrics } + }; + Debug.Assert(dictionary.Count == Count); + return dictionary; + } + + internal void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + connectionTime = 0; + startExecutionTime = default; + endExecutionTime = default; + } + } +} diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index a0b67b3c..d9b41d8f 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -66,7 +66,7 @@ public override int ExecuteNonQuery() { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); + var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!, CommandText, parameters, UseStreamingMode); var count = 0; @@ -102,7 +102,7 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); + var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!, CommandText, parameters, UseStreamingMode); var reader = new DuckDBDataReader(this, results, behavior); diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 54bfbbe6..609ad1ed 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -1,7 +1,10 @@ -using DuckDB.NET.Data.Connection; +using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.Connection; using System.ComponentModel; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.Arm; namespace DuckDB.NET.Data; @@ -12,10 +15,13 @@ public partial class DuckDBConnection : DbConnection private DuckDBConnectionString? parsedConnection; private ConnectionReference? connectionReference; private bool inMemoryDuplication = false; - private static readonly StateChangeEventArgs FromClosedToOpenEventArgs = new(ConnectionState.Closed, ConnectionState.Open); private static readonly StateChangeEventArgs FromOpenToClosedEventArgs = new(ConnectionState.Open, ConnectionState.Closed); + // Statistics support + internal SqlStatistics statistics; + private bool collectstats; + #region Protected Properties protected override DbProviderFactory? DbProviderFactory => DuckDBClientFactory.Instance; @@ -26,6 +32,11 @@ public partial class DuckDBConnection : DbConnection internal DuckDBConnectionString ParsedConnection => parsedConnection ??= DuckDBConnectionStringBuilder.Parse(ConnectionString); + internal SqlStatistics Statistics + { + get => statistics; + } + public DuckDBConnection() { ConnectionString = string.Empty; @@ -72,6 +83,77 @@ public override string DataSource public DuckDBNativeConnection NativeConnection => connectionReference?.NativeConnection ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native connection."); + public DuckDBDatabase NativeDatabase => connectionReference?.FileReferenceCounter.Database + ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native database."); + + + // + // Summary: + // When set to true, enables statistics gathering for the current connection. + // + // Value: + // Returns true if statistics gathering is enabled; otherwise false. false is the + // default. + [DefaultValue(false)] + public bool StatisticsEnabled + { + get + { + return (collectstats); + } + set + { + if (value) + { + // start + if (ConnectionState.Open == State) + { + if (statistics == null) + { + statistics = new SqlStatistics(); + statistics.openTimestamp = TimerUtils.TimerCurrent(); + } + } + } + else + { + // stop + if (statistics != null) + { + if (ConnectionState.Open == State) + { + statistics.closeTimestamp = TimerUtils.TimerCurrent(); + } + } + } + collectstats = value; + } + } + + public IDictionary RetrieveStatistics() + { + if (Statistics != null) + { + UpdateStatistics(); + return Statistics.GetDictionary(); + } + else + { + return new SqlStatistics().GetDictionary(); + } + } + + private void UpdateStatistics() + { + if (ConnectionState.Open == State) + { + // update timestamp + statistics.closeTimestamp = TimerUtils.TimerCurrent(); + } + // delegate the rest of the work to the SqlStatistics class + Statistics.UpdateStatistics(); + } + public override string ServerVersion => NativeMethods.Startup.DuckDBLibraryVersion(); public override ConnectionState State => connectionState; @@ -94,6 +176,9 @@ public override void Close() } connectionState = ConnectionState.Closed; + + SqlStatistics.StopTimer(statistics); + OnStateChange(FromOpenToClosedEventArgs); } @@ -109,6 +194,9 @@ public override void Open() : connectionManager.GetConnectionReference(ParsedConnection); connectionState = ConnectionState.Open; + + _ = SqlStatistics.StartTimer(statistics); + OnStateChange(FromClosedToOpenEventArgs); } @@ -258,6 +346,7 @@ public DuckDBConnection Duplicate() parsedConnection = ParsedConnection, inMemoryDuplication = true, connectionReference = connectionReference, + StatisticsEnabled = StatisticsEnabled }; return duplicatedConnection; diff --git a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs index 435dfc58..44125bf4 100644 --- a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs +++ b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs @@ -93,6 +93,7 @@ public static DuckDBValue ToDuckDBValue(this object? item, DuckDBLogicalType log (DuckDBType.Blob, byte[] value) => NativeMethods.Value.DuckDBCreateBlob(value, value.Length), (DuckDBType.List, ICollection value) => CreateCollectionValue(logicalType, value, true, dbType), (DuckDBType.Array, ICollection value) => CreateCollectionValue(logicalType, value, false, dbType), + (DuckDBType.Map, IDictionary value) => CreateMapValue(logicalType, value), _ when ValueCreators.TryGetValue(dbType, out var converter) => converter(item), _ => NativeMethods.Value.DuckDBCreateVarchar(item.ToString()) }; @@ -139,6 +140,29 @@ private static DuckDBValue CreateCollectionValue(DuckDBLogicalType logicalType, : NativeMethods.Value.DuckDBCreateArrayValue(collectionItemType, values, collection.Count); } + private static DuckDBValue CreateMapValue(DuckDBLogicalType logicalType, IDictionary dictionary) + { + //get key and value types from logical type + using var keyType = NativeMethods.LogicalType.DuckDBMapTypeKeyType(logicalType); + using var valueType = NativeMethods.LogicalType.DuckDBMapTypeValueType(logicalType); + + //get duckdb types for key and value types + var keyDuckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(keyType); + var valueDuckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(valueType); + + var keys = new DuckDBValue[dictionary.Count]; + var values = new DuckDBValue[dictionary.Count]; + var index = 0; + foreach (DictionaryEntry entry in dictionary) + { + keys[index] = entry.Key.ToDuckDBValue(keyType, keyDuckDBType, DbType.Object); + values[index] = entry.Value.ToDuckDBValue(valueType, valueDuckDBType, DbType.Object); + index++; + } + + return NativeMethods.Value.DuckDBCreateMapValue(logicalType, keys, values, dictionary.Count); + } + private static DuckDBValue DecimalToDuckDBValue(decimal value) { var mantissa = value.GetMantissa(); diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index d074baab..8ee3cf73 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -12,9 +12,11 @@ private PreparedStatement(DuckDBPreparedStatement statement) this.statement = statement; } - public static IEnumerable PrepareMultiple(DuckDBNativeConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) + public static IEnumerable PrepareMultiple(DuckDBConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) { - var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection, query, out var extractedStatements); + var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection.NativeConnection, query, out var extractedStatements); + + SqlStatistics statistics = null; using (extractedStatements) { @@ -26,12 +28,14 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c for (int index = 0; index < statementCount; index++) { - var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, index, out var statement); + statistics = SqlStatistics.StartTimer(connection.Statistics); + + var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection.NativeConnection, extractedStatements, index, out var statement); if (status.IsSuccess()) { using var preparedStatement = new PreparedStatement(statement); - yield return preparedStatement.Execute(parameters, useStreamingMode, connection); + yield return preparedStatement.Execute(parameters, useStreamingMode, connection.NativeConnection); } else { @@ -42,8 +46,10 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c errorMessage = "DuckDBQuery failed"; } - throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); + throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection.NativeConnection)); } + + SqlStatistics.StopTimer(statistics); } } } diff --git a/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs new file mode 100644 index 00000000..c112a5b7 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs @@ -0,0 +1,63 @@ +namespace DuckDB.NET.Native; + +public class DuckDBProfilingInfoWrapper : IDisposable +{ + private DuckDBProfilingInfo handle; + private bool disposed; + + public DuckDBProfilingInfoWrapper(DuckDBProfilingInfo handle) + { + ArgumentNullException.ThrowIfNull(handle); + + this.handle = handle; + } + + public static DuckDBProfilingInfoWrapper GetProfilingInfo(DuckDBNativeConnection connection) + { + ArgumentNullException.ThrowIfNull(connection); + + var handle = NativeMethods.ProfilingInfo.DuckDBGetProfilingInfo(connection); + if (handle.IsInvalid || handle.IsClosed) + { + Console.WriteLine("No profiling info"); + return null; + } + return new DuckDBProfilingInfoWrapper(handle); + } + + public DuckDBValue GetValue(string key) + { + ArgumentNullException.ThrowIfNull(key); + + return NativeMethods.ProfilingInfo.DuckDBProfilingInfoGetValue(handle, key); + } + + public DuckDBValue GetMetrics() + { + return NativeMethods.ProfilingInfo.DuckDBProfilingInfoGetMetrics(handle); + } + + public ulong GetChildCount() + { + return NativeMethods.ProfilingInfo.DuckDBProfilingInfoGetChildCount(handle); + } + + public DuckDBProfilingInfoWrapper GetChild(ulong index) + { + var childHandle = NativeMethods.ProfilingInfo.DuckDBProfilingInfoGetChild(handle, index); + if (childHandle.IsInvalid || childHandle.IsClosed) + { + throw new InvalidOperationException($"Failed to get child profiling info at index {index}."); + } + return new DuckDBProfilingInfoWrapper(childHandle); + } + + public void Dispose() + { + if (!disposed) + { + handle?.Dispose(); + disposed = true; + } + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs new file mode 100644 index 00000000..636083dd --- /dev/null +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -0,0 +1,77 @@ +public class ProfilingInfo +{ + private readonly DuckDBNativeConnection _connection; + private DuckDBProfilingInfoWrapper? duckDBProfilingInfoWrapper; + + public ProfilingInfo(DuckDBNativeConnection connection) + { + _connection = connection; + } + + public bool TryPrepare() + { + // 1. Get the profiling info root node + using var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(_connection); + + if (profilingInfo == null) + { + return false; + } + + duckDBProfilingInfoWrapper = profilingInfo; + + // 2. Print the profiling info recursively + //PrintProfilingNode(profilingInfo, 0); + return true; + } + + public IDictionary GetMetrics() + { + if (duckDBProfilingInfoWrapper == null) + { + throw new InvalidOperationException("Profiling info is not prepared. Call Prepare() first."); + } + + var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); + if (metricsValue.IsNull()) + { + return new Dictionary(); + } + return metricsValue.GetMapValue(); + } + + private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int indent) + { + // Example: print the 'name' and 'extra_info' keys if present + var queryNameValue = node.GetValue("QUERY_NAME"); + Console.WriteLine($"{new string(' ', indent * 2)}QueryName: {queryNameValue.GetValue()}"); + var nameValue = node.GetValue("CPU_TIME"); + ////var extraInfoValue = node.GetValue("extra_info"); + + Console.WriteLine($"{new string(' ', indent * 2)}CpuTime: {nameValue.GetValue()}"); + //if (!extraInfoValue.IsNull()) + //{ + // Console.WriteLine($"{new string(' ', indent * 2 + 2)}Extra: {extraInfoValue.GetValue()}"); + //} + + // Print metrics if needed + var metrics = node.GetMetrics(); + if (!metrics.IsNull()) + { + var dic = metrics.GetMapValue(); + Console.WriteLine($"{new string(' ', indent * 2 + 2)}Metrics:"); + foreach (var kvp in dic) + { + Console.WriteLine($"{new string(' ', indent * 3 + 2)} {kvp.Key}:{kvp.Value}"); + } + } + + //// Recurse into children + //var childCount = node.GetChildCount(); + //for (ulong i = 0; i < childCount; i++) + //{ + // using var child = node.GetChild(i); + // PrintProfilingNode(child, indent + 1); + //} + } +} \ No newline at end of file diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index f575c49c..1cac56c0 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -3,10 +3,14 @@ using DuckDB.NET.Native; using DuckDB.NET.Test.Helpers; using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; using System.Data.Common; using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading.Tasks; using static DuckDB.NET.Native.NativeMethods; namespace DuckDB.NET.Samples @@ -21,13 +25,17 @@ static void Main(string[] args) return; } - DapperSample(); + //DapperSample(); - AdoNetSamples(); + //AdoNetSamples(); - LowLevelBindingsSample(); + //LowLevelBindingsSample(); - BulkDataLoad(); + //BulkDataLoad(); + + //ParametersBinding(); + + TestParallelism().GetAwaiter().GetResult(); } private static void DapperSample() @@ -188,6 +196,82 @@ private static void BulkDataLoad() } + private static void ParametersBinding() + { + + using var duckDBConnection = new DuckDBConnection("Data Source=:memory:"); + duckDBConnection.Open(); + + var testDicto = new Dictionary + { + ["foo"] = "42", + ["bar"] = "test" + }; + var testList = new List { "foo", "bar" }; + var testDictoList = new Dictionary> + { + ["foo"] = testList + }; + + using var command = duckDBConnection.CreateCommand(); + command.CommandText = "CREATE TABLE TestTable (foo MAP(string, string), bar MAP(string, string[]), baz string[])"; + command.ExecuteNonQuery(); + + command.CommandText = "INSERT INTO TestTable (foo, bar, baz) VALUES ($testDicto, $testDictoList, $testList)"; + command.Parameters.Add(new DuckDBParameter(testDicto)); + command.Parameters.Add(new DuckDBParameter(testDictoList)); + command.Parameters.Add(new DuckDBParameter(testList)); + command.ExecuteNonQuery(); + + command.CommandText = "SELECT foo, bar, baz FROM TestTable"; + using var reader = command.ExecuteReader(); + PrintQueryResults(reader); + } + + private async static ValueTask TestParallelism() + { + var values15 = @" +SET allowed_directories = ['C:\ProgramData\IrionDQ\latest\.duckdb\extensions']; +SET temp_directory = 'C:\WINDOWS\SystemTemp\IrionDQ\.duckdb\.tmp'; +INSTALL mssql FROM community; +LOAD mssql; +CREATE OR REPLACE SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10 (TYPE mssql, HOST 'NB242', PORT 1433, DATABASE 'IrionDQWorkinglatest', USER 'IrionDQ', PASSWORD 'vA9MJiNpVlwSrmV8OaU', USE_ENCRYPT FALSE, CATALOG TRUE, SCHEMA_FILTER '^(idq10|idq1)$'); +ATTACH '' AS TO_MSSQL (TYPE mssql, SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10); +"; + + var options = new ParallelOptions + { + MaxDegreeOfParallelism = 8//Environment.ProcessorCount + }; + + await Parallel.ForEachAsync(Enumerable.Range(0, 100), options, (i, ct) => + { + using var con = new DuckDBConnection("data source=:memory:"); + con.Open(); + + using var cmd = con.CreateCommand(); + + cmd.CommandText = "PRAGMA Version;"; + using (var reader = cmd.ExecuteReader()) + PrintQueryResults(reader); + + cmd.CommandText = "SELECT database_name FROM duckdb_databases()"; + using (var reader1 = cmd.ExecuteReader()) + while (reader1.Read()) ; + //PrintQueryResults(reader1); + + // Prefer SQL without INSTALL here (LOAD + CREATE SECRET + ATTACH only) + cmd.CommandText = values15; + cmd.ExecuteNonQuery(); + + cmd.CommandText = "DETACH TO_MSSQL"; + cmd.ExecuteNonQuery(); + + Console.WriteLine($"---- {i}"); + return ValueTask.CompletedTask; + }); + } + private static void PrintQueryResults(DbDataReader queryResult) { for (var index = 0; index < queryResult.FieldCount; index++) @@ -208,7 +292,9 @@ private static void PrintQueryResults(DbDataReader queryResult) continue; } var val = queryResult.GetValue(ordinal); - Console.Write(val); + + Console.Write(FormatValue(val)); + Console.Write(" "); } @@ -240,6 +326,38 @@ private static void PrintQueryResults(DuckDBResult queryResult) Console.WriteLine(); } } + + private static string FormatValue(object? value) + { + if (value is null) + { + return "NULL"; + } + + if (value is string s) + { + return s; + } + + if (value is IDictionary dictionary) + { + var parts = new List(); + foreach (DictionaryEntry entry in dictionary) + { + parts.Add($"{FormatValue(entry.Key)}:{FormatValue(entry.Value)}"); + } + + return "{" + string.Join(", ", parts) + "}"; + } + + if (value is IEnumerable enumerable) + { + var parts = enumerable.Cast().Select(FormatValue); + return "[" + string.Join("|", parts) + "]"; + } + + return value.ToString() ?? string.Empty; + } } class FooBar diff --git a/DuckDB.NET.Test/Parameters/MapParameterTests.cs b/DuckDB.NET.Test/Parameters/MapParameterTests.cs new file mode 100644 index 00000000..aafe74a1 --- /dev/null +++ b/DuckDB.NET.Test/Parameters/MapParameterTests.cs @@ -0,0 +1,205 @@ +using System.Collections; + +namespace DuckDB.NET.Test.Parameters; + +public class MapParameterTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + public sealed record MapTypeCase( + string Name, + string DuckDbType, + Type ClrType, + Func Generator, + Func Normalize); + + private static readonly MapTypeCase[] ScalarTypeCases = + [ + new("bool", "bool", typeof(bool), faker => faker.Random.Bool(), value => value), + new("sbyte", "tinyint", typeof(sbyte), faker => faker.Random.SByte(), value => value), + new("short", "SmallInt", typeof(short), faker => faker.Random.Short(), value => value), + new("int", "int", typeof(int), faker => faker.Random.Int(), value => value), + new("long", "BigInt", typeof(long), faker => faker.Random.Long(), value => value), + new("hugeint", "HugeInt", typeof(BigInteger), faker => BigInteger.Subtract(DuckDBHugeInt.HugeIntMaxValue, faker.Random.Int(min: 0)), value => value), + new("byte", "UTinyInt", typeof(byte), faker => faker.Random.Byte(), value => value), + new("ushort", "USmallInt", typeof(ushort), faker => faker.Random.UShort(), value => value), + new("uint", "UInteger", typeof(uint), faker => faker.Random.UInt(), value => value), + new("ulong", "UBigInt", typeof(ulong), faker => faker.Random.ULong(), value => value), + new("float", "Float", typeof(float), faker => faker.Random.Float(), value => value), + new("double", "Double", typeof(double), faker => faker.Random.Double(), value => value), + new("decimal", "Decimal(38, 28)", typeof(decimal), faker => faker.Random.Decimal(), value => value), + new("guid", "UUID", typeof(Guid), faker => faker.Random.Uuid(), value => value), + new("datetime", "Date", typeof(DateTime), faker => faker.Date.Past().Date, NormalizeDateValue), + new("datetimeoffset", "TimeTZ", typeof(DateTimeOffset), faker => GetRandomTimeTz(faker), value => value), + new("string", "String", typeof(string), faker => faker.Random.Utf16String(), value => value), + new("interval", "Interval", typeof(TimeSpan), faker => + { + var timespan = faker.Date.Timespan(); + return TimeSpan.FromTicks(timespan.Ticks - timespan.Ticks % 10); + }, value => value), + new("duckdbdateonly", "Date", typeof(DuckDBDateOnly), faker => (DuckDBDateOnly)faker.Date.Past().Date, NormalizeDateValue), + new("duckdbtimeonly", "Time", typeof(DuckDBTimeOnly), faker => (DuckDBTimeOnly)faker.Date.Past(), NormalizeTimeValue), + new("dateonly", "Date", typeof(DateOnly), faker => DateOnly.FromDateTime(faker.Date.Past().Date), NormalizeDateValue), + new("timeonly", "Time", typeof(TimeOnly), faker => + { + var dateTime = faker.Date.Past(); + return new TimeOnly(dateTime.TimeOfDay.Ticks - dateTime.TimeOfDay.Ticks % 10); + }, NormalizeTimeValue) + ]; + + public static IEnumerable ScalarKeyValueCombinations => + from keyCase in ScalarTypeCases + from valueCase in ScalarTypeCases + select new object[] { keyCase, valueCase }; + + [Theory] + [MemberData(nameof(ScalarKeyValueCombinations))] + public void CanBindMapForEveryScalarKeyAndValueCombination(MapTypeCase keyCase, MapTypeCase valueCase) + { + var map = CreateMap(keyCase, valueCase); + + Action act = () => InsertSelectMap(keyCase, valueCase, map); + act.Should().NotThrow($"key={keyCase.Name}, value={valueCase.Name}"); + } + + [Fact] + public void CanBindMapWithNestedListValues() + { + var value = new Dictionary> + { + ["one"] = new List { 1, 2, 0 }, + ["two"] = new List { 3, 0, 5 } + }; + + InsertSelectMap("String", "INTEGER[]", value); + } + + [Fact] + public void CanBindMapWithNestedMapValues() + { + var value = new Dictionary> + { + [1] = new Dictionary + { + ["a"] = new TimeOnly(11, 22, 33), + ["b"] = new TimeOnly(9, 5, 2) + }, + [2] = new Dictionary + { + ["x"] = new TimeOnly(6, 30, 45) + } + }; + + InsertSelectMap("INTEGER", "MAP(String, Time)", value); + } + + private void InsertSelectMap(string keyDuckDbType, string valueDuckDbType, IDictionary map) + { + Command.Parameters.Clear(); + Command.CommandText = $"CREATE OR REPLACE TABLE ParameterMapTest (a MAP({keyDuckDbType}, {valueDuckDbType}));"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO ParameterMapTest (a) VALUES ($map);"; + Command.Parameters.Add(new DuckDBParameter(map)); + Command.ExecuteNonQuery(); + + Command.Parameters.Clear(); + Command.CommandText = "SELECT a FROM ParameterMapTest;"; + + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + var value = reader.GetValue(0); + value.Should().BeEquivalentTo(map); + } + + private void InsertSelectMap(MapTypeCase keyCase, MapTypeCase valueCase, IDictionary map) + { + Command.Parameters.Clear(); + Command.CommandText = $"CREATE OR REPLACE TABLE ParameterMapTest (a MAP({keyCase.DuckDbType}, {valueCase.DuckDbType}));"; + Command.ExecuteNonQuery(); + + Command.CommandText = "INSERT INTO ParameterMapTest (a) VALUES ($map);"; + Command.Parameters.Add(new DuckDBParameter(map)); + Command.ExecuteNonQuery(); + + Command.Parameters.Clear(); + Command.CommandText = "SELECT a FROM ParameterMapTest;"; + + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + var actual = (IDictionary)reader.GetValue(0); + + var expectedEntries = map.Keys.Cast() + .Select(key => new + { + Key = keyCase.Normalize(key), + Value = valueCase.Normalize(map[key]!) + }); + + var actualEntries = actual.Keys.Cast() + .Select(key => new + { + Key = keyCase.Normalize(key), + Value = valueCase.Normalize(actual[key]!) + }); + + actualEntries.Should().BeEquivalentTo(expectedEntries); + } + + private IDictionary CreateMap(MapTypeCase keyCase, MapTypeCase valueCase) + { + var dictionaryType = typeof(Dictionary<,>).MakeGenericType(keyCase.ClrType, valueCase.ClrType); + var map = (IDictionary)Activator.CreateInstance(dictionaryType)!; + + object? key = null; + var attempts = 0; + + while (key is null && attempts < 10) + { + key = keyCase.Generator(Faker); + attempts++; + } + + key.Should().NotBeNull(); + var value = valueCase.Generator(Faker); + map.Add(key!, value); + + return map; + } + + private static DateTimeOffset GetRandomTimeTz(Faker faker) + { + var dateTime = faker.Date.Between(DateTime.Now.AddYears(-100), DateTime.Now.AddYears(100)); + + if (dateTime.Hour < 1) + { + dateTime = dateTime.AddHours(1); + } + + dateTime = dateTime.AddTicks(-dateTime.Ticks % 10); + + return new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Unspecified), TimeSpan.FromHours(1)); + } + + private static object NormalizeDateValue(object value) + { + return value switch + { + DateTime dateTime => DateOnly.FromDateTime(dateTime), + DuckDBDateOnly duckDBDateOnly => (DateOnly)duckDBDateOnly, + DateOnly dateOnly => dateOnly, + _ => value + }; + } + + private static object NormalizeTimeValue(object value) + { + return value switch + { + DuckDBTimeOnly duckDBTimeOnly => (TimeOnly)duckDBTimeOnly, + TimeOnly timeOnly => timeOnly, + DateTime dateTime => TimeOnly.FromDateTime(dateTime), + _ => value + }; + } +} \ No newline at end of file From bc27930da14894411bb8386d133bf1a09e17ef66 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 24 Apr 2026 18:09:12 +0200 Subject: [PATCH 02/43] primo impianto rilevamento metriche --- .../Connection/QueryExecutionTimer.cs | 106 ++++++++++++++++++ .../Connection/QueryExecutionTracer.cs | 61 ++++++++++ .../Connection/StatementExecutionTracer.cs | 44 ++++++++ DuckDB.NET.Samples/NativeDebugResolver.cs | 65 +++++++++++ 4 files changed, 276 insertions(+) create mode 100644 DuckDB.NET.Data/Connection/QueryExecutionTimer.cs create mode 100644 DuckDB.NET.Data/Connection/QueryExecutionTracer.cs create mode 100644 DuckDB.NET.Data/Connection/StatementExecutionTracer.cs create mode 100644 DuckDB.NET.Samples/NativeDebugResolver.cs diff --git a/DuckDB.NET.Data/Connection/QueryExecutionTimer.cs b/DuckDB.NET.Data/Connection/QueryExecutionTimer.cs new file mode 100644 index 00000000..0f10582a --- /dev/null +++ b/DuckDB.NET.Data/Connection/QueryExecutionTimer.cs @@ -0,0 +1,106 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; + +namespace DuckDB.NET.Data.Connection +{ + internal sealed class QueryExecutionTimer + { + + // internal values that are not exposed through properties + internal long? startExecutionTimestamp; + + // internal values that are exposed through properties + internal long executionTime; + internal DateTimeOffset startExecutionTime; + internal DateTimeOffset endExecutionTime; + internal Dictionary> metrics = []; + private readonly DuckDBPreparedStatement preparedStatement; + private readonly int queryIndex; + + internal QueryExecutionTimer(DuckDBPreparedStatement preparedStatement, int queryIndex) + { + this.preparedStatement = preparedStatement; + this.queryIndex = queryIndex; + } + + internal void StartTimer() + { + if (!startExecutionTimestamp.HasValue) + { + startExecutionTimestamp = TimerUtils.TimerCurrent(); + startExecutionTime = TimerUtils.Now(); + } + } + + internal void StopTimer() + { + ReleaseAndUpdateExecutionTimer(); + } + + internal void ReleaseAndUpdateExecutionTimer() + { + if (startExecutionTimestamp.HasValue) + { + uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + executionTime += elapsed; + endExecutionTime = startExecutionTime.AddTicks(elapsed); + + startExecutionTimestamp = null; + } + } + + internal void UpdateStatistics() + { + // update connection time + if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) + { + connectionTime = closeTimestamp - openTimestamp; + } + else + { + connectionTime = long.MaxValue; + } + } + + internal void ReadMetrics(DuckDBNativeConnection connection, int index) + { + var profile = new ProfilingInfo(connection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + metrics[index] = curMetrics; + } + } + + internal IDictionary GetDictionary() + { + const int Count = 18; + var dictionary = new Dictionary(Count) + { + { "StartTime", startExecutionTime }, + { "EndTime", endExecutionTime }, + { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, + { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, + { "MetricsCount", metrics.Count }, + { "Metrics", metrics } + }; + Debug.Assert(dictionary.Count == Count); + return dictionary; + } + + internal void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + connectionTime = 0; + startExecutionTime = default; + endExecutionTime = default; + } + + public void Dispose() + { + throw new NotImplementedException(); + } + } +} diff --git a/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs b/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs new file mode 100644 index 00000000..934eeeef --- /dev/null +++ b/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs @@ -0,0 +1,61 @@ +using DuckDB.NET.Data.PreparedStatement; +using System.Net.NetworkInformation; + +namespace DuckDB.NET.Data.Connection +{ + internal sealed class QueryExecutionTracer : IDisposable + { + private readonly QueryExecutionTimer timer; + private readonly Dictionary statementTimers = []; + private readonly nint queryIdentifier; + private readonly int statementCount; + + internal QueryExecutionTracer(IntPtr queryIdentifier, int statementCount) + { + this.queryIdentifier = queryIdentifier; + this.statementCount = statementCount; + } + + internal QueryExecutionTracer(QueryExecutionTimer timer, IntPtr queryIdentifier) + { + this.timer = timer; + } + + internal StatementExecutionTracer CreateStatementTracer(DuckDBPreparedStatement preparedStatement, int queryIndex) + { + if (!statementTimers.TryGetValue(preparedStatement, out var statementTimer)) + { + statementTimer = new QueryExecutionTimer(preparedStatement, queryIndex); + statementTimers[preparedStatement] = statementTimer; + } + + return new StatementExecutionTracer(statementTimer); + } + + //internal void StartTimer() + //{ + // timer.StartTimer(); + //} + + //internal void StopTimer() + //{ + // timer.StopTimer(); + //} + + internal void AcquireMetrics(DuckDBNativeConnection connection) + { + //var profile = new ProfilingInfo(connection); + + //if (profile.TryPrepare()) + //{ + // var curMetrics = profile.GetMetrics(); + // metrics[index] = curMetrics; + //} + } + + public void Dispose() + { + timer.StopTimer(); + } + } +} diff --git a/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs b/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs new file mode 100644 index 00000000..2eebe4d1 --- /dev/null +++ b/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs @@ -0,0 +1,44 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; + +namespace DuckDB.NET.Data.Connection +{ + internal sealed class StatementExecutionTracer : IDisposable + { + private readonly QueryExecutionTimer timer; + bool isEnabled = false; + + internal StatementExecutionTracer(QueryExecutionTimer timer) + { + this.timer = timer; + } + + internal void StartTimer() + { + timer.StartTimer(); + } + + internal void StopTimer() + { + timer.StopTimer(); + } + + internal bool IsEnabled => isEnabled; + + internal void ReadMetrics(DuckDBNativeConnection connection) + { + var profile = new ProfilingInfo(connection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + metrics[index] = curMetrics; + } + } + + public void Dispose() + { + timer.StopTimer(); + } + } +} diff --git a/DuckDB.NET.Samples/NativeDebugResolver.cs b/DuckDB.NET.Samples/NativeDebugResolver.cs new file mode 100644 index 00000000..55ec933f --- /dev/null +++ b/DuckDB.NET.Samples/NativeDebugResolver.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; + +internal static class NativeDebugResolver +{ + private static IntPtr _duckdbHandle; + private static bool _initialized; + + public static void Initialize() + { + if (_initialized) return; + _initialized = true; + + //string repo = @"C:\Sources\Git\duckdb_scalarfs"; + string repo = @"C:\Users\Lucap\Downloads\duckdb 5"; + string duckdbDir = Path.Combine(repo, "build", "debug", "src", "Debug"); + + string duckdbPath = Path.Combine(duckdbDir, "duckdb.dll"); + + if (!File.Exists(duckdbPath)) throw new FileNotFoundException("duckdb.dll not found", duckdbPath); + + // Optional but useful: preload debug runtime deps near duckdb.dll + //PreloadIfExists(Path.Combine(duckdbDir, "zlibd1.dll")); + //PreloadIfExists(Path.Combine(duckdbDir, "libcrypto-3-x64.dll")); + //PreloadIfExists(Path.Combine(duckdbDir, "libssl-3-x64.dll")); + //PreloadIfExists(Path.Combine(duckdbDir, "libcurl-d.dll")); + + _duckdbHandle = NativeLibrary.Load(duckdbPath); + + // Install resolver for the assembly that contains your DllImport declarations. + // If your DllImports are in another assembly, pass that assembly instead. + NativeLibrary.SetDllImportResolver( + typeof(NativeDebugResolver).Assembly, + Resolve); + } + + private static IntPtr Resolve(string libraryName, Assembly assembly, DllImportSearchPath? searchPath) + { + if (IsDuckDbName(libraryName)) + { + return _duckdbHandle; + } + + return IntPtr.Zero; + } + + private static bool IsDuckDbName(string name) + { + return name.Equals("duckdb", StringComparison.OrdinalIgnoreCase) + || name.Equals("duckdb.dll", StringComparison.OrdinalIgnoreCase); + } + + private static void PreloadIfExists(string path) + { + if (File.Exists(path)) + { + NativeLibrary.Load(path); + } + } + +} \ No newline at end of file From 319c9bfea60be7a043e0fc5409705fef79b1258b Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 24 Apr 2026 18:09:21 +0200 Subject: [PATCH 03/43] primo impianto rilevamento metriche --- DuckDB.NET.Bindings/DuckDBWrapperObjects.cs | 4 +- .../Connection/QueryExecutionTracer.cs | 61 ------- .../Connection/StatementExecutionTracer.cs | 44 ----- DuckDB.NET.Data/DuckDBConnection.cs | 60 ++++--- .../PreparedStatement/PreparedStatement.cs | 46 ++++-- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 8 +- .../Statistics/ExecutionStatistics.cs | 39 +++++ .../Profiling/Statistics/ExecutionTracer.cs | 91 ++++++++++ .../Profiling/Statistics/IExecutionTracer.cs | 13 ++ .../Statistics/QueryExecutionStatistics.cs} | 97 ++++++----- .../Statistics/QueryExecutionTracer.cs | 39 +++++ .../Profiling/Statistics/SqlStatistics.cs | 155 ++++++++++++++++++ .../StatementExecutionStatistics.cs} | 48 ++---- .../Statistics/StatementExecutionTracer.cs | 11 ++ DuckDB.NET.Samples/Program.cs | 108 +++++++----- 15 files changed, 549 insertions(+), 275 deletions(-) delete mode 100644 DuckDB.NET.Data/Connection/QueryExecutionTracer.cs delete mode 100644 DuckDB.NET.Data/Connection/StatementExecutionTracer.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs rename DuckDB.NET.Data/{Connection/SqlStatistics.cs => Profiling/Statistics/QueryExecutionStatistics.cs} (56%) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs rename DuckDB.NET.Data/{Connection/QueryExecutionTimer.cs => Profiling/Statistics/StatementExecutionStatistics.cs} (61%) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index b4e0e809..9e09ff13 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -173,7 +173,8 @@ public T GetValue() [MethodImpl(MethodImplOptions.AggressiveInlining)] static T Cast(TSource value) => Unsafe.As(ref value); } - public Dictionary GetMapValue() where TKey : notnull + + public IDictionary GetMapValue() where TKey : notnull { var result = new Dictionary(); @@ -190,7 +191,6 @@ public Dictionary GetMapValue() where TKey : notnull } return result; - } private DateTime GetTimestampValue(DuckDBTimestampStruct timestampStruct, DuckDBType duckDBType) diff --git a/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs b/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs deleted file mode 100644 index 934eeeef..00000000 --- a/DuckDB.NET.Data/Connection/QueryExecutionTracer.cs +++ /dev/null @@ -1,61 +0,0 @@ -using DuckDB.NET.Data.PreparedStatement; -using System.Net.NetworkInformation; - -namespace DuckDB.NET.Data.Connection -{ - internal sealed class QueryExecutionTracer : IDisposable - { - private readonly QueryExecutionTimer timer; - private readonly Dictionary statementTimers = []; - private readonly nint queryIdentifier; - private readonly int statementCount; - - internal QueryExecutionTracer(IntPtr queryIdentifier, int statementCount) - { - this.queryIdentifier = queryIdentifier; - this.statementCount = statementCount; - } - - internal QueryExecutionTracer(QueryExecutionTimer timer, IntPtr queryIdentifier) - { - this.timer = timer; - } - - internal StatementExecutionTracer CreateStatementTracer(DuckDBPreparedStatement preparedStatement, int queryIndex) - { - if (!statementTimers.TryGetValue(preparedStatement, out var statementTimer)) - { - statementTimer = new QueryExecutionTimer(preparedStatement, queryIndex); - statementTimers[preparedStatement] = statementTimer; - } - - return new StatementExecutionTracer(statementTimer); - } - - //internal void StartTimer() - //{ - // timer.StartTimer(); - //} - - //internal void StopTimer() - //{ - // timer.StopTimer(); - //} - - internal void AcquireMetrics(DuckDBNativeConnection connection) - { - //var profile = new ProfilingInfo(connection); - - //if (profile.TryPrepare()) - //{ - // var curMetrics = profile.GetMetrics(); - // metrics[index] = curMetrics; - //} - } - - public void Dispose() - { - timer.StopTimer(); - } - } -} diff --git a/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs b/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs deleted file mode 100644 index 2eebe4d1..00000000 --- a/DuckDB.NET.Data/Connection/StatementExecutionTracer.cs +++ /dev/null @@ -1,44 +0,0 @@ -using DuckDB.NET.Data.Common; -using System.Diagnostics; - -namespace DuckDB.NET.Data.Connection -{ - internal sealed class StatementExecutionTracer : IDisposable - { - private readonly QueryExecutionTimer timer; - bool isEnabled = false; - - internal StatementExecutionTracer(QueryExecutionTimer timer) - { - this.timer = timer; - } - - internal void StartTimer() - { - timer.StartTimer(); - } - - internal void StopTimer() - { - timer.StopTimer(); - } - - internal bool IsEnabled => isEnabled; - - internal void ReadMetrics(DuckDBNativeConnection connection) - { - var profile = new ProfilingInfo(connection); - - if (profile.TryPrepare()) - { - var curMetrics = profile.GetMetrics(); - metrics[index] = curMetrics; - } - } - - public void Dispose() - { - timer.StopTimer(); - } - } -} diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 609ad1ed..7197fa3d 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -1,10 +1,9 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.Connection; +using DuckDB.NET.Data.Profiling.Statistics; using System.ComponentModel; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics.Arm; namespace DuckDB.NET.Data; @@ -87,15 +86,11 @@ public override string DataSource ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native database."); - // - // Summary: - // When set to true, enables statistics gathering for the current connection. - // - // Value: - // Returns true if statistics gathering is enabled; otherwise false. false is the - // default. + /// + /// Enables or disables profiling of the connection. When enabled, the connection will collect statistics about query execution times and other relevant metrics. + /// [DefaultValue(false)] - public bool StatisticsEnabled + public bool ProfilingEnabled { get { @@ -103,16 +98,17 @@ public bool StatisticsEnabled } set { + if (State != ConnectionState.Open) + { + throw new InvalidOperationException("The DuckDBConnection must be open to start collecting statistics."); + } + if (value) { // start - if (ConnectionState.Open == State) + if (statistics == null) { - if (statistics == null) - { - statistics = new SqlStatistics(); - statistics.openTimestamp = TimerUtils.TimerCurrent(); - } + statistics = new SqlStatistics(NativeConnection, value); } } else @@ -120,16 +116,17 @@ public bool StatisticsEnabled // stop if (statistics != null) { - if (ConnectionState.Open == State) - { - statistics.closeTimestamp = TimerUtils.TimerCurrent(); - } + statistics.closeTimestamp = TimerUtils.TimerCurrent(); } } collectstats = value; } } + /// + /// Retrieves the current set of SQL statistics as a dictionary. + /// + /// A dictionary containing the current SQL statistics. If no statistics are available, returns an empty dictionary. public IDictionary RetrieveStatistics() { if (Statistics != null) @@ -139,10 +136,21 @@ public IDictionary RetrieveStatistics() } else { - return new SqlStatistics().GetDictionary(); + return new SqlStatistics(NativeConnection, collectstats).GetDictionary(); } } + /// + /// Sets the minimum execution time, in milliseconds, required for a query plan to be collected for analysis. + /// + /// The minimum duration, in milliseconds, that a query must run before its plan is collected. Must be a + /// non-negative integer. + /// Thrown if the connection is not open. + public void QueryPlanCollectionThreshold(int threshold) + { + throw new NotImplementedException(); + } + private void UpdateStatistics() { if (ConnectionState.Open == State) @@ -175,9 +183,9 @@ public override void Close() connectionManager.ReturnConnectionReference(connectionReference); } - connectionState = ConnectionState.Closed; + UpdateStatistics(); - SqlStatistics.StopTimer(statistics); + connectionState = ConnectionState.Closed; OnStateChange(FromOpenToClosedEventArgs); } @@ -195,7 +203,7 @@ public override void Open() connectionState = ConnectionState.Open; - _ = SqlStatistics.StartTimer(statistics); + statistics.openTimestamp = TimerUtils.TimerCurrent(); OnStateChange(FromClosedToOpenEventArgs); } @@ -272,7 +280,7 @@ string GetTableName() /// The AppenderMap type defining the mappings /// The table name /// A type-safe mapped appender - public DuckDBMappedAppender CreateAppender(string table) + public DuckDBMappedAppender CreateAppender(string table) where TMap : Mapping.DuckDBAppenderMap, new() { return CreateAppender(null, null, table); @@ -346,7 +354,7 @@ public DuckDBConnection Duplicate() parsedConnection = ParsedConnection, inMemoryDuplication = true, connectionReference = connectionReference, - StatisticsEnabled = StatisticsEnabled + ProfilingEnabled = ProfilingEnabled }; return duplicatedConnection; diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 8ee3cf73..d2cd29e2 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,41 +1,49 @@ -using System.Linq; using DuckDB.NET.Data.Connection; +using System.Linq; namespace DuckDB.NET.Data.PreparedStatement; internal sealed class PreparedStatement : IDisposable { private readonly DuckDBPreparedStatement statement; + private readonly IntPtr queryIdentifier; - private PreparedStatement(DuckDBPreparedStatement statement) + private PreparedStatement(DuckDBPreparedStatement statement, IntPtr queryIdentifier) { this.statement = statement; + this.queryIdentifier = queryIdentifier; } public static IEnumerable PrepareMultiple(DuckDBConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) { var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection.NativeConnection, query, out var extractedStatements); - SqlStatistics statistics = null; - using (extractedStatements) - { + { + // Initialize the query tracer for the entire batch of statements. The tracer will be responsible for tracking the execution of all statements within this batch. + using var queryTracer = connection.statistics?.CreateQueryTracer(extractedStatements.ToHandle(), statementCount); + queryTracer?.StartTimer(); + if (statementCount <= 0) { var error = NativeMethods.ExtractStatements.DuckDBExtractStatementsError(extractedStatements); + + queryTracer?.SetState(DuckDBState.Error, DuckDBErrorType.Parser, error); + throw new DuckDBException(error); } for (int index = 0; index < statementCount; index++) { - statistics = SqlStatistics.StartTimer(connection.Statistics); - var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection.NativeConnection, extractedStatements, index, out var statement); + // Initialize the statement tracer for the current statement. This allows for detailed tracing of each individual statement within the batch + queryTracer?.PrepareStatementTracer(statement, index); + if (status.IsSuccess()) { - using var preparedStatement = new PreparedStatement(statement); - yield return preparedStatement.Execute(parameters, useStreamingMode, connection.NativeConnection); + using var preparedStatement = new PreparedStatement(statement, extractedStatements.ToHandle()); + yield return preparedStatement.Execute(parameters, useStreamingMode, connection); } else { @@ -46,16 +54,24 @@ public static IEnumerable PrepareMultiple(DuckDBConnection connect errorMessage = "DuckDBQuery failed"; } + using var statementTracer = queryTracer?.GetStatementTracer(statement); + statementTracer?.SetState(status, errorMessage); + throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection.NativeConnection)); } - - SqlStatistics.StopTimer(statistics); } } } - private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBNativeConnection connection) + private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBConnection connection) { + + // Tracing is discriminated by query identifier, which is a combination of the query text and the index of the statement in the case of multiple statements. + // This allows for more granular tracing of individual statements within a batch. + var queryTracer = connection.statistics?.GetQueryTracer(queryIdentifier); + using var statementTracer = queryTracer?.GetStatementTracer(statement); + statementTracer?.StartTimer(); + BindParameters(statement, parameterCollection); var status = useStreamingMode @@ -73,17 +89,21 @@ private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool errorMessage = "DuckDB execution failed"; } + statementTracer?.SetState(status, errorType, errorMessage); + if (errorType == DuckDBErrorType.Interrupt) { throw new OperationCanceledException(); } - var innerException = UdfExceptionStore.Retrieve(connection); + var innerException = UdfExceptionStore.Retrieve(connection.NativeConnection); throw innerException != null ? new DuckDBException(errorMessage, innerException) : new DuckDBException(errorMessage, errorType); } + statementTracer?.AcquireMetrics(); + return queryResult; } diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 636083dd..64e889cb 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -1,14 +1,14 @@ -public class ProfilingInfo +internal sealed class ProfilingInfo { private readonly DuckDBNativeConnection _connection; private DuckDBProfilingInfoWrapper? duckDBProfilingInfoWrapper; - public ProfilingInfo(DuckDBNativeConnection connection) + internal ProfilingInfo(DuckDBNativeConnection connection) { _connection = connection; } - public bool TryPrepare() + internal bool TryPrepare() { // 1. Get the profiling info root node using var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(_connection); @@ -25,7 +25,7 @@ public bool TryPrepare() return true; } - public IDictionary GetMetrics() + internal IDictionary GetMetrics() { if (duckDBProfilingInfoWrapper == null) { diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs new file mode 100644 index 00000000..b4d599de --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs @@ -0,0 +1,39 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal abstract class ExecutionStatistics(DuckDBNativeConnection duckDBNativeConnection) + { + protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; + + protected DuckDBState state; + protected string? errorMessage; + + internal abstract void AcquireMetrics(); + + internal abstract void Reset(); + + internal abstract void StartTimer(); + + internal abstract void StopTimer(); + + internal virtual void SetState(DuckDBState state) + { + this.state = state; + } + + internal virtual void SetState(DuckDBState state, string message) + { + this.state = state; + this.errorMessage = message; + } + + internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + this.state = state; + this.errorMessage = string.Format("%s error: %s", errorType, message); + } + + internal virtual DuckDBState State => state; + + internal virtual string ErrorMessage => errorMessage ?? string.Empty; + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs new file mode 100644 index 00000000..b3a5431e --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs @@ -0,0 +1,91 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal abstract class ExecutionTracer : IExecutionTracer, IDisposable + { + protected readonly ExecutionStatistics statistics; + private bool disposed; + + internal ExecutionTracer(ExecutionStatistics statistics) + { + this.statistics = statistics; + } + + internal virtual void AcquireMetrics() + { + statistics.AcquireMetrics(); + } + + internal virtual void StartTimer() + { + statistics.StartTimer(); + } + + internal virtual void StopTimer() + { + statistics.StopTimer(); + } + + internal virtual void Dispose() + { + statistics.StopTimer(); + } + + internal virtual void SetState(DuckDBState state) + { + statistics.SetState(state); + } + + internal virtual void SetState(DuckDBState state, string message) + { + statistics.SetState(state, message); + } + + internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + statistics.SetState(state, errorType, message); + } + + internal virtual DuckDBState State => statistics.State; + + void IDisposable.Dispose() + { + if (!disposed) + { + Dispose(); + disposed = true; + } + } + + void IExecutionTracer.AcquireMetrics() + { + AcquireMetrics(); + } + + void IExecutionTracer.StartTimer() + { + StartTimer(); + } + + void IExecutionTracer.StopTimer() + { + StopTimer(); + } + + void IExecutionTracer.SetState(DuckDBState state) + { + SetState(state); + } + + void IExecutionTracer.SetState(DuckDBState state, string message) + { + SetState(state, message); + } + + void IExecutionTracer.SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + SetState(state, errorType, message); + } + + DuckDBState IExecutionTracer.State => State; + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs new file mode 100644 index 00000000..6d7fbadc --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs @@ -0,0 +1,13 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal interface IExecutionTracer : IDisposable + { + void AcquireMetrics(); + void StartTimer(); + void StopTimer(); + void SetState(DuckDBState state); + void SetState(DuckDBState state, string message); + void SetState(DuckDBState state, DuckDBErrorType errorType, string message); + DuckDBState State { get; } + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Connection/SqlStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs similarity index 56% rename from DuckDB.NET.Data/Connection/SqlStatistics.cs rename to DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs index ee5902c9..51534cc8 100644 --- a/DuckDB.NET.Data/Connection/SqlStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs @@ -1,51 +1,35 @@ using DuckDB.NET.Data.Common; using System.Diagnostics; -namespace DuckDB.NET.Data.Connection +namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class SqlStatistics + internal sealed class QueryExecutionStatistics : ExecutionStatistics { - // internal values that are not exposed through properties - internal long closeTimestamp; - internal long openTimestamp; internal long? startExecutionTimestamp; + private readonly StatementExecutionStatistics[] executionStatistics; // internal values that are exposed through properties internal long executionTime; - internal long connectionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; internal Dictionary> metrics = []; + private readonly IntPtr queryIdentifier; + private readonly int statementCount; - internal SqlStatistics() + internal QueryExecutionStatistics(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { + this.statementCount = statementCount; + this.executionStatistics = new StatementExecutionStatistics[statementCount]; } - internal static SqlStatistics StartTimer(SqlStatistics statistics) - { - if (statistics != null && !statistics.RequestExecutionTimer()) - { - // we're re-entrant -- don't bother. - statistics = null; - } - return statistics; - } + internal IntPtr QueryIdentifier => queryIdentifier; - internal static void StopTimer(SqlStatistics statistics) + internal bool TryAddExecutionStatistics(StatementExecutionStatistics statistics, int queryIndex) { - if (statistics != null) - { - statistics.ReleaseAndUpdateExecutionTimer(); - } - } - - internal bool RequestExecutionTimer() - { - if (!startExecutionTimestamp.HasValue) + if (executionStatistics[queryIndex] == null) { - startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = TimerUtils.Now(); + executionStatistics[queryIndex] = statistics; return true; } return false; @@ -63,19 +47,6 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal void UpdateStatistics() - { - // update connection time - if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) - { - connectionTime = closeTimestamp - openTimestamp; - } - else - { - connectionTime = long.MaxValue; - } - } - internal void ReadMetrics(DuckDBNativeConnection connection, int index) { var profile = new ProfilingInfo(connection); @@ -89,25 +60,53 @@ internal void ReadMetrics(DuckDBNativeConnection connection, int index) internal IDictionary GetDictionary() { - const int Count = 18; - var dictionary = new Dictionary(Count) + //const int Count = 18; + var dictionary = new Dictionary(/*Count*/) { { "StartTime", startExecutionTime }, { "EndTime", endExecutionTime }, - { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, - { "MetricsCount", metrics.Count }, - { "Metrics", metrics } + { "StatementCount", statementCount } }; - Debug.Assert(dictionary.Count == Count); + //Debug.Assert(dictionary.Count == Count); return dictionary; } - internal void Reset() + internal override void StartTimer() + { + if (!startExecutionTimestamp.HasValue) + { + startExecutionTimestamp = TimerUtils.TimerCurrent(); + startExecutionTime = TimerUtils.Now(); + } + } + + internal override void StopTimer() + { + ReleaseAndUpdateExecutionTimer(); + } + + internal override void AcquireMetrics() + { + var profile = new ProfilingInfo(duckDBNativeConnection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + //metrics[index] = curMetrics; + } + } + + internal override void SetState(DuckDBState state, string message) + { + this.state = state; + this.errorMessage = message; + } + + internal override void Reset() { executionTime = 0; startExecutionTimestamp = null; - connectionTime = 0; startExecutionTime = default; endExecutionTime = default; } diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs new file mode 100644 index 00000000..3c70fc55 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs @@ -0,0 +1,39 @@ +using DuckDB.NET.Data.PreparedStatement; +using System.Diagnostics; +using System.Net.NetworkInformation; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class QueryExecutionTracer : ExecutionTracer + { + private readonly Dictionary statementTracers = []; + private readonly QueryExecutionStatistics queryExecutionStatistics; + private readonly DuckDBNativeConnection duckDBNativeConnection; + + internal QueryExecutionTracer(QueryExecutionStatistics queryExecutionStatistics, DuckDBNativeConnection duckDBNativeConnection): base(queryExecutionStatistics) + { + this.queryExecutionStatistics = queryExecutionStatistics; + this.duckDBNativeConnection = duckDBNativeConnection; + } + + internal void PrepareStatementTracer(DuckDBPreparedStatement preparedStatement, int statementIndex) + { + if (!statementTracers.ContainsKey(preparedStatement)) + { + var singleStatementStatistics = new StatementExecutionStatistics(preparedStatement, statementIndex, duckDBNativeConnection); + statementTracers[preparedStatement] = new StatementExecutionTracer(singleStatementStatistics); + + queryExecutionStatistics.TryAddExecutionStatistics(singleStatementStatistics, statementIndex); + } + } + + internal StatementExecutionTracer? GetStatementTracer(DuckDBPreparedStatement preparedStatement) + { + if (!statementTracers.TryGetValue(preparedStatement, out var tracer)) + { + return null; + } + return tracer; + } + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs new file mode 100644 index 00000000..c7dd8a76 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs @@ -0,0 +1,155 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class SqlStatistics + { + + // internal values that are not exposed through properties + internal long closeTimestamp; + internal long openTimestamp; + internal long? startExecutionTimestamp; + private readonly bool enableQueryExecutionTracing; + private readonly DuckDBNativeConnection duckDBNativeConnection; + private readonly Dictionary queryExecutionTracers = []; + private readonly Dictionary> queryExecutionStatistics = []; + + // internal values that are exposed through properties + internal long executionTime; + internal long connectionTime; + internal DateTimeOffset startExecutionTime; + internal DateTimeOffset endExecutionTime; + internal Dictionary> metrics = []; + + + internal SqlStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) + { + this.enableQueryExecutionTracing = enableQueryExecutionTracing; + this.duckDBNativeConnection = duckDBNativeConnection; + } + + internal QueryExecutionTracer? CreateQueryTracer(IntPtr queryIdentifier, int statementCount) + { + if (!enableQueryExecutionTracing) + { + return null; + } + + var executionStatistics = new QueryExecutionStatistics(queryIdentifier, statementCount, duckDBNativeConnection); + var tracer = new QueryExecutionTracer(executionStatistics, duckDBNativeConnection); + queryExecutionTracers[queryIdentifier] = tracer; + + // When a new tracer is created for a query, we also create a new list to hold the execution statistics for that query. + if (!queryExecutionStatistics.TryGetValue(queryIdentifier, out var executionStatisticsList)) + { + executionStatisticsList = []; + queryExecutionStatistics[queryIdentifier] = executionStatisticsList; + } + executionStatisticsList.Add(executionStatistics); + + return tracer; + } + + internal QueryExecutionTracer? GetQueryTracer(IntPtr queryIdentifier) + { + if (!enableQueryExecutionTracing) + { + return null; + } + queryExecutionTracers.TryGetValue(queryIdentifier, out var tracer); + return tracer; + } + + //internal QueryExecutionTracer? CreateExecutionTracer(IntPtr queryIdentifier, DuckDBPreparedStatement preparedStatement, int queryIndex) + //{ + // if (!enableQueryExecutionTracing) + // { + // return null; + // } + + // if (!queryExecutionTimers.TryGetValue(queryIdentifier, out var queryExecutionTimer)) + // { + // queryExecutionTimer = new QueryExecutionTimer(preparedStatement, queryIndex); + // queryExecutionTimers[queryIdentifier] = queryExecutionTimer; + // } + + // return new QueryExecutionTracer(queryExecutionTimer, queryIdentifier); + //} + + //internal void StartTimer() + //{ + // if (!startExecutionTimestamp.HasValue) + // { + // startExecutionTimestamp = TimerUtils.TimerCurrent(); + // startExecutionTime = TimerUtils.Now(); + // } + //} + + //internal void StopTimer() + //{ + // ReleaseAndUpdateExecutionTimer(); + //} + + //internal void ReleaseAndUpdateExecutionTimer() + //{ + // if (startExecutionTimestamp.HasValue) + // { + // uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + // executionTime += elapsed; + // endExecutionTime = startExecutionTime.AddTicks(elapsed); + + // startExecutionTimestamp = null; + // } + //} + + internal void UpdateStatistics() + { + // update connection time + if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) + { + connectionTime = closeTimestamp - openTimestamp; + } + else + { + connectionTime = long.MaxValue; + } + } + + internal void ReadMetrics(DuckDBNativeConnection connection, int index) + { + var profile = new ProfilingInfo(connection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + metrics[index] = curMetrics; + } + } + + internal IDictionary GetDictionary() + { + const int Count = 18; + var dictionary = new Dictionary(Count) + { + { "StartTime", startExecutionTime }, + { "EndTime", endExecutionTime }, + { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, + { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, + { "MetricsCount", metrics.Count }, + { "Metrics", metrics } + }; + Debug.Assert(dictionary.Count == Count); + return dictionary; + } + + internal void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + connectionTime = 0; + startExecutionTime = default; + endExecutionTime = default; + } + } +} diff --git a/DuckDB.NET.Data/Connection/QueryExecutionTimer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs similarity index 61% rename from DuckDB.NET.Data/Connection/QueryExecutionTimer.cs rename to DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs index 0f10582a..f2aced86 100644 --- a/DuckDB.NET.Data/Connection/QueryExecutionTimer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs @@ -1,11 +1,10 @@ using DuckDB.NET.Data.Common; using System.Diagnostics; -namespace DuckDB.NET.Data.Connection +namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class QueryExecutionTimer + internal sealed class StatementExecutionStatistics: ExecutionStatistics { - // internal values that are not exposed through properties internal long? startExecutionTimestamp; @@ -17,13 +16,13 @@ internal sealed class QueryExecutionTimer private readonly DuckDBPreparedStatement preparedStatement; private readonly int queryIndex; - internal QueryExecutionTimer(DuckDBPreparedStatement preparedStatement, int queryIndex) + internal StatementExecutionStatistics(DuckDBPreparedStatement preparedStatement, int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { this.preparedStatement = preparedStatement; this.queryIndex = queryIndex; } - internal void StartTimer() + internal override void StartTimer() { if (!startExecutionTimestamp.HasValue) { @@ -32,7 +31,7 @@ internal void StartTimer() } } - internal void StopTimer() + internal override void StopTimer() { ReleaseAndUpdateExecutionTimer(); } @@ -49,58 +48,37 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal void UpdateStatistics() + internal override void AcquireMetrics() { - // update connection time - if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) - { - connectionTime = closeTimestamp - openTimestamp; - } - else - { - connectionTime = long.MaxValue; - } - } - - internal void ReadMetrics(DuckDBNativeConnection connection, int index) - { - var profile = new ProfilingInfo(connection); + var profile = new ProfilingInfo(duckDBNativeConnection); if (profile.TryPrepare()) { var curMetrics = profile.GetMetrics(); - metrics[index] = curMetrics; + //metrics[index] = curMetrics; } } internal IDictionary GetDictionary() { - const int Count = 18; - var dictionary = new Dictionary(Count) + //const int Count = 18; + var dictionary = new Dictionary(/*Count*/) { { "StartTime", startExecutionTime }, { "EndTime", endExecutionTime }, - { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, - { "MetricsCount", metrics.Count }, - { "Metrics", metrics } + { "StatementIndex", queryIndex } }; - Debug.Assert(dictionary.Count == Count); + //Debug.Assert(dictionary.Count == Count); return dictionary; } - internal void Reset() + internal override void Reset() { executionTime = 0; startExecutionTimestamp = null; - connectionTime = 0; startExecutionTime = default; endExecutionTime = default; } - - public void Dispose() - { - throw new NotImplementedException(); - } } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs new file mode 100644 index 00000000..e5996f50 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs @@ -0,0 +1,11 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class StatementExecutionTracer : ExecutionTracer + { + internal StatementExecutionTracer(StatementExecutionStatistics statistics): base(statistics) { } + + } +} \ No newline at end of file diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 1cac56c0..5c4c33b3 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -10,7 +10,7 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Threading.Tasks; +using System.Reflection.Metadata; using static DuckDB.NET.Native.NativeMethods; namespace DuckDB.NET.Samples @@ -19,11 +19,15 @@ class Program { static void Main(string[] args) { - if (!NativeLibraryHelper.TryLoad()) - { - Console.Error.WriteLine("native assembly not found"); - return; - } + //if (!NativeLibraryHelper.TryLoad()) + //{ + // Console.Error.WriteLine("native assembly not found"); + // return; + //} + NativeDebugResolver.Initialize(); + + PrintVersion(); + //DapperSample(); @@ -35,7 +39,19 @@ static void Main(string[] args) //ParametersBinding(); - TestParallelism().GetAwaiter().GetResult(); + Test(); + } + + private static void PrintVersion() + { + using var con = new DuckDBConnection("data source=:memory:"); + con.Open(); + + using var cmd = con.CreateCommand(); + cmd.CommandText = "PRAGMA Version"; + + var version = cmd.ExecuteScalar(); + Console.WriteLine("DuckDB version: {0}", version); } private static void DapperSample() @@ -145,7 +161,7 @@ private static void LowLevelBindingsSample() } } } - + private static void BulkDataLoad() { Stopwatch stopwatch = Stopwatch.StartNew(); @@ -175,8 +191,8 @@ private static void BulkDataLoad() var row = appender.CreateRow(); row .AppendValue(i) - .AppendValue(Convert.ToSingle(i+2)) - .AppendValue(Convert.ToDouble(i+4)) + .AppendValue(Convert.ToSingle(i + 2)) + .AppendValue(Convert.ToDouble(i + 4)) .AppendValue($"varchar {i.ToString()}") .EndRow(); } @@ -193,7 +209,7 @@ private static void BulkDataLoad() Console.WriteLine($"ElapsedMilliseconds {stopwatch.ElapsedMilliseconds}"); Console.WriteLine($"connection state before {connection.State}"); Console.WriteLine($"connection state after {connection.State}"); - + } private static void ParametersBinding() @@ -220,7 +236,7 @@ private static void ParametersBinding() command.CommandText = "INSERT INTO TestTable (foo, bar, baz) VALUES ($testDicto, $testDictoList, $testList)"; command.Parameters.Add(new DuckDBParameter(testDicto)); command.Parameters.Add(new DuckDBParameter(testDictoList)); - command.Parameters.Add(new DuckDBParameter(testList)); + command.Parameters.Add(new DuckDBParameter(testList)); command.ExecuteNonQuery(); command.CommandText = "SELECT foo, bar, baz FROM TestTable"; @@ -228,48 +244,58 @@ private static void ParametersBinding() PrintQueryResults(reader); } - private async static ValueTask TestParallelism() + private static void Test() { - var values15 = @" -SET allowed_directories = ['C:\ProgramData\IrionDQ\latest\.duckdb\extensions']; -SET temp_directory = 'C:\WINDOWS\SystemTemp\IrionDQ\.duckdb\.tmp'; -INSTALL mssql FROM community; -LOAD mssql; -CREATE OR REPLACE SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10 (TYPE mssql, HOST 'NB242', PORT 1433, DATABASE 'IrionDQWorkinglatest', USER 'IrionDQ', PASSWORD 'vA9MJiNpVlwSrmV8OaU', USE_ENCRYPT FALSE, CATALOG TRUE, SCHEMA_FILTER '^(idq10|idq1)$'); -ATTACH '' AS TO_MSSQL (TYPE mssql, SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10); -"; - - var options = new ParallelOptions - { - MaxDegreeOfParallelism = 8//Environment.ProcessorCount - }; - await Parallel.ForEachAsync(Enumerable.Range(0, 100), options, (i, ct) => + var script = @" +CREATE OR REPLACE SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10 ( +TYPE mssql, HOST 'NB242', PORT 1433, DATABASE 'IrionDQWorkinglatest', USER 'IrionDQ', PASSWORD 'vA9MJiNpVlwSrmV8OaU', USE_ENCRYPT FALSE, CATALOG TRUE, SCHEMA_FILTER '^(idq10|idq1)$'); +ATTACH '' AS TO_MSSQL (TYPE mssql, SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10);"; + + for (int i = 0; i < 100; i++) { using var con = new DuckDBConnection("data source=:memory:"); con.Open(); using var cmd = con.CreateCommand(); - cmd.CommandText = "PRAGMA Version;"; - using (var reader = cmd.ExecuteReader()) - PrintQueryResults(reader); + //cmd.CommandText = "PRAGMA Version;"; + //using var reader = cmd.ExecuteReader(); + //PrintQueryResults(reader); - cmd.CommandText = "SELECT database_name FROM duckdb_databases()"; - using (var reader1 = cmd.ExecuteReader()) - while (reader1.Read()) ; + //cmd.CommandText = "SELECT database_name FROM duckdb_databases()"; + //using var reader1 = cmd.ExecuteReader(); //PrintQueryResults(reader1); - // Prefer SQL without INSTALL here (LOAD + CREATE SECRET + ATTACH only) - cmd.CommandText = values15; + cmd.CommandText = script; cmd.ExecuteNonQuery(); - cmd.CommandText = "DETACH TO_MSSQL"; - cmd.ExecuteNonQuery(); + //cmd.CommandText = "DETACH TO_MSSQL"; + //cmd.ExecuteNonQuery(); + + Console.WriteLine($"----{i}----"); + } + + // using var con = new DuckDBConnection("data source=:memory:"); + // con.Open(); + + // using var cmd = con.CreateCommand(); + + // cmd.CommandText = @" + //CALL enable_profiling( + // format := 'json', + // save_location := '/path/to/output.json', + // coverage := 'select', + // mode := 'standard', + // metrics := ['QUERY_NAME', 'LATENCY', 'OPERATOR_TIMING'] + //);"; + // cmd.ExecuteNonQuery(); + + // cmd.CommandText = "SELECT a:1;"; + // var scalar = cmd.ExecuteScalar(); + + // Console.WriteLine(scalar); - Console.WriteLine($"---- {i}"); - return ValueTask.CompletedTask; - }); } private static void PrintQueryResults(DbDataReader queryResult) @@ -281,7 +307,7 @@ private static void PrintQueryResults(DbDataReader queryResult) } Console.WriteLine(); - + while (queryResult.Read()) { for (int ordinal = 0; ordinal < queryResult.FieldCount; ordinal++) From 258461f730a31e70b179c3a48b1699b043837d4b Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 8 May 2026 17:27:08 +0200 Subject: [PATCH 04/43] review of profiling info and statistics management --- DuckDB.NET.Data/DuckDBCommand.cs | 4 +- DuckDB.NET.Data/DuckDBConnection.cs | 59 +++---- .../PreparedStatement/PreparedStatement.cs | 41 ++--- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 20 +-- .../Profiling/ProfilingInfoMetrics.cs | 110 ++++++++++++ .../Statistics/ConnectionStatistics.cs | 156 ++++++++++++++++++ .../Profiling/Statistics/ProfilingSummary.cs | 27 +++ .../Statistics/QueryExecutionTracer.cs | 39 ----- ...xecutionStatistics.cs => QueryProfiler.cs} | 25 +-- .../Statistics/QueryProfilerTracer.cs | 44 +++++ .../Profiling/Statistics/SqlStatistics.cs | 155 ----------------- .../Statistics/StatementExecutionTracer.cs | 11 -- ...tionStatistics.cs => StatementProfiler.cs} | 4 +- .../Statistics/StatementProfilerTracer.cs | 23 +++ DuckDB.NET.Samples/Program.cs | 70 +++----- 15 files changed, 452 insertions(+), 336 deletions(-) create mode 100644 DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs rename DuckDB.NET.Data/Profiling/Statistics/{QueryExecutionStatistics.cs => QueryProfiler.cs} (75%) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs rename DuckDB.NET.Data/Profiling/Statistics/{StatementExecutionStatistics.cs => StatementProfiler.cs} (91%) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index d9b41d8f..a0b67b3c 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -66,7 +66,7 @@ public override int ExecuteNonQuery() { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!, CommandText, parameters, UseStreamingMode); + var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); var count = 0; @@ -102,7 +102,7 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!, CommandText, parameters, UseStreamingMode); + var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); var reader = new DuckDBDataReader(this, results, behavior); diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 7197fa3d..10bd3042 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -18,7 +18,7 @@ public partial class DuckDBConnection : DbConnection private static readonly StateChangeEventArgs FromOpenToClosedEventArgs = new(ConnectionState.Open, ConnectionState.Closed); // Statistics support - internal SqlStatistics statistics; + internal ConnectionStatistics? statistics; private bool collectstats; #region Protected Properties @@ -31,11 +31,6 @@ public partial class DuckDBConnection : DbConnection internal DuckDBConnectionString ParsedConnection => parsedConnection ??= DuckDBConnectionStringBuilder.Parse(ConnectionString); - internal SqlStatistics Statistics - { - get => statistics; - } - public DuckDBConnection() { ConnectionString = string.Empty; @@ -82,6 +77,9 @@ public override string DataSource public DuckDBNativeConnection NativeConnection => connectionReference?.NativeConnection ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native connection."); + /// + /// Gets the underlying native DuckDB database instance associated with the current connection. + /// public DuckDBDatabase NativeDatabase => connectionReference?.FileReferenceCounter.Database ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native database."); @@ -106,37 +104,32 @@ public bool ProfilingEnabled if (value) { // start - if (statistics == null) - { - statistics = new SqlStatistics(NativeConnection, value); - } + statistics ??= new ConnectionStatistics(NativeConnection, value); } else { // stop - if (statistics != null) - { - statistics.closeTimestamp = TimerUtils.TimerCurrent(); - } + statistics?.closeTimestamp = TimerUtils.TimerCurrent(); } collectstats = value; } } /// - /// Retrieves the current set of SQL statistics as a dictionary. + /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. + /// If profiling is not enabled, this method will return an empty summary. /// - /// A dictionary containing the current SQL statistics. If no statistics are available, returns an empty dictionary. - public IDictionary RetrieveStatistics() + /// A containing the collected statistics. + public ProfilingSummary RetrieveStatistics() { - if (Statistics != null) + if (statistics != null) { UpdateStatistics(); - return Statistics.GetDictionary(); + return statistics.GetSummary(); } else { - return new SqlStatistics(NativeConnection, collectstats).GetDictionary(); + return new ConnectionStatistics(NativeConnection, collectstats).GetSummary(); } } @@ -151,17 +144,6 @@ public void QueryPlanCollectionThreshold(int threshold) throw new NotImplementedException(); } - private void UpdateStatistics() - { - if (ConnectionState.Open == State) - { - // update timestamp - statistics.closeTimestamp = TimerUtils.TimerCurrent(); - } - // delegate the rest of the work to the SqlStatistics class - Statistics.UpdateStatistics(); - } - public override string ServerVersion => NativeMethods.Startup.DuckDBLibraryVersion(); public override ConnectionState State => connectionState; @@ -203,7 +185,7 @@ public override void Open() connectionState = ConnectionState.Open; - statistics.openTimestamp = TimerUtils.TimerCurrent(); + statistics?.openTimestamp = TimerUtils.TimerCurrent(); OnStateChange(FromClosedToOpenEventArgs); } @@ -326,6 +308,8 @@ protected override void Dispose(bool disposing) { Close(); } + + statistics?.Dispose(); } base.Dispose(disposing); @@ -374,4 +358,15 @@ public DuckDBQueryProgress GetQueryProgress() EnsureConnectionOpen(); return NativeMethods.Startup.DuckDBQueryProgress(NativeConnection); } + + private void UpdateStatistics() + { + if (ConnectionState.Open == State) + { + // update timestamp + statistics?.closeTimestamp = TimerUtils.TimerCurrent(); + } + // delegate the rest of the work to the SqlStatistics class + statistics?.UpdateStatistics(); + } } diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index d2cd29e2..6fa918b2 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Connection; +using DuckDB.NET.Data.Profiling.Statistics; using System.Linq; namespace DuckDB.NET.Data.PreparedStatement; @@ -6,22 +7,21 @@ namespace DuckDB.NET.Data.PreparedStatement; internal sealed class PreparedStatement : IDisposable { private readonly DuckDBPreparedStatement statement; - private readonly IntPtr queryIdentifier; - private PreparedStatement(DuckDBPreparedStatement statement, IntPtr queryIdentifier) + private PreparedStatement(DuckDBPreparedStatement statement) { this.statement = statement; - this.queryIdentifier = queryIdentifier; } - public static IEnumerable PrepareMultiple(DuckDBConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) + public static IEnumerable PrepareMultiple(DuckDBNativeConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) { - var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection.NativeConnection, query, out var extractedStatements); + var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection, query, out var extractedStatements); using (extractedStatements) - { + { + _ = ConnectionStatistics.TryGetFor(connection, out var stats); // Initialize the query tracer for the entire batch of statements. The tracer will be responsible for tracking the execution of all statements within this batch. - using var queryTracer = connection.statistics?.CreateQueryTracer(extractedStatements.ToHandle(), statementCount); + using var queryTracer = stats?.CreateQueryProfilerTracer(extractedStatements.ToHandle(), statementCount); queryTracer?.StartTimer(); if (statementCount <= 0) @@ -35,14 +35,14 @@ public static IEnumerable PrepareMultiple(DuckDBConnection connect for (int index = 0; index < statementCount; index++) { - var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection.NativeConnection, extractedStatements, index, out var statement); + var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, index, out var statement); - // Initialize the statement tracer for the current statement. This allows for detailed tracing of each individual statement within the batch - queryTracer?.PrepareStatementTracer(statement, index); + var statementTracer = queryTracer?.CreateStatementProfilerTracer(statement, index); + StatementProfilerTracer.Current = statementTracer; if (status.IsSuccess()) { - using var preparedStatement = new PreparedStatement(statement, extractedStatements.ToHandle()); + using var preparedStatement = new PreparedStatement(statement); yield return preparedStatement.Execute(parameters, useStreamingMode, connection); } else @@ -54,22 +54,24 @@ public static IEnumerable PrepareMultiple(DuckDBConnection connect errorMessage = "DuckDBQuery failed"; } - using var statementTracer = queryTracer?.GetStatementTracer(statement); - statementTracer?.SetState(status, errorMessage); + // Initialize the statement tracer for the current statement. This allows for detailed tracing of each individual statement within the batch + using (statementTracer) + { + statementTracer?.SetState(status, errorMessage); + } + StatementProfilerTracer.Current = null; - throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection.NativeConnection)); + throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); } } } } - private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBConnection connection) + private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBNativeConnection connection) { - // Tracing is discriminated by query identifier, which is a combination of the query text and the index of the statement in the case of multiple statements. // This allows for more granular tracing of individual statements within a batch. - var queryTracer = connection.statistics?.GetQueryTracer(queryIdentifier); - using var statementTracer = queryTracer?.GetStatementTracer(statement); + using var statementTracer = StatementProfilerTracer.Current; statementTracer?.StartTimer(); BindParameters(statement, parameterCollection); @@ -96,13 +98,14 @@ private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool throw new OperationCanceledException(); } - var innerException = UdfExceptionStore.Retrieve(connection.NativeConnection); + var innerException = UdfExceptionStore.Retrieve(connection); throw innerException != null ? new DuckDBException(errorMessage, innerException) : new DuckDBException(errorMessage, errorType); } statementTracer?.AcquireMetrics(); + StatementProfilerTracer.Current = null; return queryResult; } diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 64e889cb..f25d3e61 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -1,17 +1,18 @@ -internal sealed class ProfilingInfo +using DuckDB.NET.Data.Profiling; + +internal sealed class ProfilingInfo { - private readonly DuckDBNativeConnection _connection; + private readonly DuckDBNativeConnection connection; private DuckDBProfilingInfoWrapper? duckDBProfilingInfoWrapper; internal ProfilingInfo(DuckDBNativeConnection connection) { - _connection = connection; + this.connection = connection; } internal bool TryPrepare() { - // 1. Get the profiling info root node - using var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(_connection); + using var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(connection); if (profilingInfo == null) { @@ -19,13 +20,10 @@ internal bool TryPrepare() } duckDBProfilingInfoWrapper = profilingInfo; - - // 2. Print the profiling info recursively - //PrintProfilingNode(profilingInfo, 0); return true; } - internal IDictionary GetMetrics() + internal ProfilingInfoMetrics GetMetrics() { if (duckDBProfilingInfoWrapper == null) { @@ -35,9 +33,9 @@ internal IDictionary GetMetrics() var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); if (metricsValue.IsNull()) { - return new Dictionary(); + return default; } - return metricsValue.GetMapValue(); + return ProfilingInfoMetrics.FromDictionary(metricsValue.GetMapValue()); } private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int indent) diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs b/DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs new file mode 100644 index 00000000..62ac82d9 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace DuckDB.NET.Data.Profiling; + +/// +/// Strongly-typed, immutable view over profiling metric entries that can be created from +/// an and exported back to a dictionary. +/// +public readonly record struct ProfilingInfoMetrics( + double? ElapsedMilliseconds, + long? Rows, + long? ResultSetSize, + long? ScannedRows, + long? BytesRead, + long? BytesWritten, + long? TotalMemoryAllocated, + long? PeakMemoryAllocation, + int? Spills, + double? WalWriteLatencyMilliseconds, + string? Statement) +{ + + public static ProfilingInfoMetrics FromDictionary(IDictionary dict) + { + if (dict is null) throw new ArgumentNullException(nameof(dict)); + + static double? ToNullableDouble(object? v) + { + if (v is null) return null; + if (v is double d) return d; + if (v is float f) return Convert.ToDouble(f); + if (v is decimal dec) return Convert.ToDouble(dec); + if (v is long l) return Convert.ToDouble(l); + if (v is int i) return Convert.ToDouble(i); + if (v is string s && double.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; + if (v is TimeSpan ts) return ts.TotalMilliseconds; + if (v is IConvertible conv) return Convert.ToDouble(conv); + return null; + } + + static long? ToNullableLong(object? v) + { + if (v is null) return null; + if (v is long l) return l; + if (v is int i) return i; + if (v is short s) return s; + if (v is double d) return Convert.ToInt64(d); + if (v is decimal dec) return Convert.ToInt64(dec); + if (v is string str && long.TryParse(str, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; + if (v is IConvertible conv) return Convert.ToInt64(conv); + return null; + } + + static int? ToNullableInt(object? v) + { + if (v is null) return null; + if (v is int i) return i; + if (v is long l) return (int)l; + if (v is string s && int.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; + if (v is IConvertible conv) return Convert.ToInt32(conv); + return null; + } + + dict.TryGetValue("Elapsed", out var elapsed); + dict.TryGetValue("Rows", out var rows); + dict.TryGetValue("ResultSetSize", out var resultSetSize); + dict.TryGetValue("ScannedRows", out var scannedRows); + dict.TryGetValue("BytesRead", out var bytesRead); + dict.TryGetValue("BytesWritten", out var bytesWritten); + dict.TryGetValue("TotalMemoryAllocated", out var totalMemoryAllocated); + dict.TryGetValue("PeakMemoryAllocation", out var peakMemoryAllocation); + dict.TryGetValue("Spills", out var spills); + dict.TryGetValue("WALWriteLatency", out var walWriteLatency); + dict.TryGetValue("Statement", out var statement); + + return new ProfilingInfoMetrics( + ElapsedMilliseconds: ToNullableDouble(elapsed), + Rows: ToNullableLong(rows), + ResultSetSize: ToNullableLong(resultSetSize), + ScannedRows: ToNullableLong(scannedRows), + BytesRead: ToNullableLong(bytesRead), + BytesWritten: ToNullableLong(bytesWritten), + TotalMemoryAllocated: ToNullableLong(totalMemoryAllocated), + PeakMemoryAllocation: ToNullableLong(peakMemoryAllocation), + Spills: ToNullableInt(spills), + WalWriteLatencyMilliseconds: ToNullableDouble(walWriteLatency), + Statement: statement?.ToString()); + } + + public IDictionary ToDictionary() + { + var dict = new Dictionary(11); + + if (ElapsedMilliseconds is not null) dict["Elapsed"] = ElapsedMilliseconds; + if (Rows is not null) dict["Rows"] = Rows; + if (ResultSetSize is not null) dict["ResultSetSize"] = ResultSetSize; + if (ScannedRows is not null) dict["ScannedRows"] = ScannedRows; + if (BytesRead is not null) dict["BytesRead"] = BytesRead; + if (BytesWritten is not null) dict["BytesWritten"] = BytesWritten; + if (TotalMemoryAllocated is not null) dict["TotalMemoryAllocated"] = TotalMemoryAllocated; + if (PeakMemoryAllocation is not null) dict["PeakMemoryAllocation"] = PeakMemoryAllocation; + if (Spills is not null) dict["Spills"] = Spills; + if (WalWriteLatencyMilliseconds is not null) dict["WALWriteLatency"] = WalWriteLatencyMilliseconds; + if (Statement is not null) dict["Statement"] = Statement; + + return dict; + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs new file mode 100644 index 00000000..dd3a036e --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -0,0 +1,156 @@ +using DuckDB.NET.Data.Common; +using System.Collections.Concurrent; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class ConnectionStatistics : IDisposable + { + + // internal values that are not exposed through properties + private static readonly ConcurrentDictionary ByNativeConnection + = new(ReferenceEqualityComparer.Instance); + + + internal long closeTimestamp; + internal long openTimestamp; + internal long? startExecutionTimestamp; + private readonly bool enableQueryExecutionTracing; + private readonly DuckDBNativeConnection duckDBNativeConnection; + private readonly Dictionary queryExecutionTracers = []; + private readonly Dictionary> queryExecutionStatistics = []; + private bool isDisposed = false; + + // internal values that are exposed through properties + internal long executionTime; + internal long connectionTime; + internal DateTimeOffset startExecutionTime; + internal DateTimeOffset endExecutionTime; + internal Dictionary metrics = []; + + /// + /// Initializes a new instance of the class. + /// + /// The native DuckDB connection. + /// Indicates whether query execution tracing is enabled. + internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) + { + this.enableQueryExecutionTracing = enableQueryExecutionTracing; + this.duckDBNativeConnection = duckDBNativeConnection; + ByNativeConnection.TryAdd(duckDBNativeConnection, this); + } + + /// + /// Attempts to retrieve a associated with the specified connection. + /// + /// The native DuckDB connection for which to obtain statistics. + /// When this method returns, contains the statistics for the specified connection if found; otherwise, . + /// if statistics are found for the specified connection; otherwise, . + public static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) + => ByNativeConnection.TryGetValue(nativeConn, out stats); + + /// + /// Creates a new for the specified query if query execution tracing is enabled. + /// + /// A pointer that uniquely identifies the query for which the profiler tracer is created. + /// The number of statements in the query to be profiled. Must be non-negative. + /// A new instance of if query execution tracing is enabled; otherwise, . + internal QueryProfilerTracer? CreateQueryProfilerTracer(IntPtr queryIdentifier, int statementCount) + { + if (!enableQueryExecutionTracing) + { + return null; + } + + var executionStatistics = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); + var tracer = new QueryProfilerTracer(executionStatistics, duckDBNativeConnection); + queryExecutionTracers[queryIdentifier] = tracer; + + // When a new tracer is created for a query, we also create a new list to hold the execution statistics for that query. + if (!queryExecutionStatistics.TryGetValue(queryIdentifier, out var executionStatisticsList)) + { + executionStatisticsList = []; + queryExecutionStatistics[queryIdentifier] = executionStatisticsList; + } + executionStatisticsList.Add(executionStatistics); + + return tracer; + } + + internal QueryProfilerTracer? GetQueryTracer(IntPtr queryIdentifier) + { + if (!enableQueryExecutionTracing) + { + return null; + } + queryExecutionTracers.TryGetValue(queryIdentifier, out var tracer); + return tracer; + } + + internal StatementProfilerTracer? GetStatementTracer(IntPtr queryIdentifier, DuckDBPreparedStatement preparedStatement) + { + var queryTracer = GetQueryTracer(queryIdentifier); + return queryTracer?.GetOrCreateStatementProfilerTracer(preparedStatement); + } + + internal void UpdateStatistics() + { + // update connection time + if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) + { + connectionTime = closeTimestamp - openTimestamp; + } + else + { + connectionTime = long.MaxValue; + } + } + + internal void ReadMetrics(DuckDBNativeConnection connection, int index) + { + var profile = new ProfilingInfo(connection); + + if (profile.TryPrepare()) + { + var curMetrics = profile.GetMetrics(); + metrics[index] = curMetrics; + } + } + + internal ProfilingSummary GetSummary() + { + return new ProfilingSummary( + startExecutionTime, + endExecutionTime, + TimerUtils.TimerToMilliseconds(connectionTime), + TimerUtils.TimerToMilliseconds(executionTime), + metrics.Count, + [.. metrics.Values]); + } + + internal void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + connectionTime = 0; + startExecutionTime = default; + endExecutionTime = default; + } + + // call on connection close/dispose + private void RemoveMapping() + { + ByNativeConnection.TryRemove(duckDBNativeConnection, out _); + } + + public void Dispose() + { + if (!isDisposed) + { + RemoveMapping(); + isDisposed = true; + } + } + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs new file mode 100644 index 00000000..bff696b4 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using DuckDB.NET.Data.Profiling; + +namespace DuckDB.NET.Data.Profiling.Statistics; + +public readonly record struct ProfilingSummary( + DateTimeOffset StartTime, + DateTimeOffset EndTime, + double ConnectionTimeMilliseconds, + double ExecutionTimeMilliseconds, + int MetricsCount, + ProfilingInfoMetrics[] Metrics) +{ + public IDictionary ToDictionary() + { + return new Dictionary(6) + { + ["StartTime"] = StartTime, + ["EndTime"] = EndTime, + ["ConnectionTime"] = ConnectionTimeMilliseconds, + ["ExecutionTime"] = ExecutionTimeMilliseconds, + ["MetricsCount"] = MetricsCount, + ["Metrics"] = Metrics + }; + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs deleted file mode 100644 index 3c70fc55..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionTracer.cs +++ /dev/null @@ -1,39 +0,0 @@ -using DuckDB.NET.Data.PreparedStatement; -using System.Diagnostics; -using System.Net.NetworkInformation; - -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class QueryExecutionTracer : ExecutionTracer - { - private readonly Dictionary statementTracers = []; - private readonly QueryExecutionStatistics queryExecutionStatistics; - private readonly DuckDBNativeConnection duckDBNativeConnection; - - internal QueryExecutionTracer(QueryExecutionStatistics queryExecutionStatistics, DuckDBNativeConnection duckDBNativeConnection): base(queryExecutionStatistics) - { - this.queryExecutionStatistics = queryExecutionStatistics; - this.duckDBNativeConnection = duckDBNativeConnection; - } - - internal void PrepareStatementTracer(DuckDBPreparedStatement preparedStatement, int statementIndex) - { - if (!statementTracers.ContainsKey(preparedStatement)) - { - var singleStatementStatistics = new StatementExecutionStatistics(preparedStatement, statementIndex, duckDBNativeConnection); - statementTracers[preparedStatement] = new StatementExecutionTracer(singleStatementStatistics); - - queryExecutionStatistics.TryAddExecutionStatistics(singleStatementStatistics, statementIndex); - } - } - - internal StatementExecutionTracer? GetStatementTracer(DuckDBPreparedStatement preparedStatement) - { - if (!statementTracers.TryGetValue(preparedStatement, out var tracer)) - { - return null; - } - return tracer; - } - } -} diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs similarity index 75% rename from DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs rename to DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 51534cc8..859bf81f 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryExecutionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -3,11 +3,11 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class QueryExecutionStatistics : ExecutionStatistics + internal sealed class QueryProfiler : ExecutionStatistics { // internal values that are not exposed through properties internal long? startExecutionTimestamp; - private readonly StatementExecutionStatistics[] executionStatistics; + private readonly StatementProfiler[] statementProfilers; // internal values that are exposed through properties internal long executionTime; @@ -17,19 +17,19 @@ internal sealed class QueryExecutionStatistics : ExecutionStatistics private readonly IntPtr queryIdentifier; private readonly int statementCount; - internal QueryExecutionStatistics(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) + internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { this.statementCount = statementCount; - this.executionStatistics = new StatementExecutionStatistics[statementCount]; + this.statementProfilers = new StatementProfiler[statementCount]; } internal IntPtr QueryIdentifier => queryIdentifier; - internal bool TryAddExecutionStatistics(StatementExecutionStatistics statistics, int queryIndex) + internal bool RegisterStatementProfiler(StatementProfiler statistics, int queryIndex) { - if (executionStatistics[queryIndex] == null) + if (statementProfilers[queryIndex] == null) { - executionStatistics[queryIndex] = statistics; + statementProfilers[queryIndex] = statistics; return true; } return false; @@ -47,17 +47,6 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal void ReadMetrics(DuckDBNativeConnection connection, int index) - { - var profile = new ProfilingInfo(connection); - - if (profile.TryPrepare()) - { - var curMetrics = profile.GetMetrics(); - metrics[index] = curMetrics; - } - } - internal IDictionary GetDictionary() { //const int Count = 18; diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs new file mode 100644 index 00000000..88c493c3 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs @@ -0,0 +1,44 @@ +using System.Diagnostics; +using System.Threading; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class QueryProfilerTracer : ExecutionTracer + { + private readonly Dictionary statementTracers = []; + private readonly QueryProfiler queryProfiler; + private readonly DuckDBNativeConnection duckDBNativeConnection; + + internal QueryProfilerTracer(QueryProfiler queryExecutionStatistics, DuckDBNativeConnection duckDBNativeConnection) : base(queryExecutionStatistics) + { + this.queryProfiler = queryExecutionStatistics; + this.duckDBNativeConnection = duckDBNativeConnection; + } + + internal StatementProfilerTracer CreateStatementProfilerTracer(DuckDBPreparedStatement preparedStatement, int statementIndex) + { + if (statementTracers.ContainsKey(preparedStatement)) + { + throw new InvalidOperationException("Statement profiler tracer already exists for the given prepared statement."); + } + + var statementProfiler = new StatementProfiler(preparedStatement, statementIndex, duckDBNativeConnection); + statementTracers[preparedStatement] = new StatementProfilerTracer(statementProfiler); + + queryProfiler.RegisterStatementProfiler(statementProfiler, statementIndex); + + return statementTracers[preparedStatement]; + } + + internal StatementProfilerTracer GetOrCreateStatementProfilerTracer(DuckDBPreparedStatement preparedStatement, int? statementIndex = null) + { + if (!statementTracers.TryGetValue(preparedStatement, out var tracer)) + { + var index = statementIndex ?? throw new ArgumentNullException(nameof(statementIndex)); + return CreateStatementProfilerTracer(preparedStatement, index); + } + + return tracer; + } + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs deleted file mode 100644 index c7dd8a76..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/SqlStatistics.cs +++ /dev/null @@ -1,155 +0,0 @@ -using DuckDB.NET.Data.Common; -using System.Diagnostics; - -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class SqlStatistics - { - - // internal values that are not exposed through properties - internal long closeTimestamp; - internal long openTimestamp; - internal long? startExecutionTimestamp; - private readonly bool enableQueryExecutionTracing; - private readonly DuckDBNativeConnection duckDBNativeConnection; - private readonly Dictionary queryExecutionTracers = []; - private readonly Dictionary> queryExecutionStatistics = []; - - // internal values that are exposed through properties - internal long executionTime; - internal long connectionTime; - internal DateTimeOffset startExecutionTime; - internal DateTimeOffset endExecutionTime; - internal Dictionary> metrics = []; - - - internal SqlStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) - { - this.enableQueryExecutionTracing = enableQueryExecutionTracing; - this.duckDBNativeConnection = duckDBNativeConnection; - } - - internal QueryExecutionTracer? CreateQueryTracer(IntPtr queryIdentifier, int statementCount) - { - if (!enableQueryExecutionTracing) - { - return null; - } - - var executionStatistics = new QueryExecutionStatistics(queryIdentifier, statementCount, duckDBNativeConnection); - var tracer = new QueryExecutionTracer(executionStatistics, duckDBNativeConnection); - queryExecutionTracers[queryIdentifier] = tracer; - - // When a new tracer is created for a query, we also create a new list to hold the execution statistics for that query. - if (!queryExecutionStatistics.TryGetValue(queryIdentifier, out var executionStatisticsList)) - { - executionStatisticsList = []; - queryExecutionStatistics[queryIdentifier] = executionStatisticsList; - } - executionStatisticsList.Add(executionStatistics); - - return tracer; - } - - internal QueryExecutionTracer? GetQueryTracer(IntPtr queryIdentifier) - { - if (!enableQueryExecutionTracing) - { - return null; - } - queryExecutionTracers.TryGetValue(queryIdentifier, out var tracer); - return tracer; - } - - //internal QueryExecutionTracer? CreateExecutionTracer(IntPtr queryIdentifier, DuckDBPreparedStatement preparedStatement, int queryIndex) - //{ - // if (!enableQueryExecutionTracing) - // { - // return null; - // } - - // if (!queryExecutionTimers.TryGetValue(queryIdentifier, out var queryExecutionTimer)) - // { - // queryExecutionTimer = new QueryExecutionTimer(preparedStatement, queryIndex); - // queryExecutionTimers[queryIdentifier] = queryExecutionTimer; - // } - - // return new QueryExecutionTracer(queryExecutionTimer, queryIdentifier); - //} - - //internal void StartTimer() - //{ - // if (!startExecutionTimestamp.HasValue) - // { - // startExecutionTimestamp = TimerUtils.TimerCurrent(); - // startExecutionTime = TimerUtils.Now(); - // } - //} - - //internal void StopTimer() - //{ - // ReleaseAndUpdateExecutionTimer(); - //} - - //internal void ReleaseAndUpdateExecutionTimer() - //{ - // if (startExecutionTimestamp.HasValue) - // { - // uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); - // executionTime += elapsed; - // endExecutionTime = startExecutionTime.AddTicks(elapsed); - - // startExecutionTimestamp = null; - // } - //} - - internal void UpdateStatistics() - { - // update connection time - if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) - { - connectionTime = closeTimestamp - openTimestamp; - } - else - { - connectionTime = long.MaxValue; - } - } - - internal void ReadMetrics(DuckDBNativeConnection connection, int index) - { - var profile = new ProfilingInfo(connection); - - if (profile.TryPrepare()) - { - var curMetrics = profile.GetMetrics(); - metrics[index] = curMetrics; - } - } - - internal IDictionary GetDictionary() - { - const int Count = 18; - var dictionary = new Dictionary(Count) - { - { "StartTime", startExecutionTime }, - { "EndTime", endExecutionTime }, - { "ConnectionTime", TimerUtils.TimerToMilliseconds(connectionTime) }, - { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, - { "MetricsCount", metrics.Count }, - { "Metrics", metrics } - }; - Debug.Assert(dictionary.Count == Count); - return dictionary; - } - - internal void Reset() - { - executionTime = 0; - startExecutionTimestamp = null; - connectionTime = 0; - startExecutionTime = default; - endExecutionTime = default; - } - } -} diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs deleted file mode 100644 index e5996f50..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionTracer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using DuckDB.NET.Data.Common; -using System.Diagnostics; - -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class StatementExecutionTracer : ExecutionTracer - { - internal StatementExecutionTracer(StatementExecutionStatistics statistics): base(statistics) { } - - } -} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs similarity index 91% rename from DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs rename to DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index f2aced86..981440af 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementExecutionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -3,7 +3,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class StatementExecutionStatistics: ExecutionStatistics + internal sealed class StatementProfiler: ExecutionStatistics { // internal values that are not exposed through properties internal long? startExecutionTimestamp; @@ -16,7 +16,7 @@ internal sealed class StatementExecutionStatistics: ExecutionStatistics private readonly DuckDBPreparedStatement preparedStatement; private readonly int queryIndex; - internal StatementExecutionStatistics(DuckDBPreparedStatement preparedStatement, int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) + internal StatementProfiler(DuckDBPreparedStatement preparedStatement, int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { this.preparedStatement = preparedStatement; this.queryIndex = queryIndex; diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs new file mode 100644 index 00000000..3ac885e1 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs @@ -0,0 +1,23 @@ +using DuckDB.NET.Data.Common; +using System.Diagnostics; +using System.Threading; + +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal sealed class StatementProfilerTracer : ExecutionTracer + { + private static readonly AsyncLocal currentTracer = new(); + + internal StatementProfilerTracer(StatementProfiler statistics): base(statistics) { } + + /// + /// Gets or sets the current for the current + /// execution context. This flows across async calls. + /// + public static StatementProfilerTracer? Current + { + get => currentTracer.Value; + set => currentTracer.Value = value; + } + } +} \ No newline at end of file diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 5c4c33b3..bdabbbf3 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -19,12 +19,12 @@ class Program { static void Main(string[] args) { - //if (!NativeLibraryHelper.TryLoad()) - //{ - // Console.Error.WriteLine("native assembly not found"); - // return; - //} - NativeDebugResolver.Initialize(); + if (!NativeLibraryHelper.TryLoad()) + { + Console.Error.WriteLine("native assembly not found"); + return; + } + //NativeDebugResolver.Initialize(); PrintVersion(); @@ -247,54 +247,30 @@ private static void ParametersBinding() private static void Test() { - var script = @" -CREATE OR REPLACE SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10 ( -TYPE mssql, HOST 'NB242', PORT 1433, DATABASE 'IrionDQWorkinglatest', USER 'IrionDQ', PASSWORD 'vA9MJiNpVlwSrmV8OaU', USE_ENCRYPT FALSE, CATALOG TRUE, SCHEMA_FILTER '^(idq10|idq1)$'); -ATTACH '' AS TO_MSSQL (TYPE mssql, SECRET queryEngineMssqlSecret_033e34a6eea44b09a330e42827d1da10);"; - - for (int i = 0; i < 100; i++) - { - using var con = new DuckDBConnection("data source=:memory:"); - con.Open(); - - using var cmd = con.CreateCommand(); - - //cmd.CommandText = "PRAGMA Version;"; - //using var reader = cmd.ExecuteReader(); - //PrintQueryResults(reader); - - //cmd.CommandText = "SELECT database_name FROM duckdb_databases()"; - //using var reader1 = cmd.ExecuteReader(); - //PrintQueryResults(reader1); - - cmd.CommandText = script; - cmd.ExecuteNonQuery(); + using var con = new DuckDBConnection("data source=:memory:"); + con.Open(); - //cmd.CommandText = "DETACH TO_MSSQL"; - //cmd.ExecuteNonQuery(); + con.ProfilingEnabled = true; - Console.WriteLine($"----{i}----"); - } - // using var con = new DuckDBConnection("data source=:memory:"); - // con.Open(); + using var cmd = con.CreateCommand(); - // using var cmd = con.CreateCommand(); + cmd.CommandText = @" + CALL enable_profiling( + format := 'json', + save_location := '/path/to/output.json', + coverage := 'select', + mode := 'standard', + metrics := ['QUERY_NAME', 'LATENCY', 'OPERATOR_TIMING'] + );"; + cmd.ExecuteNonQuery(); - // cmd.CommandText = @" - //CALL enable_profiling( - // format := 'json', - // save_location := '/path/to/output.json', - // coverage := 'select', - // mode := 'standard', - // metrics := ['QUERY_NAME', 'LATENCY', 'OPERATOR_TIMING'] - //);"; - // cmd.ExecuteNonQuery(); + cmd.CommandText = "SELECT a:1;"; + using var reader = cmd.ExecuteReader(); - // cmd.CommandText = "SELECT a:1;"; - // var scalar = cmd.ExecuteScalar(); + var metrics = con.RetrieveStatistics(); - // Console.WriteLine(scalar); + PrintQueryResults(reader); } From 97d1f3bf7e3b723d841ce25adef2ba58167f1b2b Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 8 May 2026 18:24:26 +0200 Subject: [PATCH 05/43] =?UTF-8?q?aggiornamento=20a=20duckdb=201.5.2=20pi?= =?UTF-8?q?=C3=B9=20alcune=20revisioni?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DuckDB.NET.Data/DuckDBConnection.cs | 4 +- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 16 +++++-- .../Statistics/ConnectionStatistics.cs | 48 ++++--------------- .../Statistics/ExecutionStatistics.cs | 36 +------------- .../Statistics/ExecutionStatisticsBase.cs | 37 ++++++++++++++ .../Profiling/Statistics/ExecutionTracer.cs | 28 ++++------- .../Profiling/Statistics/IExecutionTracer.cs | 8 +--- .../Statistics/IExecutionTracerBase.cs | 12 +++++ .../{ => Statistics}/ProfilingInfoMetrics.cs | 2 +- .../Statistics/ProfilingQuerySummary.cs | 21 ++++++++ .../Profiling/Statistics/ProfilingSummary.cs | 9 ++-- .../Profiling/Statistics/QueryProfiler.cs | 44 ++++++++--------- .../Statistics/QueryProfilerTracer.cs | 9 ++-- .../Profiling/Statistics/StatementProfiler.cs | 23 ++------- .../Statistics/StatementProfilerTracer.cs | 14 +++++- DuckDB.NET.Samples/Program.cs | 1 - 16 files changed, 152 insertions(+), 160 deletions(-) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs create mode 100644 DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs rename DuckDB.NET.Data/Profiling/{ => Statistics}/ProfilingInfoMetrics.cs (99%) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 10bd3042..e8670759 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -125,11 +125,11 @@ public ProfilingSummary RetrieveStatistics() if (statistics != null) { UpdateStatistics(); - return statistics.GetSummary(); + return statistics.GetProfilingSummary(); } else { - return new ConnectionStatistics(NativeConnection, collectstats).GetSummary(); + return new ConnectionStatistics(NativeConnection, collectstats).GetProfilingSummary(); } } diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index f25d3e61..6404b13a 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -1,9 +1,10 @@ -using DuckDB.NET.Data.Profiling; +using DuckDB.NET.Data.Profiling.Statistics; -internal sealed class ProfilingInfo +internal sealed class ProfilingInfo: IDisposable { private readonly DuckDBNativeConnection connection; private DuckDBProfilingInfoWrapper? duckDBProfilingInfoWrapper; + private bool isDisposed = false; internal ProfilingInfo(DuckDBNativeConnection connection) { @@ -12,7 +13,7 @@ internal ProfilingInfo(DuckDBNativeConnection connection) internal bool TryPrepare() { - using var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(connection); + var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(connection); if (profilingInfo == null) { @@ -72,4 +73,13 @@ private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int inde // PrintProfilingNode(child, indent + 1); //} } + + public void Dispose() + { + if (!isDisposed) + { + duckDBProfilingInfoWrapper?.Dispose(); + isDisposed = true; + } + } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index dd3a036e..767b4014 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -16,8 +16,7 @@ private static readonly ConcurrentDictionary queryExecutionTracers = []; - private readonly Dictionary> queryExecutionStatistics = []; + private readonly Dictionary queryProfilers = []; private bool isDisposed = false; // internal values that are exposed through properties @@ -25,7 +24,6 @@ private static readonly ConcurrentDictionary metrics = []; /// /// Initializes a new instance of the class. @@ -63,37 +61,14 @@ public static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionSt return null; } - var executionStatistics = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); - var tracer = new QueryProfilerTracer(executionStatistics, duckDBNativeConnection); - queryExecutionTracers[queryIdentifier] = tracer; + var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); + var tracer = new QueryProfilerTracer(queryProfiler, duckDBNativeConnection); - // When a new tracer is created for a query, we also create a new list to hold the execution statistics for that query. - if (!queryExecutionStatistics.TryGetValue(queryIdentifier, out var executionStatisticsList)) - { - executionStatisticsList = []; - queryExecutionStatistics[queryIdentifier] = executionStatisticsList; - } - executionStatisticsList.Add(executionStatistics); + queryProfilers[queryIdentifier] = queryProfiler; return tracer; } - internal QueryProfilerTracer? GetQueryTracer(IntPtr queryIdentifier) - { - if (!enableQueryExecutionTracing) - { - return null; - } - queryExecutionTracers.TryGetValue(queryIdentifier, out var tracer); - return tracer; - } - - internal StatementProfilerTracer? GetStatementTracer(IntPtr queryIdentifier, DuckDBPreparedStatement preparedStatement) - { - var queryTracer = GetQueryTracer(queryIdentifier); - return queryTracer?.GetOrCreateStatementProfilerTracer(preparedStatement); - } - internal void UpdateStatistics() { // update connection time @@ -107,26 +82,23 @@ internal void UpdateStatistics() } } - internal void ReadMetrics(DuckDBNativeConnection connection, int index) + private IEnumerable ReadSummaries() { - var profile = new ProfilingInfo(connection); - - if (profile.TryPrepare()) + foreach (var queryProfiler in queryProfilers.Values) { - var curMetrics = profile.GetMetrics(); - metrics[index] = curMetrics; + yield return queryProfiler.GetQuerySummary(); } } - internal ProfilingSummary GetSummary() + internal ProfilingSummary GetProfilingSummary() { return new ProfilingSummary( startExecutionTime, endExecutionTime, TimerUtils.TimerToMilliseconds(connectionTime), TimerUtils.TimerToMilliseconds(executionTime), - metrics.Count, - [.. metrics.Values]); + queryProfilers.Values.Count, + [.. ReadSummaries()]); } internal void Reset() diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs index b4d599de..b17d4e0b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs @@ -1,39 +1,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal abstract class ExecutionStatistics(DuckDBNativeConnection duckDBNativeConnection) + internal abstract class ExecutionStatistics(DuckDBNativeConnection duckDBNativeConnection): ExecutionStatisticsBase(duckDBNativeConnection) { - protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; - - protected DuckDBState state; - protected string? errorMessage; - - internal abstract void AcquireMetrics(); - - internal abstract void Reset(); - - internal abstract void StartTimer(); - - internal abstract void StopTimer(); - - internal virtual void SetState(DuckDBState state) - { - this.state = state; - } - - internal virtual void SetState(DuckDBState state, string message) - { - this.state = state; - this.errorMessage = message; - } - - internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) - { - this.state = state; - this.errorMessage = string.Format("%s error: %s", errorType, message); - } - - internal virtual DuckDBState State => state; - - internal virtual string ErrorMessage => errorMessage ?? string.Empty; + internal abstract void AcquireProfilingInfo(); } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs new file mode 100644 index 00000000..bb2d3ae3 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs @@ -0,0 +1,37 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal abstract class ExecutionStatisticsBase(DuckDBNativeConnection duckDBNativeConnection) + { + protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; + + protected DuckDBState state; + protected string? errorMessage; + + internal abstract void Reset(); + + internal abstract void StartTimer(); + + internal abstract void StopTimer(); + + internal virtual void SetState(DuckDBState state) + { + this.state = state; + } + + internal virtual void SetState(DuckDBState state, string message) + { + this.state = state; + this.errorMessage = message; + } + + internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + this.state = state; + this.errorMessage = string.Format("%s error: %s", errorType, message); + } + + internal virtual DuckDBState State => state; + + internal virtual string ErrorMessage => errorMessage ?? string.Empty; + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs index b3a5431e..852a3c76 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs @@ -1,20 +1,15 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal abstract class ExecutionTracer : IExecutionTracer, IDisposable + internal abstract class ExecutionTracer : IExecutionTracerBase, IDisposable { - protected readonly ExecutionStatistics statistics; + protected readonly ExecutionStatisticsBase statistics; private bool disposed; - internal ExecutionTracer(ExecutionStatistics statistics) + internal ExecutionTracer(ExecutionStatisticsBase statistics) { this.statistics = statistics; } - internal virtual void AcquireMetrics() - { - statistics.AcquireMetrics(); - } - internal virtual void StartTimer() { statistics.StartTimer(); @@ -56,36 +51,31 @@ void IDisposable.Dispose() } } - void IExecutionTracer.AcquireMetrics() - { - AcquireMetrics(); - } - - void IExecutionTracer.StartTimer() + void IExecutionTracerBase.StartTimer() { StartTimer(); } - void IExecutionTracer.StopTimer() + void IExecutionTracerBase.StopTimer() { StopTimer(); } - void IExecutionTracer.SetState(DuckDBState state) + void IExecutionTracerBase.SetState(DuckDBState state) { SetState(state); } - void IExecutionTracer.SetState(DuckDBState state, string message) + void IExecutionTracerBase.SetState(DuckDBState state, string message) { SetState(state, message); } - void IExecutionTracer.SetState(DuckDBState state, DuckDBErrorType errorType, string message) + void IExecutionTracerBase.SetState(DuckDBState state, DuckDBErrorType errorType, string message) { SetState(state, errorType, message); } - DuckDBState IExecutionTracer.State => State; + DuckDBState IExecutionTracerBase.State => State; } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs index 6d7fbadc..c9aaeede 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs @@ -1,13 +1,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal interface IExecutionTracer : IDisposable + internal interface IExecutionTracer : IExecutionTracerBase { void AcquireMetrics(); - void StartTimer(); - void StopTimer(); - void SetState(DuckDBState state); - void SetState(DuckDBState state, string message); - void SetState(DuckDBState state, DuckDBErrorType errorType, string message); - DuckDBState State { get; } } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs new file mode 100644 index 00000000..5ad59e12 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs @@ -0,0 +1,12 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + internal interface IExecutionTracerBase : IDisposable + { + void StartTimer(); + void StopTimer(); + void SetState(DuckDBState state); + void SetState(DuckDBState state, string message); + void SetState(DuckDBState state, DuckDBErrorType errorType, string message); + DuckDBState State { get; } + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs similarity index 99% rename from DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs rename to DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs index 62ac82d9..5f1a7877 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfoMetrics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Globalization; -namespace DuckDB.NET.Data.Profiling; +namespace DuckDB.NET.Data.Profiling.Statistics; /// /// Strongly-typed, immutable view over profiling metric entries that can be created from diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs new file mode 100644 index 00000000..2d7829d1 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs @@ -0,0 +1,21 @@ +namespace DuckDB.NET.Data.Profiling.Statistics; + +public readonly record struct ProfilingQuerySummary( + DateTimeOffset StartTime, + DateTimeOffset EndTime, + double ExecutionTimeMilliseconds, + int StatementCount, + ProfilingInfoMetrics[] Infos) +{ + public IDictionary ToDictionary() + { + return new Dictionary(6) + { + ["StartTime"] = StartTime, + ["EndTime"] = EndTime, + ["ExecutionTime"] = ExecutionTimeMilliseconds, + ["StatementCount"] = StatementCount, + ["Infos"] = Infos + }; + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs index bff696b4..98d1ba3c 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using DuckDB.NET.Data.Profiling; namespace DuckDB.NET.Data.Profiling.Statistics; @@ -9,8 +8,8 @@ public readonly record struct ProfilingSummary( DateTimeOffset EndTime, double ConnectionTimeMilliseconds, double ExecutionTimeMilliseconds, - int MetricsCount, - ProfilingInfoMetrics[] Metrics) + int QueryCount, + ProfilingQuerySummary[] QuerySummaryList) { public IDictionary ToDictionary() { @@ -20,8 +19,8 @@ public IDictionary ToDictionary() ["EndTime"] = EndTime, ["ConnectionTime"] = ConnectionTimeMilliseconds, ["ExecutionTime"] = ExecutionTimeMilliseconds, - ["MetricsCount"] = MetricsCount, - ["Metrics"] = Metrics + ["QueryCount"] = QueryCount, + ["QuerySummaryList"] = QuerySummaryList }; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 859bf81f..60ebe59b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -1,9 +1,8 @@ using DuckDB.NET.Data.Common; -using System.Diagnostics; namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class QueryProfiler : ExecutionStatistics + internal sealed class QueryProfiler : ExecutionStatisticsBase { // internal values that are not exposed through properties internal long? startExecutionTimestamp; @@ -13,11 +12,11 @@ internal sealed class QueryProfiler : ExecutionStatistics internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - internal Dictionary> metrics = []; + private Dictionary infos = []; private readonly IntPtr queryIdentifier; private readonly int statementCount; - internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) + internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection) : base(duckDBNativeConnection) { this.statementCount = statementCount; this.statementProfilers = new StatementProfiler[statementCount]; @@ -25,6 +24,8 @@ internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeC internal IntPtr QueryIdentifier => queryIdentifier; + internal int StatementCount => statementCount; + internal bool RegisterStatementProfiler(StatementProfiler statistics, int queryIndex) { if (statementProfilers[queryIndex] == null) @@ -47,18 +48,22 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal IDictionary GetDictionary() + private IEnumerable GetStatementInfo() { - //const int Count = 18; - var dictionary = new Dictionary(/*Count*/) + foreach (var statementProfiler in statementProfilers) { - { "StartTime", startExecutionTime }, - { "EndTime", endExecutionTime }, - { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, - { "StatementCount", statementCount } - }; - //Debug.Assert(dictionary.Count == Count); - return dictionary; + yield return statementProfiler?.Info ?? default; + } + } + + internal ProfilingQuerySummary GetQuerySummary() + { + return new ProfilingQuerySummary( + startExecutionTime, + endExecutionTime, + TimerUtils.TimerToMilliseconds(executionTime), + statementCount, + [.. GetStatementInfo()]); } internal override void StartTimer() @@ -75,17 +80,6 @@ internal override void StopTimer() ReleaseAndUpdateExecutionTimer(); } - internal override void AcquireMetrics() - { - var profile = new ProfilingInfo(duckDBNativeConnection); - - if (profile.TryPrepare()) - { - var curMetrics = profile.GetMetrics(); - //metrics[index] = curMetrics; - } - } - internal override void SetState(DuckDBState state, string message) { this.state = state; diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs index 88c493c3..1a70e59e 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs @@ -1,7 +1,4 @@ -using System.Diagnostics; -using System.Threading; - -namespace DuckDB.NET.Data.Profiling.Statistics +namespace DuckDB.NET.Data.Profiling.Statistics { internal sealed class QueryProfilerTracer : ExecutionTracer { @@ -9,9 +6,9 @@ internal sealed class QueryProfilerTracer : ExecutionTracer private readonly QueryProfiler queryProfiler; private readonly DuckDBNativeConnection duckDBNativeConnection; - internal QueryProfilerTracer(QueryProfiler queryExecutionStatistics, DuckDBNativeConnection duckDBNativeConnection) : base(queryExecutionStatistics) + internal QueryProfilerTracer(QueryProfiler queryProfiler, DuckDBNativeConnection duckDBNativeConnection) : base(queryProfiler) { - this.queryProfiler = queryExecutionStatistics; + this.queryProfiler = queryProfiler; this.duckDBNativeConnection = duckDBNativeConnection; } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 981440af..0d641133 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -12,7 +12,7 @@ internal sealed class StatementProfiler: ExecutionStatistics internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - internal Dictionary> metrics = []; + private ProfilingInfoMetrics info; private readonly DuckDBPreparedStatement preparedStatement; private readonly int queryIndex; @@ -22,6 +22,8 @@ internal StatementProfiler(DuckDBPreparedStatement preparedStatement, int queryI this.queryIndex = queryIndex; } + public ProfilingInfoMetrics Info => info; + internal override void StartTimer() { if (!startExecutionTimestamp.HasValue) @@ -48,31 +50,16 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal override void AcquireMetrics() + internal override void AcquireProfilingInfo() { var profile = new ProfilingInfo(duckDBNativeConnection); if (profile.TryPrepare()) { - var curMetrics = profile.GetMetrics(); - //metrics[index] = curMetrics; + info = profile.GetMetrics(); } } - internal IDictionary GetDictionary() - { - //const int Count = 18; - var dictionary = new Dictionary(/*Count*/) - { - { "StartTime", startExecutionTime }, - { "EndTime", endExecutionTime }, - { "ExecutionTime", TimerUtils.TimerToMilliseconds(executionTime) }, - { "StatementIndex", queryIndex } - }; - //Debug.Assert(dictionary.Count == Count); - return dictionary; - } - internal override void Reset() { executionTime = 0; diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs index 3ac885e1..391e446b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs @@ -4,7 +4,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class StatementProfilerTracer : ExecutionTracer + internal sealed class StatementProfilerTracer : ExecutionTracer, IExecutionTracer { private static readonly AsyncLocal currentTracer = new(); @@ -19,5 +19,17 @@ public static StatementProfilerTracer? Current get => currentTracer.Value; set => currentTracer.Value = value; } + + private StatementProfiler StatementProfiler => (StatementProfiler)statistics; + + internal void AcquireMetrics() + { + StatementProfiler.AcquireProfilingInfo(); + } + + void IExecutionTracer.AcquireMetrics() + { + AcquireMetrics(); + } } } \ No newline at end of file diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index bdabbbf3..702d0c15 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -258,7 +258,6 @@ private static void Test() cmd.CommandText = @" CALL enable_profiling( format := 'json', - save_location := '/path/to/output.json', coverage := 'select', mode := 'standard', metrics := ['QUERY_NAME', 'LATENCY', 'OPERATOR_TIMING'] From b2af0f199d4ae8a8da24a9eb532b8fdb22604f09 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 11 May 2026 09:04:26 +0200 Subject: [PATCH 06/43] benchmark dot net project init --- BenchmarkSuite/BenchmarkSuite.csproj | 12 ++++++++++++ BenchmarkSuite/Benchmarks.cs | 28 ++++++++++++++++++++++++++++ BenchmarkSuite/Program.cs | 12 ++++++++++++ DuckDB.NET.slnx | 1 + 4 files changed, 53 insertions(+) create mode 100644 BenchmarkSuite/BenchmarkSuite.csproj create mode 100644 BenchmarkSuite/Benchmarks.cs create mode 100644 BenchmarkSuite/Program.cs diff --git a/BenchmarkSuite/BenchmarkSuite.csproj b/BenchmarkSuite/BenchmarkSuite.csproj new file mode 100644 index 00000000..da8719f5 --- /dev/null +++ b/BenchmarkSuite/BenchmarkSuite.csproj @@ -0,0 +1,12 @@ + + + + net10.0 + Exe + + + + + + + diff --git a/BenchmarkSuite/Benchmarks.cs b/BenchmarkSuite/Benchmarks.cs new file mode 100644 index 00000000..cdae90a7 --- /dev/null +++ b/BenchmarkSuite/Benchmarks.cs @@ -0,0 +1,28 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.VSDiagnostics; +using System; +using System.Security.Cryptography; + +namespace BenchmarkSuite +{ + // For more information on the VS BenchmarkDotNet Diagnosers see https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet + [CPUUsageDiagnoser] + public class Benchmarks + { + private SHA256 sha256 = SHA256.Create(); + private byte[] data; + + [GlobalSetup] + public void Setup() + { + data = new byte[10000]; + new Random(42).NextBytes(data); + } + + [Benchmark] + public byte[] Sha256() + { + return sha256.ComputeHash(data); + } + } +} diff --git a/BenchmarkSuite/Program.cs b/BenchmarkSuite/Program.cs new file mode 100644 index 00000000..70c79b2f --- /dev/null +++ b/BenchmarkSuite/Program.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Running; + +namespace BenchmarkSuite +{ + internal class Program + { + static void Main(string[] args) + { + var _ = BenchmarkRunner.Run(typeof(Program).Assembly); + } + } +} diff --git a/DuckDB.NET.slnx b/DuckDB.NET.slnx index e7a87142..32d3b74d 100644 --- a/DuckDB.NET.slnx +++ b/DuckDB.NET.slnx @@ -1,4 +1,5 @@ + From b55b126a30d80feae0297429eeb22cfbf6400622 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 11 May 2026 16:06:55 +0200 Subject: [PATCH 07/43] added benchmark project --- DuckDB.NET.slnx | 1 + .../Config/CustomConfig.cs | 28 +++ .../Irion.DuckDB.NET.Benchmark.csproj | 45 ++++ Irion.DuckDB.NET.Benchmark/Program.cs | 31 +++ .../TpchBenchmarks_Baseline.cs | 205 ++++++++++++++++++ 5 files changed, 310 insertions(+) create mode 100644 Irion.DuckDB.NET.Benchmark/Config/CustomConfig.cs create mode 100644 Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj create mode 100644 Irion.DuckDB.NET.Benchmark/Program.cs create mode 100644 Irion.DuckDB.NET.Benchmark/TpchBenchmarks_Baseline.cs diff --git a/DuckDB.NET.slnx b/DuckDB.NET.slnx index e7a87142..5eb9fbe9 100644 --- a/DuckDB.NET.slnx +++ b/DuckDB.NET.slnx @@ -3,5 +3,6 @@ + diff --git a/Irion.DuckDB.NET.Benchmark/Config/CustomConfig.cs b/Irion.DuckDB.NET.Benchmark/Config/CustomConfig.cs new file mode 100644 index 00000000..a85f215f --- /dev/null +++ b/Irion.DuckDB.NET.Benchmark/Config/CustomConfig.cs @@ -0,0 +1,28 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; + +namespace Irion.DuckDB.NET.Benchmark.Config +{ + internal class CustomConfig : ManualConfig + { + public CustomConfig() + { + // bring in default loggers/exporters/columns so output appears + foreach (var logger in DefaultConfig.Instance.GetLoggers()) AddLogger(logger); + foreach (var exporter in DefaultConfig.Instance.GetExporters()) AddExporter(exporter); + foreach (var column in DefaultConfig.Instance.GetColumnProviders()) AddColumnProvider(column); + foreach (var diagnoser in DefaultConfig.Instance.GetDiagnosers()) AddDiagnoser(diagnoser); + + var job = Job.Default.WithId("MyBaseline") + .WithIterationCount(50) + .WithRuntime(CoreRuntime.Core10_0) + .WithStrategy(RunStrategy.Monitoring); + + AddJob(job); + AddDiagnoser(MemoryDiagnoser.Default); + } + } +} diff --git a/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj new file mode 100644 index 00000000..039fac69 --- /dev/null +++ b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj @@ -0,0 +1,45 @@ + + + + net10.0;net8 + Exe + + Irion.DuckDB.NET.Benchmark + Irion.DuckDB.NET.Benchmark + Irion.DuckDB.NET.Benchmark + + + + Full + + + + + + + + + + + + + + + + + + + + + + + + + + false + PreserveNewest + \runtimes + runtimes\%(RecursiveDir)\%(FileName)%(Extension) + + + diff --git a/Irion.DuckDB.NET.Benchmark/Program.cs b/Irion.DuckDB.NET.Benchmark/Program.cs new file mode 100644 index 00000000..d7a0f7a9 --- /dev/null +++ b/Irion.DuckDB.NET.Benchmark/Program.cs @@ -0,0 +1,31 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using Irion.DuckDB.NET.Benchmark.Config; +using System.Diagnostics; +using System.Linq; + +namespace Irion.DuckDB.NET.Benchmark +{ + internal class Program + { + static void Main(string[] args) + { + ManualConfig config; + + if (!Debugger.IsAttached) + { + config = new CustomConfig(); + } + else + { + config = ManualConfig.Create(new DebugInProcessConfig()); + } + + //clean 1.5.2 benchmark + var type = typeof(TpchBenchmarks_Baseline); + + BenchmarkRunner.Run(type, config, args); + } + } +} diff --git a/Irion.DuckDB.NET.Benchmark/TpchBenchmarks_Baseline.cs b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks_Baseline.cs new file mode 100644 index 00000000..e684c8b5 --- /dev/null +++ b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks_Baseline.cs @@ -0,0 +1,205 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using DuckDB.NET.Test.Helpers; +using Irion.DuckDB.NET.Benchmark.Config; +using Perfolizer.Mathematics.OutlierDetection; +using System; +using System.IO; + +namespace Irion.DuckDB.NET.Benchmark +{ + //[CPUUsageDiagnoser] + [Outliers(OutlierMode.DontRemove)] + [Config(typeof(CustomConfig))] + public class TpchBenchmarks_Baseline + { + private DuckDBConnection? connection; + private DuckDBCommand? command; + // Per-iteration connection/command used for extension loading & secret creation + private DuckDBConnection? iterationConnection; + private DuckDBCommand? iterationCommand; + private string? attachedDbPath; + [GlobalSetup] + public void Setup() + { + if (!NativeLibraryHelper.TryLoad()) + { + Console.Error.WriteLine("native assembly not found"); + return; + } + + // Initialize an in-memory DuckDB connection and attempt to install/load the tpch extension. + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + command = connection.CreateCommand(); + try + { + command.CommandText = "INSTALL tpch;"; + command.ExecuteNonQuery(); + command.CommandText = "LOAD tpch;"; + command.ExecuteNonQuery(); + // Generate TPCH sf=1 dataset into the current database + command.CommandText = "CALL dbgen(sf=1);"; + command.ExecuteNonQuery(); + } + catch (Exception) + { + // If the extension isn't available, fall back to a simple generated dataset that approximates scale. + command.CommandText = "CREATE TABLE IF NOT EXISTS lineitem (l_orderkey INTEGER, l_partkey INTEGER, l_quantity INTEGER);"; + command.ExecuteNonQuery(); + // Insert a moderate number of rows to simulate work (1k) + command.CommandText = "BEGIN TRANSACTION;"; + command.ExecuteNonQuery(); + var rnd = new Random(42); + for (int i = 0; i < 1000; i++) + { + command.CommandText = $"INSERT INTO lineitem VALUES ({rnd.Next(1, 100000)}, {rnd.Next(1, 100000)}, {rnd.Next(1, 100)});"; + command.ExecuteNonQuery(); + } + + command.CommandText = "COMMIT;"; + command.ExecuteNonQuery(); + } + } + + [GlobalCleanup] + public void Cleanup() + { + try + { + if (!string.IsNullOrEmpty(attachedDbPath) && File.Exists(attachedDbPath)) + { + File.Delete(attachedDbPath); + } + } + catch + { + } + + command?.Dispose(); + connection?.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + // Each iteration needs a fresh connection where the extension is installed and the secret created. + iterationConnection = new DuckDBConnection("DataSource=:memory:"); + try + { + iterationConnection.Open(); + iterationCommand = iterationConnection.CreateCommand(); + + // Install and load httpfs, then create a secret using the CREATE SECRET syntax. + iterationCommand.CommandText = "INSTALL httpfs;"; + iterationCommand.ExecuteNonQuery(); + iterationCommand.CommandText = "LOAD httpfs;"; + iterationCommand.ExecuteNonQuery(); + } + catch + { + // Extension or CREATE SECRET not available on this environment; clean up the iteration resources + // and leave iterationCommand null so the benchmark method can detect the unsupported case. + iterationCommand?.Dispose(); + iterationConnection?.Dispose(); + iterationCommand = null; + iterationConnection = null; + } + } + + [IterationCleanup] + public void IterationCleanup() + { + iterationCommand?.Dispose(); + iterationConnection?.Dispose(); + iterationCommand = null; + iterationConnection = null; + } + + [Benchmark] + public long CountLineitem() + { + command!.CommandText = "SELECT COUNT(*) FROM lineitem;"; + var result = command.ExecuteScalar(); + return Convert.ToInt64(result ?? 0); + } + + [Benchmark] + public void SimpleAggregation() + { + command!.CommandText = "SELECT l_partkey, SUM(l_quantity) as s FROM lineitem GROUP BY l_partkey LIMIT 100;"; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { /* iterate */ + } + } + + [Benchmark] + public void AttachDatabaseAndQuery() + { + // create a temporary on-disk database and attach it + attachedDbPath = Path.Combine(Path.GetTempPath(), $"duckdb_bench_{Guid.NewGuid():N}.db"); + command!.CommandText = $"ATTACH DATABASE '{attachedDbPath}' AS attached_db;"; + command.ExecuteNonQuery(); + // run a simple query against the attached database (it will be empty) + command.CommandText = "SELECT COUNT(*) FROM attached_db.sqlite_master WHERE 1=0;"; + try + { + command.ExecuteScalar(); + } + catch + { /* ignore — attached DB may not have that table */ + } + + command.CommandText = "DETACH DATABASE attached_db;"; + command.ExecuteNonQuery(); + // cleanup file + try + { + File.Delete(attachedDbPath); + } + catch + { + } + + attachedDbPath = null; + } + + [Benchmark] + public void LoadExtension() + { + // Each iteration needs a fresh connection where the extension is installed and the secret created. + using var connection = new DuckDBConnection("DataSource=:memory:"); + + connection.Open(); + using var command = connection.CreateCommand(); + + // Install and load httpfs, then create a secret using the CREATE SECRET syntax. + command.CommandText = "INSTALL httpfs;"; + command.ExecuteNonQuery(); + command.CommandText = "LOAD httpfs;"; + command.ExecuteNonQuery(); + } + + [Benchmark] + public void CreateSecret() + { + // The extension must have been installed and the secret created during IterationSetup. The + // benchmark itself only verifies the secret/extension is usable, keeping the measured work focused. + if (iterationCommand is null) + { + // Extension or CREATE SECRET not supported in this iteration; skip measurable work. + return; + } + + // Use CREATE SECRET syntax to register fake credentials for the extension. + // If the running DuckDB build does not support CREATE SECRET, an exception will be thrown and + // we will treat this iteration as not supporting the extension. + iterationCommand.CommandText = "CREATE SECRET s3_credentials (type s3, KEY_ID 'AKIAFAKE', SECRET 'fakeSecret');"; + iterationCommand.ExecuteNonQuery(); + + iterationCommand.CommandText = "SELECT 1;"; + _ = iterationCommand.ExecuteScalar(); + } + } +} \ No newline at end of file From 56911b4f063def96bb7250f9e397d5a918752da9 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 11 May 2026 16:32:59 +0200 Subject: [PATCH 08/43] adjusted artifact path --- Irion.DuckDB.NET.Benchmark/Program.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Irion.DuckDB.NET.Benchmark/Program.cs b/Irion.DuckDB.NET.Benchmark/Program.cs index d7a0f7a9..311df375 100644 --- a/Irion.DuckDB.NET.Benchmark/Program.cs +++ b/Irion.DuckDB.NET.Benchmark/Program.cs @@ -2,7 +2,9 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using Irion.DuckDB.NET.Benchmark.Config; +using System; using System.Diagnostics; +using System.IO; using System.Linq; namespace Irion.DuckDB.NET.Benchmark @@ -25,6 +27,8 @@ static void Main(string[] args) //clean 1.5.2 benchmark var type = typeof(TpchBenchmarks_Baseline); + config.ArtifactsPath = Path.Combine("BenchmarkDotNet.Artifacts", $"{type.Name}_{DateTime.Now:yyyyMMdd-HHmmss}"); + BenchmarkRunner.Run(type, config, args); } } From bfb4f4b4c5a8ec56d00ea14e8ab969dec048793d Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 13 May 2026 12:40:18 +0200 Subject: [PATCH 09/43] benchmark and some rework --- BenchmarkSuite/BenchmarkSuite.csproj | 12 - BenchmarkSuite/Benchmarks.cs | 28 -- BenchmarkSuite/Program.cs | 12 - DuckDB.NET.Bindings/DuckDBNativeObjects.cs | 102 ++++++- DuckDB.NET.Data/DuckDBConnection.cs | 238 +++++++++++---- .../Extensions/MetricsExtensions.cs | 92 ++++++ .../Profiling/DuckDBMetricTypeCollection.cs | 11 + DuckDB.NET.Data/Profiling/MetricType.cs | 28 ++ DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 12 +- DuckDB.NET.Data/Profiling/ProfilingOptions.cs | 12 + .../Statistics/ConnectionStatistics.cs | 2 + .../Statistics/ProfilingInfoMetrics.cs | 110 +------ .../Profiling/Statistics/QueryProfiler.cs | 2 +- DuckDB.NET.Samples/Program.cs | 65 ++++- DuckDB.NET.slnx | 1 - Irion.DuckDB.NET.Benchmark/Program.cs | 5 +- Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs | 274 ++++++++++++++++++ 17 files changed, 778 insertions(+), 228 deletions(-) delete mode 100644 BenchmarkSuite/BenchmarkSuite.csproj delete mode 100644 BenchmarkSuite/Benchmarks.cs delete mode 100644 BenchmarkSuite/Program.cs create mode 100644 DuckDB.NET.Data/Extensions/MetricsExtensions.cs create mode 100644 DuckDB.NET.Data/Profiling/DuckDBMetricTypeCollection.cs create mode 100644 DuckDB.NET.Data/Profiling/MetricType.cs create mode 100644 DuckDB.NET.Data/Profiling/ProfilingOptions.cs create mode 100644 Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs diff --git a/BenchmarkSuite/BenchmarkSuite.csproj b/BenchmarkSuite/BenchmarkSuite.csproj deleted file mode 100644 index da8719f5..00000000 --- a/BenchmarkSuite/BenchmarkSuite.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - - net10.0 - Exe - - - - - - - diff --git a/BenchmarkSuite/Benchmarks.cs b/BenchmarkSuite/Benchmarks.cs deleted file mode 100644 index cdae90a7..00000000 --- a/BenchmarkSuite/Benchmarks.cs +++ /dev/null @@ -1,28 +0,0 @@ -using BenchmarkDotNet.Attributes; -using Microsoft.VSDiagnostics; -using System; -using System.Security.Cryptography; - -namespace BenchmarkSuite -{ - // For more information on the VS BenchmarkDotNet Diagnosers see https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet - [CPUUsageDiagnoser] - public class Benchmarks - { - private SHA256 sha256 = SHA256.Create(); - private byte[] data; - - [GlobalSetup] - public void Setup() - { - data = new byte[10000]; - new Random(42).NextBytes(data); - } - - [Benchmark] - public byte[] Sha256() - { - return sha256.ComputeHash(data); - } - } -} diff --git a/BenchmarkSuite/Program.cs b/BenchmarkSuite/Program.cs deleted file mode 100644 index 70c79b2f..00000000 --- a/BenchmarkSuite/Program.cs +++ /dev/null @@ -1,12 +0,0 @@ -using BenchmarkDotNet.Running; - -namespace BenchmarkSuite -{ - internal class Program - { - static void Main(string[] args) - { - var _ = BenchmarkRunner.Run(typeof(Program).Assembly); - } - } -} diff --git a/DuckDB.NET.Bindings/DuckDBNativeObjects.cs b/DuckDB.NET.Bindings/DuckDBNativeObjects.cs index 9dba9689..66a71de9 100644 --- a/DuckDB.NET.Bindings/DuckDBNativeObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBNativeObjects.cs @@ -1,4 +1,6 @@ -namespace DuckDB.NET.Native; +using System.ComponentModel; + +namespace DuckDB.NET.Native; public enum DuckDBState { @@ -249,4 +251,102 @@ public struct DuckDBQueryProgress public double Percentage { get; } public ulong RowsProcessed { get; } public ulong TotalRowsToProcess { get; } +} + +public enum DuckDBMetricType +{ + // Core metrics + CpuTime = 2, + CumulativeCardinality = 4, + CumulativeRowsScanned = 7, + ExtraInfo = 3, + Latency = 11, + QueryName = 0, + ResultSetSize = 10, + [Description("Not available yet see(https://github.com/duckdb/duckdb/issues/22592)")] + RowsReturned = 12, + // Execution metrics + BlockedThreadTime = 1, + SystemPeakBufferMemory = 14, + SystemPeakTempDirSize = 15, + TotalMemoryAllocated = 91, + // File metrics + AttachLoadStorageLatency = 92, + AttachReplayWalLatency = 93, + CheckpointLatency = 94, + CommitLocalStorageLatency = 95, + TotalBytesRead = 16, + TotalBytesWritten = 17, + WaitingToAttachLatency = 96, + WalReplayEntryCount = 97, + WriteToWalLatency = 98, + // Operator metrics + OperatorCardinality = 6, + OperatorName = 13, + OperatorRowsScanned = 8, + OperatorTiming = 9, + OperatorType = 5, + // Optimizer metrics + OptimizerExpressionRewriter = 26, + OptimizerFilterPullup = 27, + OptimizerFilterPushdown = 28, + OptimizerEmptyResultPullup = 29, + OptimizerCteFilterPusher = 30, + OptimizerRegexRange = 31, + OptimizerInClause = 32, + OptimizerJoinOrder = 33, + OptimizerDeliminator = 34, + OptimizerUnnestRewriter = 35, + OptimizerUnusedColumns = 36, + OptimizerStatisticsPropagation = 37, + OptimizerCommonSubexpressions = 38, + OptimizerCommonAggregate = 39, + OptimizerColumnLifetime = 40, + OptimizerBuildSideProbeSide = 41, + OptimizerLimitPushdown = 42, + OptimizerTopN = 43, + OptimizerCompressedMaterialization = 44, + OptimizerDuplicateGroups = 45, + OptimizerReorderFilter = 46, + OptimizerSamplingPushdown = 47, + OptimizerJoinFilterPushdown = 48, + OptimizerExtension = 49, + OptimizerMaterializedCte = 50, + OptimizerSumRewriter = 51, + OptimizerLateMaterialization = 52, + OptimizerCteInlining = 53, + OptimizerRowGroupPruner = 54, + OptimizerTopNWindowElimination = 55, + OptimizerCommonSubplan = 56, + OptimizerJoinElimination = 57, + OptimizerWindowSelfJoin = 58, + // PhaseTiming metrics + AllOptimizers = 18, + CumulativeOptimizerTiming = 19, + PhysicalPlanner = 22, + PhysicalPlannerColumnBinding = 23, + PhysicalPlannerCreatePlan = 25, + PhysicalPlannerResolveTypes = 24, + Planner = 20, + PlannerBinding = 21 +} + +public enum DuckDBProfilingFormat +{ + QueryTree, + Json, + QueryTreeOptimizer, + NoOutput +} + +public enum DuckDBProfilingMode +{ + Standard, + Detailed +} + +public enum DuckDBProfilingCoverage +{ + Select, + All } \ No newline at end of file diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index e8670759..7a8e2b59 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -1,5 +1,6 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.Connection; +using DuckDB.NET.Data.Profiling; using DuckDB.NET.Data.Profiling.Statistics; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; @@ -18,8 +19,9 @@ public partial class DuckDBConnection : DbConnection private static readonly StateChangeEventArgs FromOpenToClosedEventArgs = new(ConnectionState.Open, ConnectionState.Closed); // Statistics support - internal ConnectionStatistics? statistics; + internal ConnectionStatistics? profilingInfo; private bool collectstats; + private ProfilingOptions profilingOptions; #region Protected Properties @@ -84,55 +86,6 @@ public override string DataSource ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native database."); - /// - /// Enables or disables profiling of the connection. When enabled, the connection will collect statistics about query execution times and other relevant metrics. - /// - [DefaultValue(false)] - public bool ProfilingEnabled - { - get - { - return (collectstats); - } - set - { - if (State != ConnectionState.Open) - { - throw new InvalidOperationException("The DuckDBConnection must be open to start collecting statistics."); - } - - if (value) - { - // start - statistics ??= new ConnectionStatistics(NativeConnection, value); - } - else - { - // stop - statistics?.closeTimestamp = TimerUtils.TimerCurrent(); - } - collectstats = value; - } - } - - /// - /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. - /// If profiling is not enabled, this method will return an empty summary. - /// - /// A containing the collected statistics. - public ProfilingSummary RetrieveStatistics() - { - if (statistics != null) - { - UpdateStatistics(); - return statistics.GetProfilingSummary(); - } - else - { - return new ConnectionStatistics(NativeConnection, collectstats).GetProfilingSummary(); - } - } - /// /// Sets the minimum execution time, in milliseconds, required for a query plan to be collected for analysis. /// @@ -185,7 +138,7 @@ public override void Open() connectionState = ConnectionState.Open; - statistics?.openTimestamp = TimerUtils.TimerCurrent(); + InitProfiling(); OnStateChange(FromClosedToOpenEventArgs); } @@ -309,7 +262,7 @@ protected override void Dispose(bool disposing) Close(); } - statistics?.Dispose(); + profilingInfo?.Dispose(); } base.Dispose(disposing); @@ -338,9 +291,14 @@ public DuckDBConnection Duplicate() parsedConnection = ParsedConnection, inMemoryDuplication = true, connectionReference = connectionReference, - ProfilingEnabled = ProfilingEnabled }; + // Enable profiling on the duplicated connection so it collects the same metrics/statistics + if (ProfilingEnabled) + { + duplicatedConnection.EnableProfiling(true, profilingOptions); + } + return duplicatedConnection; } @@ -359,14 +317,184 @@ public DuckDBQueryProgress GetQueryProgress() return NativeMethods.Startup.DuckDBQueryProgress(NativeConnection); } + #region profiling + + public bool ProfilingEnabled => profilingInfo != null && collectstats; + + /// + /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. + /// If profiling is not enabled, this method will return an empty summary. + /// + /// A containing the collected statistics. + public ProfilingSummary RetrieveStatistics() + { + if (profilingInfo != null) + { + UpdateStatistics(); + return profilingInfo.GetProfilingSummary(); + } + else + { + return new ConnectionStatistics(NativeConnection, collectstats).GetProfilingSummary(); + } + } + + /// + /// Enables or disables profiling for the current connection, optionally applying the specified profiling options. + /// + /// Profiling collects statistics about the connection's activity. If profiling is enabled while the + /// connection is open, statistics collection begins immediately. Disabling profiling stops statistics collection and + /// finalizes the current profile. + /// A value indicating whether profiling should be enabled. Set to to enable profiling; + /// otherwise, to disable profiling. + /// The profiling options to apply when enabling profiling. If , default profiling options are + /// used. This parameter is ignored when disabling profiling. + public void EnableProfiling(bool enabled, ProfilingOptions? options = null) + { + if (collectstats == enabled) + { + return; + } + + if (enabled) + { + this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null + + if (State == ConnectionState.Open) + { + InitProfiling(); + } + } + else + { + // stop + profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); + DisableProfiling(); + } + + collectstats = enabled; + } + + /// + /// Initializes the profiling statistics for the current connection. + /// + /// This method prepares internal data structures to collect and store profiling information + /// related to the connection's activity. It should be called before attempting to access profiling or statistics + /// data for the connection. + private void InitProfiling() + { + EnsureConnectionOpen(); + + profilingInfo = new ConnectionStatistics(NativeConnection, collectstats); + LoadStatisticsProfile(); + } + + /// + /// Modifies the current profiling options used for collecting performance statistics. + /// + /// The new profiling options to apply. Cannot be null. + /// Thrown if profiling is not enabled when attempting to edit profiling options. + public void EditProfilingOptions(ProfilingOptions options) + { + if (!ProfilingEnabled) + { + throw new InvalidOperationException("Profiling must be enabled to edit profiling options."); + } + this.profilingOptions = options; + + profilingInfo?.Reset(); + LoadStatisticsProfile(); + } + + /// + /// Disables query profiling for the current database connection if profiling is enabled. + /// + /// This method has no effect if profiling is already disabled. Disabling profiling may improve + /// performance for subsequent queries. + /// Thrown if an error occurs while disabling profiling on the database connection. + private void DisableProfiling() + { + if (ProfilingEnabled && State == ConnectionState.Open) + { + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, "PRAGMA disable_profiling; PRAGMA disable_profile;", out _); + if (!state.IsSuccess()) + { + throw new DuckDBException("Error disabling profiling."); + } + } + } + + /// + /// Configures and enables query profiling for the current database connection based on the specified profiling + /// options. + /// + /// Profiling is only enabled for original in-memory connections; duplicated in-memory + /// connections share the same profiling state and are not reconfigured. This method applies settings such as + /// profiling format, coverage, mode, output path, and enabled metrics according to the current profiling + /// options. + /// Thrown if an error occurs while setting profiling configuration options. + private void LoadStatisticsProfile() + { + profilingInfo?.openTimestamp = TimerUtils.TimerCurrent(); + + // Do not enable profiling again for duplicated in-memory connections; the original connection + // already has profiling enabled and the duplicated connection shares the same underlying state. + if (ProfilingEnabled && !inMemoryDuplication) + { + + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET enable_profiling = '{profilingOptions.Format.ToDuckDBProfilingFormatString()}'", out _); + if (!state.IsSuccess()) + { + throw new DuckDBException($"Error setting '{"enable_profiling"}' to '{profilingOptions.Format}'"); + } + + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_coverage = '{profilingOptions.Coverage}'", out _); + if (!state.IsSuccess()) + { + throw new DuckDBException($"Error setting '{"profiling_coverage"}' to '{profilingOptions.Coverage}'"); + } + + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_mode = '{profilingOptions.Mode}'", out _); + if (!state.IsSuccess()) + { + throw new DuckDBException($"Error setting '{"profiling_mode"}' to '{profilingOptions.Mode}'"); + } + + if (!string.IsNullOrEmpty(profilingOptions.OutputPath)) + { + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_output = '{profilingOptions.OutputPath}'", out _); + if (!state.IsSuccess()) + { + throw new DuckDBException($"Error setting '{"profiling_output"}' to '{profilingOptions.OutputPath}'"); + } + } + + var enabledMetrics = profilingOptions.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET custom_profiling_settings = '{enabledMetrics.ToDuckDBMetricString()}'", out _); + if (!state.IsSuccess()) + { + DuckDBException? innerEx = null; + + if (enabledMetrics.Contains(DuckDBMetricType.RowsReturned)) + { + innerEx = new DuckDBException($"The '{DuckDBMetricType.RowsReturned}' metric is not supported yet.", DuckDBErrorType.InvalidInput); + } + + throw new DuckDBException($"Error setting '{"custom_profiling_settings"}' to '{enabledMetrics.ToDuckDBMetricString()}'", innerEx); + + } + } + } + private void UpdateStatistics() { if (ConnectionState.Open == State) { // update timestamp - statistics?.closeTimestamp = TimerUtils.TimerCurrent(); + profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); } // delegate the rest of the work to the SqlStatistics class - statistics?.UpdateStatistics(); + profilingInfo?.UpdateStatistics(); } + #endregion } diff --git a/DuckDB.NET.Data/Extensions/MetricsExtensions.cs b/DuckDB.NET.Data/Extensions/MetricsExtensions.cs new file mode 100644 index 00000000..4486a309 --- /dev/null +++ b/DuckDB.NET.Data/Extensions/MetricsExtensions.cs @@ -0,0 +1,92 @@ +using DuckDB.NET.Data.Profiling; +using System.Collections.Frozen; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; + +namespace DuckDB.NET.Data.Extensions +{ + internal static class MetricsExtensions + { + // Cache the snake_case conversions of DuckDBProfilingFormat enum values for efficient lookup. + private static readonly FrozenDictionary ProfilingFormatCache = + Enum.GetValues() + .ToFrozenDictionary(f => f, f => JsonNamingPolicy.SnakeCaseLower.ConvertName(f.ToString())); + + // Cache the snake_case conversions of DuckDBMetricType enum values for efficient lookup. + private static readonly FrozenDictionary MetricTypeCache = + Enum.GetValues() + .ToFrozenDictionary(m => m, m => JsonNamingPolicy.SnakeCaseUpper.ConvertName(m.ToString())); + + // Create a reverse lookup cache for string to DuckDBMetricType conversions. + private static readonly FrozenDictionary StringToMetricTypeCache = + MetricTypeCache.ToFrozenDictionary(kv => kv.Value, kv => kv.Key); + + // create a reverse lookup cache for string to DuckDBProfilingFormat conversions. + private static readonly FrozenDictionary StringToProfilingFormatCache = + ProfilingFormatCache.ToFrozenDictionary(kv => kv.Value, kv => kv.Key); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string ToDuckDBMetricString(this DuckDBMetricTypeCollection metrics) + { + return $"{{{string.Join(",", metrics.Select(m => $"\"{m.ToDuckDBMetricTypeString()}\": \"TRUE\""))}}}"; + } + + /// + /// Converts the specified DuckDB profiling format to its corresponding string representation. + /// + /// The DuckDB profiling format to convert. + /// A string that represents the specified DuckDB profiling format. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string ToDuckDBProfilingFormatString(this DuckDBProfilingFormat format) + { + return ProfilingFormatCache[format]; + } + + /// + /// Converts the specified DuckDB metric type to its corresponding string representation. + /// + /// The DuckDB metric type to convert. + /// A string that represents the specified DuckDB metric type. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string ToDuckDBMetricTypeString(this DuckDBMetricType metricType) + { + if (metricType == DuckDBMetricType.RowsReturned) + { + throw new DuckDBException($"The '{DuckDBMetricType.RowsReturned}' metric is not supported yet.", DuckDBErrorType.InvalidInput); + } + + return MetricTypeCache[metricType]; + } + + /// + /// Attempts to parse the specified string into a corresponding DuckDBMetricType value. + /// + /// The string representation of the DuckDB metric type to parse. Cannot be null. + /// When this method returns, contains the DuckDBMetricType value equivalent to the parsed string, if the conversion + /// succeeded; otherwise, the default value. + /// true if the string was successfully parsed into a DuckDBMetricType value; otherwise, false. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool TryParseDuckDBMetricType(string metricTypeString, out DuckDBMetricType metricType) + { + return StringToMetricTypeCache.TryGetValue(metricTypeString, out metricType); + } + + /// + /// Attempts to parse the specified string into a corresponding DuckDB profiling format value. + /// + /// Use this method to safely convert a string to a DuckDB profiling format without + /// throwing an exception if the conversion fails. + /// The string representation of the DuckDB profiling format to parse. This value is compared case-sensitively + /// against known profiling format names. + /// When this method returns, contains the parsed DuckDB profiling format value if the parse operation succeeds; + /// otherwise, contains the default value. + /// true if the string was successfully parsed into a DuckDB profiling format; otherwise, false. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool TryParseDuckDBProfilingFormat(string profilingFormatString, out DuckDBProfilingFormat profilingFormat) + { + return StringToProfilingFormatCache.TryGetValue(profilingFormatString, out profilingFormat); + } + + } +} diff --git a/DuckDB.NET.Data/Profiling/DuckDBMetricTypeCollection.cs b/DuckDB.NET.Data/Profiling/DuckDBMetricTypeCollection.cs new file mode 100644 index 00000000..6a71e8f6 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/DuckDBMetricTypeCollection.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace DuckDB.NET.Data.Profiling +{ + public class DuckDBMetricTypeCollection : List + { + public DuckDBMetricTypeCollection() { } + + public DuckDBMetricTypeCollection(IEnumerable collection) : base(collection) { } + } +} diff --git a/DuckDB.NET.Data/Profiling/MetricType.cs b/DuckDB.NET.Data/Profiling/MetricType.cs new file mode 100644 index 00000000..c9ffb9b2 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/MetricType.cs @@ -0,0 +1,28 @@ +using System.Linq; + +namespace DuckDB.NET.Data.Profiling +{ + public static class DuckDBMetrics + { + public static IEnumerable AllMetrics => Enum.GetValues().Cast(); + + public static IEnumerable DefaultMetrics => + [ + DuckDBMetricType.QueryName, + DuckDBMetricType.Latency, + DuckDBMetricType.CpuTime, + DuckDBMetricType.BlockedThreadTime, + DuckDBMetricType.ResultSetSize, + DuckDBMetricType.RowsReturned, + DuckDBMetricType.CumulativeRowsScanned, + DuckDBMetricType.TotalBytesRead, + DuckDBMetricType.TotalBytesWritten, + DuckDBMetricType.TotalMemoryAllocated, + DuckDBMetricType.SystemPeakBufferMemory, + DuckDBMetricType.SystemPeakTempDirSize, + DuckDBMetricType.WriteToWalLatency + ]; + + } + +} diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 6404b13a..2743146b 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -34,9 +34,17 @@ internal ProfilingInfoMetrics GetMetrics() var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); if (metricsValue.IsNull()) { - return default; + return []; } - return ProfilingInfoMetrics.FromDictionary(metricsValue.GetMapValue()); + + //var dic = metricsValue.GetMapValue(); + //Console.WriteLine($"{new string(' ', 1 * 2 + 2)}Metrics:"); + //foreach (var kvp in dic) + //{ + // Console.WriteLine($"{new string(' ', 1 * 3 + 2)} {kvp.Key}:{kvp.Value}"); + //} + + return ProfilingInfoMetrics.FromMetricsDictionary(metricsValue.GetMapValue()); } private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int indent) diff --git a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs new file mode 100644 index 00000000..8e934fb8 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs @@ -0,0 +1,12 @@ +namespace DuckDB.NET.Data.Profiling +{ + public readonly struct ProfilingOptions + { + public DuckDBProfilingFormat Format { get; init; } + public DuckDBProfilingMode Mode { get; init; } + public DuckDBProfilingCoverage Coverage { get; init; } + public string OutputPath { get; init; } + + public DuckDBMetricTypeCollection EnabledMetrics { get; init; } + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 767b4014..5f72dc4c 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -108,6 +108,8 @@ internal void Reset() connectionTime = 0; startExecutionTime = default; endExecutionTime = default; + + queryProfilers.Clear(); } // call on connection close/dispose diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs index 5f1a7877..5f83335d 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs @@ -1,110 +1,22 @@ -using System; -using System.Collections.Generic; -using System.Globalization; - namespace DuckDB.NET.Data.Profiling.Statistics; -/// -/// Strongly-typed, immutable view over profiling metric entries that can be created from -/// an and exported back to a dictionary. -/// -public readonly record struct ProfilingInfoMetrics( - double? ElapsedMilliseconds, - long? Rows, - long? ResultSetSize, - long? ScannedRows, - long? BytesRead, - long? BytesWritten, - long? TotalMemoryAllocated, - long? PeakMemoryAllocation, - int? Spills, - double? WalWriteLatencyMilliseconds, - string? Statement) -{ - public static ProfilingInfoMetrics FromDictionary(IDictionary dict) +public sealed class ProfilingInfoMetrics : Dictionary +{ + internal static ProfilingInfoMetrics FromMetricsDictionary(IDictionary dict) { - if (dict is null) throw new ArgumentNullException(nameof(dict)); + ArgumentNullException.ThrowIfNull(dict); - static double? ToNullableDouble(object? v) - { - if (v is null) return null; - if (v is double d) return d; - if (v is float f) return Convert.ToDouble(f); - if (v is decimal dec) return Convert.ToDouble(dec); - if (v is long l) return Convert.ToDouble(l); - if (v is int i) return Convert.ToDouble(i); - if (v is string s && double.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; - if (v is TimeSpan ts) return ts.TotalMilliseconds; - if (v is IConvertible conv) return Convert.ToDouble(conv); - return null; - } - - static long? ToNullableLong(object? v) - { - if (v is null) return null; - if (v is long l) return l; - if (v is int i) return i; - if (v is short s) return s; - if (v is double d) return Convert.ToInt64(d); - if (v is decimal dec) return Convert.ToInt64(dec); - if (v is string str && long.TryParse(str, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; - if (v is IConvertible conv) return Convert.ToInt64(conv); - return null; - } + ProfilingInfoMetrics result = []; - static int? ToNullableInt(object? v) + foreach (var kvp in dict) { - if (v is null) return null; - if (v is int i) return i; - if (v is long l) return (int)l; - if (v is string s && int.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var r)) return r; - if (v is IConvertible conv) return Convert.ToInt32(conv); - return null; + if (MetricsExtensions.TryParseDuckDBMetricType(kvp.Key, out var metricType)) + { + result.Add(metricType, kvp.Value); + } } - dict.TryGetValue("Elapsed", out var elapsed); - dict.TryGetValue("Rows", out var rows); - dict.TryGetValue("ResultSetSize", out var resultSetSize); - dict.TryGetValue("ScannedRows", out var scannedRows); - dict.TryGetValue("BytesRead", out var bytesRead); - dict.TryGetValue("BytesWritten", out var bytesWritten); - dict.TryGetValue("TotalMemoryAllocated", out var totalMemoryAllocated); - dict.TryGetValue("PeakMemoryAllocation", out var peakMemoryAllocation); - dict.TryGetValue("Spills", out var spills); - dict.TryGetValue("WALWriteLatency", out var walWriteLatency); - dict.TryGetValue("Statement", out var statement); - - return new ProfilingInfoMetrics( - ElapsedMilliseconds: ToNullableDouble(elapsed), - Rows: ToNullableLong(rows), - ResultSetSize: ToNullableLong(resultSetSize), - ScannedRows: ToNullableLong(scannedRows), - BytesRead: ToNullableLong(bytesRead), - BytesWritten: ToNullableLong(bytesWritten), - TotalMemoryAllocated: ToNullableLong(totalMemoryAllocated), - PeakMemoryAllocation: ToNullableLong(peakMemoryAllocation), - Spills: ToNullableInt(spills), - WalWriteLatencyMilliseconds: ToNullableDouble(walWriteLatency), - Statement: statement?.ToString()); - } - - public IDictionary ToDictionary() - { - var dict = new Dictionary(11); - - if (ElapsedMilliseconds is not null) dict["Elapsed"] = ElapsedMilliseconds; - if (Rows is not null) dict["Rows"] = Rows; - if (ResultSetSize is not null) dict["ResultSetSize"] = ResultSetSize; - if (ScannedRows is not null) dict["ScannedRows"] = ScannedRows; - if (BytesRead is not null) dict["BytesRead"] = BytesRead; - if (BytesWritten is not null) dict["BytesWritten"] = BytesWritten; - if (TotalMemoryAllocated is not null) dict["TotalMemoryAllocated"] = TotalMemoryAllocated; - if (PeakMemoryAllocation is not null) dict["PeakMemoryAllocation"] = PeakMemoryAllocation; - if (Spills is not null) dict["Spills"] = Spills; - if (WalWriteLatencyMilliseconds is not null) dict["WALWriteLatency"] = WalWriteLatencyMilliseconds; - if (Statement is not null) dict["Statement"] = Statement; - - return dict; + return result; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 60ebe59b..974576d2 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -52,7 +52,7 @@ private IEnumerable GetStatementInfo() { foreach (var statementProfiler in statementProfilers) { - yield return statementProfiler?.Info ?? default; + yield return statementProfiler?.Info ?? []; } } diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 702d0c15..62c2809f 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -1,5 +1,7 @@ using Dapper; using DuckDB.NET.Data; +using DuckDB.NET.Data.Profiling; +using DuckDB.NET.Data.Profiling.Statistics; using DuckDB.NET.Native; using DuckDB.NET.Test.Helpers; using System; @@ -39,7 +41,7 @@ static void Main(string[] args) //ParametersBinding(); - Test(); + Profiling(); } private static void PrintVersion() @@ -244,35 +246,70 @@ private static void ParametersBinding() PrintQueryResults(reader); } - private static void Test() + private static void Profiling() { using var con = new DuckDBConnection("data source=:memory:"); - con.Open(); - - con.ProfilingEnabled = true; + var options = new ProfilingOptions + { + Format = DuckDBProfilingFormat.NoOutput, + Mode = DuckDBProfilingMode.Detailed, + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = + [ + DuckDBMetricType.QueryName, + DuckDBMetricType.Latency, + DuckDBMetricType.CpuTime, + DuckDBMetricType.ResultSetSize, + DuckDBMetricType.CumulativeRowsScanned, + DuckDBMetricType.TotalBytesRead, + DuckDBMetricType.TotalBytesWritten, + DuckDBMetricType.TotalMemoryAllocated, + DuckDBMetricType.SystemPeakBufferMemory, + DuckDBMetricType.SystemPeakTempDirSize, + DuckDBMetricType.WriteToWalLatency, + ] + }; + con.EnableProfiling(true, options); + con.Open(); using var cmd = con.CreateCommand(); - cmd.CommandText = @" - CALL enable_profiling( - format := 'json', - coverage := 'select', - mode := 'standard', - metrics := ['QUERY_NAME', 'LATENCY', 'OPERATOR_TIMING'] - );"; - cmd.ExecuteNonQuery(); - cmd.CommandText = "SELECT a:1;"; using var reader = cmd.ExecuteReader(); var metrics = con.RetrieveStatistics(); + + PrintMetrics(metrics); PrintQueryResults(reader); } + private static void PrintMetrics(ProfilingSummary profilingSummary) + { + profilingSummary.ToDictionary().ToList().ForEach(kv => Console.WriteLine($"{kv.Key}: {(kv.Value is ProfilingQuerySummary[] ? PrintQueryInfo((ProfilingQuerySummary[])kv.Value) : kv.Value)}")); + } + private static string PrintQueryInfo(ProfilingQuerySummary[] profilingSummary) + { + return string.Join("", profilingSummary.Select((q, i) => $"\nQuery {i}: " + (PrintQueryInfo(q)))); + } + private static string PrintQueryInfo(ProfilingQuerySummary profilingSummary) + { + return string.Join("", profilingSummary.ToDictionary().ToList().Select(kv => $"\n\t{kv.Key}: {(kv.Value is ProfilingInfoMetrics[] ? PrintStatementMetrics((ProfilingInfoMetrics[])kv.Value) : kv.Value)}")); + } + + private static string PrintStatementMetrics(ProfilingInfoMetrics[] profilingSummaries) + { + return string.Join("", profilingSummaries.Select((s, i) => $"\n\tStatement {i}: " + PrintStatementMetrics(s))); + } + + private static string PrintStatementMetrics(ProfilingInfoMetrics profilingSummary) + { + return string.Join("", profilingSummary.ToDictionary().ToList().Select(kv => $"\n\t\t{kv.Key}: {kv.Value}")); + } + private static void PrintQueryResults(DbDataReader queryResult) { for (var index = 0; index < queryResult.FieldCount; index++) diff --git a/DuckDB.NET.slnx b/DuckDB.NET.slnx index 45ecff20..5eb9fbe9 100644 --- a/DuckDB.NET.slnx +++ b/DuckDB.NET.slnx @@ -1,5 +1,4 @@ - diff --git a/Irion.DuckDB.NET.Benchmark/Program.cs b/Irion.DuckDB.NET.Benchmark/Program.cs index 311df375..8b3d2a10 100644 --- a/Irion.DuckDB.NET.Benchmark/Program.cs +++ b/Irion.DuckDB.NET.Benchmark/Program.cs @@ -1,11 +1,9 @@ using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using Irion.DuckDB.NET.Benchmark.Config; using System; using System.Diagnostics; using System.IO; -using System.Linq; namespace Irion.DuckDB.NET.Benchmark { @@ -25,7 +23,8 @@ static void Main(string[] args) } //clean 1.5.2 benchmark - var type = typeof(TpchBenchmarks_Baseline); + //var type = typeof(TpchBenchmarks_Baseline); + var type = typeof(TpchBenchmarks); config.ArtifactsPath = Path.Combine("BenchmarkDotNet.Artifacts", $"{type.Name}_{DateTime.Now:yyyyMMdd-HHmmss}"); diff --git a/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs new file mode 100644 index 00000000..6c93300f --- /dev/null +++ b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs @@ -0,0 +1,274 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using DuckDB.NET.Data.Profiling; +using DuckDB.NET.Native; +using DuckDB.NET.Test.Helpers; +using Irion.DuckDB.NET.Benchmark.Config; +using Perfolizer.Mathematics.OutlierDetection; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Irion.DuckDB.NET.Benchmark +{ + //[CPUUsageDiagnoser] + [Outliers(OutlierMode.DontRemove)] + [Config(typeof(CustomConfig))] + public class TpchBenchmarks + { + private DuckDBConnection? mainConnection; + private DuckDBCommand? mainCommand; + // Per-iteration connection/command used for extension loading & secret creation + private DuckDBConnection? iterationConnection; + private DuckDBCommand? iterationCommand; + private string? attachedDbPath; + + + //[ParamsAllValues] + [Params(true)] + public bool MetricsEnabled { get; set; } = false; + + //[ParamsAllValues] + [Params(DuckDBProfilingMode.Standard)] + public DuckDBProfilingMode ProfilingMode { get; set; } + + [GlobalSetup] + public void Setup() + { + if (!NativeLibraryHelper.TryLoad()) + { + Console.Error.WriteLine("native assembly not found"); + return; + } + + // Initialize an in-memory DuckDB connection and attempt to install/load the tpch extension. + mainConnection = new DuckDBConnection("DataSource=:memory:"); + mainConnection.Open(); + mainCommand = mainConnection.CreateCommand(); + try + { + mainCommand.CommandText = "INSTALL tpch;"; + mainCommand.ExecuteNonQuery(); + mainCommand.CommandText = "LOAD tpch;"; + mainCommand.ExecuteNonQuery(); + // Generate TPCH sf=1 dataset into the current database + mainCommand.CommandText = "CALL dbgen(sf=1);"; + mainCommand.ExecuteNonQuery(); + } + catch (Exception) + { + // If the extension isn't available, fall back to a simple generated dataset that approximates scale. + mainCommand.CommandText = "CREATE TABLE IF NOT EXISTS lineitem (l_orderkey INTEGER, l_partkey INTEGER, l_quantity INTEGER);"; + mainCommand.ExecuteNonQuery(); + // Insert a moderate number of rows to simulate work (1k) + mainCommand.CommandText = "BEGIN TRANSACTION;"; + mainCommand.ExecuteNonQuery(); + var rnd = new Random(42); + for (int i = 0; i < 1000; i++) + { + mainCommand.CommandText = $"INSERT INTO lineitem VALUES ({rnd.Next(1, 100000)}, {rnd.Next(1, 100000)}, {rnd.Next(1, 100)});"; + mainCommand.ExecuteNonQuery(); + } + + mainCommand.CommandText = "COMMIT;"; + mainCommand.ExecuteNonQuery(); + } + + EnableProfiling(mainConnection); + } + + private void EnableProfiling(DuckDBConnection connection) + { + if (MetricsEnabled) + { + var options = new ProfilingOptions + { + Format = DuckDBProfilingFormat.NoOutput, + Mode = ProfilingMode, //DuckDBProfilingMode.Detailed, + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = [] + }; + connection.EnableProfiling(true, options); + } + } + + private void ReadMetrics(DuckDBConnection connection) + { + if (MetricsEnabled) + { + var metrics = connection.RetrieveStatistics(); + } + } + + [GlobalCleanup] + public void Cleanup() + { + try + { + if (!string.IsNullOrEmpty(attachedDbPath) && File.Exists(attachedDbPath)) + { + File.Delete(attachedDbPath); + } + } + catch + { + } + + mainCommand?.Dispose(); + mainConnection?.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + // Each iteration needs a fresh connection where the extension is installed and the secret created. + iterationConnection = new DuckDBConnection("DataSource=:memory:"); + try + { + iterationConnection.Open(); + iterationCommand = iterationConnection.CreateCommand(); + + // Install and load httpfs, then create a secret using the CREATE SECRET syntax. + iterationCommand.CommandText = "INSTALL httpfs;"; + iterationCommand.ExecuteNonQuery(); + iterationCommand.CommandText = "LOAD httpfs;"; + iterationCommand.ExecuteNonQuery(); + + EnableProfiling(iterationConnection); + } + catch + { + // Extension or CREATE SECRET not available on this environment; clean up the iteration resources + // and leave iterationCommand null so the benchmark method can detect the unsupported case. + iterationCommand?.Dispose(); + iterationConnection?.Dispose(); + iterationCommand = null; + iterationConnection = null; + } + } + + [IterationCleanup] + public void IterationCleanup() + { + iterationCommand?.Dispose(); + iterationConnection?.Dispose(); + iterationCommand = null; + iterationConnection = null; + } + + public static IEnumerable Queries => Enumerable.Range(1, 22); + + [Benchmark] + [ArgumentsSource(nameof(Queries))] + public void RunTpchQuery(int query) + { + mainCommand!.CommandText = $"PRAGMA tpch({query});"; + mainCommand.ExecuteNonQuery(); + + ReadMetrics(mainConnection); + } + + [Benchmark] + public void SimpleAggregation() + { + mainCommand!.CommandText = "SELECT l_partkey, SUM(l_quantity) as s FROM lineitem GROUP BY l_partkey LIMIT 100;"; + using var reader = mainCommand.ExecuteReader(); + while (reader.Read()) + { /* iterate */ + } + + ReadMetrics(mainConnection); + + } + + [Benchmark] + public void AttachDatabaseAndQuery() + { + // create a temporary on-disk database and attach it + attachedDbPath = Path.Combine(Path.GetTempPath(), $"duckdb_bench_{Guid.NewGuid():N}.db"); + mainCommand!.CommandText = $"ATTACH DATABASE '{attachedDbPath}' AS attached_db;"; + mainCommand.ExecuteNonQuery(); + + ReadMetrics(mainConnection); + + // run a simple query against the attached database (it will be empty) + mainCommand.CommandText = "SELECT COUNT(*) FROM attached_db.sqlite_master WHERE 1=0;"; + try + { + mainCommand.ExecuteScalar(); + + ReadMetrics(mainConnection); + + } + catch + { /* ignore — attached DB may not have that table */ + } + + mainCommand.CommandText = "DETACH DATABASE attached_db;"; + mainCommand.ExecuteNonQuery(); + + ReadMetrics(mainConnection); + + // cleanup file + try + { + File.Delete(attachedDbPath); + } + catch + { + } + + attachedDbPath = null; + } + + [Benchmark] + public void LoadExtension() + { + // Each iteration needs a fresh connection where the extension is installed and the secret created. + using var connection = new DuckDBConnection("DataSource=:memory:"); + + connection.Open(); + using var command = connection.CreateCommand(); + + // Install and load httpfs, then create a secret using the CREATE SECRET syntax. + command.CommandText = "INSTALL httpfs;"; + command.ExecuteNonQuery(); + + ReadMetrics(connection); + + command.CommandText = "LOAD httpfs;"; + command.ExecuteNonQuery(); + + ReadMetrics(connection); + + } + + [Benchmark] + public void CreateSecret() + { + // The extension must have been installed and the secret created during IterationSetup. The + // benchmark itself only verifies the secret/extension is usable, keeping the measured work focused. + if (iterationCommand is null) + { + // Extension or CREATE SECRET not supported in this iteration; skip measurable work. + return; + } + + // Use CREATE SECRET syntax to register fake credentials for the extension. + // If the running DuckDB build does not support CREATE SECRET, an exception will be thrown and + // we will treat this iteration as not supporting the extension. + iterationCommand.CommandText = "CREATE SECRET s3_credentials (type s3, KEY_ID 'AKIAFAKE', SECRET 'fakeSecret');"; + iterationCommand.ExecuteNonQuery(); + + ReadMetrics(iterationConnection); + + iterationCommand.CommandText = "SELECT 1;"; + _ = iterationCommand.ExecuteScalar(); + + ReadMetrics(iterationConnection); + + } + } +} \ No newline at end of file From e8a3b54658905a349d77d09c6a3beabf0e8b7039 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 13 May 2026 12:55:13 +0200 Subject: [PATCH 10/43] fix init profiling logic --- DuckDB.NET.Data/DuckDBConnection.cs | 12 ++++++++---- DuckDB.NET.Samples/Program.cs | 25 +++++++++++++++++++++---- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 7a8e2b59..5082b348 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -319,7 +319,7 @@ public DuckDBQueryProgress GetQueryProgress() #region profiling - public bool ProfilingEnabled => profilingInfo != null && collectstats; + public bool ProfilingEnabled => collectstats; /// /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. @@ -356,6 +356,8 @@ public void EnableProfiling(bool enabled, ProfilingOptions? options = null) return; } + collectstats = enabled; + if (enabled) { this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null @@ -372,7 +374,6 @@ public void EnableProfiling(bool enabled, ProfilingOptions? options = null) DisableProfiling(); } - collectstats = enabled; } /// @@ -385,8 +386,11 @@ private void InitProfiling() { EnsureConnectionOpen(); - profilingInfo = new ConnectionStatistics(NativeConnection, collectstats); - LoadStatisticsProfile(); + if (ProfilingEnabled) + { + profilingInfo = new ConnectionStatistics(NativeConnection, collectstats); + LoadStatisticsProfile(); + } } /// diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 62c2809f..dda5d6b9 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -270,23 +270,40 @@ private static void Profiling() DuckDBMetricType.WriteToWalLatency, ] }; - con.EnableProfiling(true, options); con.Open(); + LoadTpch(con); + + con.EnableProfiling(true, options); + using var cmd = con.CreateCommand(); + //cmd.CommandText = "SELECT a:1;"; + //using var reader = cmd.ExecuteReader(); - cmd.CommandText = "SELECT a:1;"; - using var reader = cmd.ExecuteReader(); + cmd.CommandText = $"PRAGMA tpch(1);"; + cmd.ExecuteNonQuery(); var metrics = con.RetrieveStatistics(); PrintMetrics(metrics); - PrintQueryResults(reader); + //PrintQueryResults(reader); } + private static void LoadTpch(DuckDBConnection connection, int scaleFactor = 1) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = "INSTALL tpch;"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "LOAD tpch;"; + cmd.ExecuteNonQuery(); + // Generate TPCH sf=1 dataset into the current database + cmd.CommandText = $"CALL dbgen(sf={scaleFactor});"; + cmd.ExecuteNonQuery(); + } + private static void PrintMetrics(ProfilingSummary profilingSummary) { profilingSummary.ToDictionary().ToList().ForEach(kv => Console.WriteLine($"{kv.Key}: {(kv.Value is ProfilingQuerySummary[] ? PrintQueryInfo((ProfilingQuerySummary[])kv.Value) : kv.Value)}")); From f58b6e6b00c676ee8d7c2cd36d2837388bc3ad5d Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Tue, 19 May 2026 13:01:06 +0200 Subject: [PATCH 11/43] Refactor and enhance profiling functionality Refactored `EnableProfiling` in `DuckDBConnection` to simplify its interface by removing the `enabled` parameter. Added `DisableProfiling` and `ResetStatistics` methods for more granular control over profiling. Enhanced `ProfilingOptions` to support finer control over coverage and metrics. Updated `ConnectionStatistics` and `ProfilingInfo` to enable dynamic query execution tracing and improve thread safety. Modified `Program.cs` and `TpchBenchmarks.cs` to demonstrate the new profiling API. Added comprehensive unit tests in `ProfilingTests.cs` to validate profiling functionality, including edge cases and unsupported metrics. Improved exception handling and error messages for invalid profiling configurations. --- DuckDB.NET.Data/DuckDBConnection.cs | 69 ++-- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 7 - .../Statistics/ConnectionStatistics.cs | 29 +- DuckDB.NET.Samples/Program.cs | 74 +++- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 329 ++++++++++++++++++ Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs | 2 +- 6 files changed, 461 insertions(+), 49 deletions(-) create mode 100644 DuckDB.NET.Test/Profiling/ProfilingTests.cs diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 5082b348..90191d39 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -296,7 +296,7 @@ public DuckDBConnection Duplicate() // Enable profiling on the duplicated connection so it collects the same metrics/statistics if (ProfilingEnabled) { - duplicatedConnection.EnableProfiling(true, profilingOptions); + duplicatedConnection.EnableProfiling(profilingOptions); } return duplicatedConnection; @@ -340,40 +340,52 @@ public ProfilingSummary RetrieveStatistics() } /// - /// Enables or disables profiling for the current connection, optionally applying the specified profiling options. + /// Enables profiling for the current connection, optionally using the specified profiling options. /// - /// Profiling collects statistics about the connection's activity. If profiling is enabled while the - /// connection is open, statistics collection begins immediately. Disabling profiling stops statistics collection and - /// finalizes the current profile. - /// A value indicating whether profiling should be enabled. Set to to enable profiling; - /// otherwise, to disable profiling. - /// The profiling options to apply when enabling profiling. If , default profiling options are - /// used. This parameter is ignored when disabling profiling. - public void EnableProfiling(bool enabled, ProfilingOptions? options = null) + /// If profiling is enabled while the connection is already open, profiling is initialized immediately. + /// Otherwise, profiling will be initialized when the connection is opened. + /// An optional set of profiling options to configure profiling behavior. If null, default options are used. + public void EnableProfiling(ProfilingOptions? options = null) { - if (collectstats == enabled) + collectstats = true; + + this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null + + if (State == ConnectionState.Open) { - return; + InitProfiling(); } + } - collectstats = enabled; + /// + /// Disables profiling for the current session, with an option to reset collected statistics. + /// + /// If profiling is already disabled, calling this method has no effect. Resetting statistics + /// clears all previously collected profiling data, which cannot be recovered. + /// true to reset all collected profiling statistics after disabling profiling; otherwise, false. + public void DisableProfiling(bool resetStatistics = false) + { + // stop + profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); + DisableProfiling(); - if (enabled) + if (resetStatistics) { - this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null - - if (State == ConnectionState.Open) - { - InitProfiling(); - } + ResetStatistics(); } - else + } + + /// + /// Resets all collected profiling statistics to their initial state. + /// + /// This method has no effect if profiling is not enabled. Use this method to clear accumulated + /// profiling data before starting a new measurement period. + public void ResetStatistics() + { + if (ProfilingEnabled) { - // stop - profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); - DisableProfiling(); + profilingInfo?.Reset(); } - } /// @@ -426,6 +438,9 @@ private void DisableProfiling() throw new DuckDBException("Error disabling profiling."); } } + + profilingInfo?.DisableQueryExecutionTracing(); + collectstats = false; } /// @@ -483,9 +498,9 @@ private void LoadStatisticsProfile() { innerEx = new DuckDBException($"The '{DuckDBMetricType.RowsReturned}' metric is not supported yet.", DuckDBErrorType.InvalidInput); } - + throw new DuckDBException($"Error setting '{"custom_profiling_settings"}' to '{enabledMetrics.ToDuckDBMetricString()}'", innerEx); - + } } } diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 2743146b..952980fb 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -37,13 +37,6 @@ internal ProfilingInfoMetrics GetMetrics() return []; } - //var dic = metricsValue.GetMapValue(); - //Console.WriteLine($"{new string(' ', 1 * 2 + 2)}Metrics:"); - //foreach (var kvp in dic) - //{ - // Console.WriteLine($"{new string(' ', 1 * 3 + 2)} {kvp.Key}:{kvp.Value}"); - //} - return ProfilingInfoMetrics.FromMetricsDictionary(metricsValue.GetMapValue()); } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 5f72dc4c..4d2c95ac 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -14,7 +14,7 @@ private static readonly ConcurrentDictionary queryProfilers = []; private bool isDisposed = false; @@ -34,7 +34,30 @@ internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, boo { this.enableQueryExecutionTracing = enableQueryExecutionTracing; this.duckDBNativeConnection = duckDBNativeConnection; - ByNativeConnection.TryAdd(duckDBNativeConnection, this); + + // Add or update the mapping for the native connection to this ConnectionStatistics instance + ByNativeConnection.AddOrUpdate(duckDBNativeConnection, this, (_, _) => this); + } + + /// + /// Enables tracing of query execution for diagnostic or debugging purposes. + /// + /// After calling this method, additional diagnostic information about query execution may be collected + /// or logged. This can assist in troubleshooting or performance analysis. Tracing remains enabled until explicitly + /// disabled + internal void EnableQueryExecutionTracing() + { + this.enableQueryExecutionTracing = true; + } + + /// + /// Disables tracing of query execution for the current instance. + /// + /// After calling this method, query execution tracing will no longer be performed until explicitly + /// re-enabled. This may affect the ability to diagnose or audit query behavior. + internal void DisableQueryExecutionTracing() + { + this.enableQueryExecutionTracing = false; } /// @@ -45,7 +68,7 @@ internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, boo /// langword="null"/>. /// if statistics are found for the specified connection; otherwise, . - public static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) + internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) => ByNativeConnection.TryGetValue(nativeConn, out stats); /// diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index dda5d6b9..5b4c829b 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -254,7 +254,7 @@ private static void Profiling() { Format = DuckDBProfilingFormat.NoOutput, Mode = DuckDBProfilingMode.Detailed, - Coverage = DuckDBProfilingCoverage.All, + Coverage = DuckDBProfilingCoverage.Select, EnabledMetrics = [ DuckDBMetricType.QueryName, @@ -273,20 +273,72 @@ private static void Profiling() con.Open(); - LoadTpch(con); + var before = con.RetrieveStatistics(); + var beforeCount = before.QuerySummaryList.Length; - con.EnableProfiling(true, options); + var id = Guid.NewGuid().ToString("N"); - using var cmd = con.CreateCommand(); - //cmd.CommandText = "SELECT a:1;"; - //using var reader = cmd.ExecuteReader(); + using var dBCommand = con.CreateCommand(); - cmd.CommandText = $"PRAGMA tpch(1);"; - cmd.ExecuteNonQuery(); + //LoadTpch(con); + + for (var selectPosition = 0; selectPosition < 3; selectPosition++) + { + + con.EnableProfiling(options); + + try + { + + var val = 200 + selectPosition; + string a = $"t_{id}_a"; + string b = $"t_{id}_b"; + + string cmd; + if (selectPosition == 0) + { + cmd = $"SELECT {val}; CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER);"; + } + else if (selectPosition == 1) + { + cmd = $"CREATE TABLE {a}(i INTEGER); SELECT {val}; CREATE TABLE {b}(i INTEGER);"; + } + else + { + cmd = $"CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER); SELECT {val};"; + } + + dBCommand.CommandText = cmd; + using var reader = dBCommand.ExecuteReader(); + do { } while (reader.NextResult()); + + var summary = con.RetrieveStatistics(); + var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); + Console.WriteLine($"newSummaries length: {newSummaries.Length}"); + } + finally + { + con.DisableProfiling(); + } + } + + //using var cmd = con.CreateCommand(); + //cmd.CommandText = "SELECT a:1;"; + //using var reader = cmd.ExecuteReader(); + + //cmd.CommandText = $"PRAGMA tpch(1);"; + //cmd.ExecuteNonQuery(); + + //var metrics = con.RetrieveStatistics(); + //Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); + + //cmd.CommandText = $"PRAGMA tpch(2); CREATE TABLE test (id int);"; + //cmd.ExecuteNonQuery(); + + //metrics = con.RetrieveStatistics(); + //Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); - var metrics = con.RetrieveStatistics(); - - PrintMetrics(metrics); + //PrintMetrics(metrics); //PrintQueryResults(reader); diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs new file mode 100644 index 00000000..1c3859a5 --- /dev/null +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -0,0 +1,329 @@ +using DuckDB.NET.Data.Profiling; + +namespace DuckDB.NET.Test.Profiling; + +public class ProfilingTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + + [Fact] + public void MetricsDictionaryContainsEnabledMetrics() + { + // enable profiling with specific metrics enabled + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName, DuckDBMetricType.CpuTime }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + var before = Connection.RetrieveStatistics(); + var beforeCount = before.QuerySummaryList.Length; + + // execute a small batch that contains different statement types + Command.CommandText = "CREATE TABLE profiling_test(a INTEGER); INSERT INTO profiling_test VALUES (1); SELECT a FROM profiling_test;"; + using var reader = Command.ExecuteReader(); + + // advance through all result sets to ensure statements are executed + do { } while (reader.NextResult()); + + var summary = Connection.RetrieveStatistics(); + // If no new summaries were produced, profiling might not be available in this environment. + var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); + newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); + var last = newSummaries.Last(); + + // check that enabled metrics appear in at least one statement's metrics + var containsQueryName = last.Infos.Any(i => i != null && i.ContainsKey(DuckDBMetricType.QueryName)); + var containsCpuTime = last.Infos.Any(i => i != null && i.ContainsKey(DuckDBMetricType.CpuTime)); + + containsQueryName.Should().BeTrue(); + containsCpuTime.Should().BeTrue(); + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void MultipleEnabledMetricsArePresentInMetricsDictionary() + { + var metricsToEnable = new[] + { + DuckDBMetricType.QueryName, + DuckDBMetricType.CpuTime, + DuckDBMetricType.Latency, + DuckDBMetricType.TotalBytesRead + }; + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(metricsToEnable), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + var before = Connection.RetrieveStatistics(); + var beforeCount = before.QuerySummaryList.Length; + + Command.CommandText = "CREATE TABLE profiling_metrics(a INTEGER); INSERT INTO profiling_metrics VALUES (1); SELECT a FROM profiling_metrics;"; + using var reader = Command.ExecuteReader(); + do { } while (reader.NextResult()); + + var summary = Connection.RetrieveStatistics(); + var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); + newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); + var last = newSummaries.Last(); + last.Infos.Should().NotBeNull(); + last.Infos.Length.Should().BeGreaterThan(0, "Expected some profiling info when metrics are enabled"); + + // ensure at least one statement collected all requested metrics + bool found = last.Infos.Any(info => info != null && metricsToEnable.All(m => info.ContainsKey(m))); + + found.Should().BeTrue("At least one statement should include all the enabled metrics (QueryName, CpuTime, Latency, TotalBytesRead)"); + } + finally + { + Connection.DisableProfiling(); + } + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(2)] + public void SelectCoverageRespectsSelectPosition(int selectPosition) + { + var id = Guid.NewGuid().ToString("N"); + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.Select, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + var before = Connection.RetrieveStatistics(); + var beforeCount = before.QuerySummaryList.Length; + + var val = 200 + selectPosition; + string a = $"t_{id}_a"; + string b = $"t_{id}_b"; + + string cmd; + if (selectPosition == 0) + { + cmd = $"SELECT {val}; CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER);"; + } + else if (selectPosition == 1) + { + cmd = $"CREATE TABLE {a}(i INTEGER); SELECT {val}; CREATE TABLE {b}(i INTEGER);"; + } + else + { + cmd = $"CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER); SELECT {val};"; + } + + Command.CommandText = cmd; + using var reader = Command.ExecuteReader(); + do { } while (reader.NextResult()); + + var summary = Connection.RetrieveStatistics(); + var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); + newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); + var last = newSummaries.Last(); + last.StatementCount.Should().Be(3); + + last.Infos.Should().NotBeNull(); + + if (last.Infos.Length >= last.StatementCount) + { + for (int i = 0; i < last.StatementCount; i++) + { + var info = i < last.Infos.Length ? last.Infos[i] : null; + if (i == selectPosition) + { + info.Should().NotBeNull(); + info.Count.Should().BeGreaterThan(0); + } + else + { + info.Should().NotBeNull(); + info.Count.Should().Be(0); + } + } + } + else + { + // If Infos length doesn't map 1:1 to statements, at least ensure exactly one statement collected metrics + var nonEmpty = last.Infos.Count(i => i != null && i.Count > 0); + nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); + } + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void DisableStopsCollectingNewQueries() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + // run first batch + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var before = Connection.RetrieveStatistics(); + var beforeCount = before.QueryCount; + + // disable and run another batch + Connection.DisableProfiling(); + Command.CommandText = "SELECT 2;"; + using (var r = Command.ExecuteReader()) { } + + var after = Connection.RetrieveStatistics(); + // after disabling, QueryCount should not have increased + after.QueryCount.Should().Be(beforeCount); + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void ResetCleanCollectedStatistics() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + // run first batch + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + // disable and run another batch + Connection.ResetStatistics(); + var after = Connection.RetrieveStatistics(); + // after resetting, QueryCount should be zero + after.QueryCount.Should().Be(0); + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void EnablingWithRowsReturnedMetricThrows() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.RowsReturned }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Action act = () => Connection.EnableProfiling(options); + act.Should().Throw().Where(e => e.Message.Contains("RowsReturned") || (e.InnerException != null && e.InnerException.Message.Contains("RowsReturned"))); + } + + [Fact] + public void CoverageSelectOnlyProfilesOnlySelectStatements() + { + // enable profiling for SELECT coverage only + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.Select, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + var before = Connection.RetrieveStatistics(); + var beforeCount = before.QuerySummaryList.Length; + + // execute a batch with CREATE, INSERT and SELECT (three statements) + Command.CommandText = "CREATE TABLE profiling_cov(a INTEGER); INSERT INTO profiling_cov VALUES (1); SELECT a FROM profiling_cov;"; + using var reader = Command.ExecuteReader(); + + // advance through all result sets + do { } while (reader.NextResult()); + + var summary = Connection.RetrieveStatistics(); + var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); + newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); + var last = newSummaries.Last(); + + last.StatementCount.Should().BeGreaterOrEqualTo(3); + + // For SELECT coverage, expect only the SELECT (last) statement to have metrics collected + if (last.Infos.Length >= last.StatementCount) + { + for (int i = 0; i < last.StatementCount; i++) + { + var info = i < last.Infos.Length ? last.Infos[i] : null; + if (i == last.StatementCount - 1) + { + info.Should().NotBeNull(); + info.Count.Should().BeGreaterThan(0); + } + else + { + info.Should().NotBeNull(); + info.Count.Should().Be(0); + } + } + } + else + { + var nonEmpty = last.Infos.Count(i => i != null && i.Count > 0); + nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); + } + } + finally + { + Connection.DisableProfiling(); + } + } +} diff --git a/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs index 6c93300f..f4615e6e 100644 --- a/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs +++ b/Irion.DuckDB.NET.Benchmark/TpchBenchmarks.cs @@ -90,7 +90,7 @@ private void EnableProfiling(DuckDBConnection connection) Coverage = DuckDBProfilingCoverage.All, EnabledMetrics = [] }; - connection.EnableProfiling(true, options); + connection.EnableProfiling(options); } } From 1bef8e564e432fee689caaeb58c66ccd36a20f41 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Tue, 19 May 2026 15:53:59 +0200 Subject: [PATCH 12/43] Refactor and enhance profiling support in DuckDB Refactored profiling state management by introducing `IsProfilingEnabled` and `ProfilingOptions` at the `FileReference` level to ensure consistent profiling behavior across file-backed connections. Replaced `collectstats` with `isProfilingEnabled` for clarity and updated initialization logic to synchronize profiling state between connections. Improved profiling behavior for duplicated connections: - In-memory duplicates now maintain independent profiling statistics. - File-backed duplicates share profiling state and metrics. Refactored `LoadStatisticsProfile` for better error handling and simplified profiling configuration logic. Added unit tests to validate profiling behavior across various scenarios, ensuring correctness and reliability. Fixed a bug in `QueryProfiler` related to uninitialized dictionaries and removed redundant profiling code in duplicated connections. --- DuckDB.NET.Data/Connection/FileReference.cs | 14 +- DuckDB.NET.Data/DuckDBConnection.cs | 67 +++--- .../Profiling/Statistics/QueryProfiler.cs | 1 - DuckDB.NET.Test/Profiling/ProfilingTests.cs | 202 ++++++++++++++++++ 4 files changed, 254 insertions(+), 30 deletions(-) diff --git a/DuckDB.NET.Data/Connection/FileReference.cs b/DuckDB.NET.Data/Connection/FileReference.cs index 24482986..2c5b4450 100644 --- a/DuckDB.NET.Data/Connection/FileReference.cs +++ b/DuckDB.NET.Data/Connection/FileReference.cs @@ -1,4 +1,5 @@ -using System.IO; +using DuckDB.NET.Data.Profiling; +using System.IO; namespace DuckDB.NET.Data.Connection; @@ -13,6 +14,17 @@ internal class FileReference(string filename) public long ConnectionCount { get; private set; } //don't need a long, but it is slightly faster on 64 bit systems + /// + /// Gets a value indicating whether profiling is enabled for the current context. + /// Is used to ensure that the DuckDBDatabase is created with the correct profiling mode when a connection is opened on the same database + /// + public bool IsProfilingEnabled { get; internal set; } + + /// + /// Gets the profiling options used to configure performance profiling behavior. + /// + public ProfilingOptions? ProfilingOptions { get; internal set; } + public long Decrement() { return --ConnectionCount; diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 90191d39..ecc09055 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -20,8 +20,8 @@ public partial class DuckDBConnection : DbConnection // Statistics support internal ConnectionStatistics? profilingInfo; - private bool collectstats; - private ProfilingOptions profilingOptions; + private bool isProfilingEnabled; + private ProfilingOptions? profilingOptions; #region Protected Properties @@ -138,6 +138,20 @@ public override void Open() connectionState = ConnectionState.Open; + // If profiling is not enabled, profilingOptions will be null and we need to initialize it with the + // connectionReference.FileReferenceCounter to ensure that if profiling is later enabled for this connection, + // it will use the correct options, same for isProfilingEnabled + if (profilingOptions.IsNull()) + { + isProfilingEnabled = connectionReference.FileReferenceCounter.IsProfilingEnabled; + profilingOptions = connectionReference.FileReferenceCounter.ProfilingOptions; + } + else + { + connectionReference.FileReferenceCounter.IsProfilingEnabled = isProfilingEnabled; + connectionReference.FileReferenceCounter.ProfilingOptions = profilingOptions.Value; + } + InitProfiling(); OnStateChange(FromClosedToOpenEventArgs); @@ -291,14 +305,9 @@ public DuckDBConnection Duplicate() parsedConnection = ParsedConnection, inMemoryDuplication = true, connectionReference = connectionReference, + isProfilingEnabled = isProfilingEnabled, }; - // Enable profiling on the duplicated connection so it collects the same metrics/statistics - if (ProfilingEnabled) - { - duplicatedConnection.EnableProfiling(profilingOptions); - } - return duplicatedConnection; } @@ -319,7 +328,7 @@ public DuckDBQueryProgress GetQueryProgress() #region profiling - public bool ProfilingEnabled => collectstats; + public bool ProfilingEnabled => isProfilingEnabled; /// /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. @@ -335,7 +344,7 @@ public ProfilingSummary RetrieveStatistics() } else { - return new ConnectionStatistics(NativeConnection, collectstats).GetProfilingSummary(); + return new ConnectionStatistics(NativeConnection, isProfilingEnabled).GetProfilingSummary(); } } @@ -347,10 +356,12 @@ public ProfilingSummary RetrieveStatistics() /// An optional set of profiling options to configure profiling behavior. If null, default options are used. public void EnableProfiling(ProfilingOptions? options = null) { - collectstats = true; - + isProfilingEnabled = true; this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null + connectionReference?.FileReferenceCounter?.IsProfilingEnabled = true; + connectionReference?.FileReferenceCounter?.ProfilingOptions = this.profilingOptions.Value; + if (State == ConnectionState.Open) { InitProfiling(); @@ -373,6 +384,8 @@ public void DisableProfiling(bool resetStatistics = false) { ResetStatistics(); } + + connectionReference?.FileReferenceCounter?.IsProfilingEnabled = false; } /// @@ -394,13 +407,13 @@ public void ResetStatistics() /// This method prepares internal data structures to collect and store profiling information /// related to the connection's activity. It should be called before attempting to access profiling or statistics /// data for the connection. - private void InitProfiling() + private void InitProfiling() { EnsureConnectionOpen(); if (ProfilingEnabled) { - profilingInfo = new ConnectionStatistics(NativeConnection, collectstats); + profilingInfo = new ConnectionStatistics(NativeConnection, isProfilingEnabled); LoadStatisticsProfile(); } } @@ -440,7 +453,7 @@ private void DisableProfiling() } profilingInfo?.DisableQueryExecutionTracing(); - collectstats = false; + isProfilingEnabled = false; } /// @@ -456,39 +469,37 @@ private void LoadStatisticsProfile() { profilingInfo?.openTimestamp = TimerUtils.TimerCurrent(); - // Do not enable profiling again for duplicated in-memory connections; the original connection - // already has profiling enabled and the duplicated connection shares the same underlying state. - if (ProfilingEnabled && !inMemoryDuplication) + if (ProfilingEnabled && profilingOptions is { } options) { - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET enable_profiling = '{profilingOptions.Format.ToDuckDBProfilingFormatString()}'", out _); + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET enable_profiling = '{options.Format.ToDuckDBProfilingFormatString()}'", out _); if (!state.IsSuccess()) { - throw new DuckDBException($"Error setting '{"enable_profiling"}' to '{profilingOptions.Format}'"); + throw new DuckDBException($"Error setting '{"enable_profiling"}' to '{options.Format}'"); } - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_coverage = '{profilingOptions.Coverage}'", out _); + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_coverage = '{options.Coverage}'", out _); if (!state.IsSuccess()) { - throw new DuckDBException($"Error setting '{"profiling_coverage"}' to '{profilingOptions.Coverage}'"); + throw new DuckDBException($"Error setting '{"profiling_coverage"}' to '{options.Coverage}'"); } - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_mode = '{profilingOptions.Mode}'", out _); + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_mode = '{options.Mode}'", out _); if (!state.IsSuccess()) { - throw new DuckDBException($"Error setting '{"profiling_mode"}' to '{profilingOptions.Mode}'"); + throw new DuckDBException($"Error setting '{"profiling_mode"}' to '{options.Mode}'"); } - if (!string.IsNullOrEmpty(profilingOptions.OutputPath)) + if (!string.IsNullOrEmpty(options.OutputPath)) { - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_output = '{profilingOptions.OutputPath}'", out _); + state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_output = '{options.OutputPath}'", out _); if (!state.IsSuccess()) { - throw new DuckDBException($"Error setting '{"profiling_output"}' to '{profilingOptions.OutputPath}'"); + throw new DuckDBException($"Error setting '{"profiling_output"}' to '{options.OutputPath}'"); } } - var enabledMetrics = profilingOptions.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); + var enabledMetrics = options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET custom_profiling_settings = '{enabledMetrics.ToDuckDBMetricString()}'", out _); if (!state.IsSuccess()) { diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 974576d2..ee194aef 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -12,7 +12,6 @@ internal sealed class QueryProfiler : ExecutionStatisticsBase internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - private Dictionary infos = []; private readonly IntPtr queryIdentifier; private readonly int statementCount; diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 1c3859a5..1fe0b1f1 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Profiling; +using DuckDB.NET.Test.Helpers; namespace DuckDB.NET.Test.Profiling; @@ -50,6 +51,207 @@ public void MetricsDictionaryContainsEnabledMetrics() } } + [Fact] + public async Task FileBackedConnectionSharesProfilingStateWhenEnabledOnFirst() + { + // create a physical file-backed database + using var dbInfo = DisposableFile.GenerateInTemp("db"); + + // open first connection and enable profiling + await using (var conn1 = new DuckDBConnection(dbInfo.ConnectionString)) + { + conn1.Open(); + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + conn1.EnableProfiling(options); + + // run a query to ensure profiling collects something + using var cmd1 = conn1.CreateCommand(); + cmd1.CommandText = "SELECT 1;"; + using (var r = cmd1.ExecuteReader()) { } + + // open a second connection to the same file WITHOUT enabling profiling explicitly + await using (var conn2 = new DuckDBConnection(dbInfo.ConnectionString)) + { + conn2.Open(); + + // conn2 should have profiling enabled because it shares the file-backed DB state + conn2.ProfilingEnabled.Should().BeTrue(); + + // run a query on conn2 and ensure it has its own statistics recorded + using var cmd2 = conn2.CreateCommand(); + cmd2.CommandText = "SELECT 2;"; + using (var r = cmd2.ExecuteReader()) { } + + var s2 = conn2.RetrieveStatistics(); + s2.QueryCount.Should().BeGreaterThan(0); + } + + conn1.DisableProfiling(true); + } + } + + [Fact] + public void InMemoryDuplicateHasIndependentStatistics() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + // ensure clean start + Connection.ResetStatistics(); + + // run a query on the parent + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var parentSummary = Connection.RetrieveStatistics(); + parentSummary.QueryCount.Should().BeGreaterThanOrEqualTo(1); + var parentCount = parentSummary.QueryCount; + + // create a duplicate connection and run a separate query + using var dup = Connection.Duplicate(); + dup.Open(); + using var dupCmd = dup.CreateCommand(); + dupCmd.CommandText = "SELECT 2;"; + using (var r = dupCmd.ExecuteReader()) { } + + var dupSummary = dup.RetrieveStatistics(); + // the duplicate should have its own stats with only the query it ran + dupSummary.QueryCount.Should().Be(1); + + // parent stats should not include duplicate's query + var parentAfter = Connection.RetrieveStatistics(); + parentAfter.QueryCount.Should().Be(parentCount); + } + finally + { + Connection.DisableProfiling(true); + } + } + + [Fact] + public async Task FileBackedDuplicateReturnsSameMetricsAsOriginal() + { + using var dbInfo = DisposableFile.GenerateInTemp("db"); + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName, DuckDBMetricType.CpuTime, DuckDBMetricType.Latency }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + await using (var conn1 = new DuckDBConnection(dbInfo.ConnectionString)) + { + conn1.Open(); + conn1.EnableProfiling(options); + + using (var c = conn1.CreateCommand()) + { + c.CommandText = "SELECT 1;"; + using var r = c.ExecuteReader(); + } + + await using (var conn2 = new DuckDBConnection(dbInfo.ConnectionString)) + { + conn2.Open(); + // conn2 should have profiling enabled implicitly for file-backed DB when conn1 enabled it + conn2.ProfilingEnabled.Should().BeTrue(); + + using (var c2 = conn2.CreateCommand()) + { + c2.CommandText = "SELECT 2;"; + using var r2 = c2.ExecuteReader(); + } + + var s1 = conn1.RetrieveStatistics(); + var s2 = conn2.RetrieveStatistics(); + + s1.QuerySummaryList.Length.Should().BeGreaterThan(0, "Expected profiling summaries for original connection"); + s2.QuerySummaryList.Length.Should().BeGreaterThan(0, "Expected profiling summaries for duplicate connection"); + + var last1 = s1.QuerySummaryList.Last(); + var last2 = s2.QuerySummaryList.Last(); + + var keys1 = new HashSet(last1.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + var keys2 = new HashSet(last2.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + + // strict check: both sets of metric keys must match + keys1.Should().BeEquivalentTo(keys2); + } + + conn1.DisableProfiling(true); + } + } + + [Fact] + public void InMemoryDuplicateReturnsSameMetricsAsOriginal() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName, DuckDBMetricType.CpuTime, DuckDBMetricType.Latency }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + // run a query on the original + Command.CommandText = "SELECT 10;"; + using (var r = Command.ExecuteReader()) { } + + // duplicate in-memory and open + using var dup = Connection.Duplicate(); + dup.Open(); + + // run a query on duplicate + using var cdup = dup.CreateCommand(); + cdup.CommandText = "SELECT 20;"; + using (var r = cdup.ExecuteReader()) { } + + var sOriginal = Connection.RetrieveStatistics(); + var sDup = dup.RetrieveStatistics(); + + sOriginal.QuerySummaryList.Length.Should().BeGreaterThan(0, "Expected profiling summaries for original connection"); + sDup.QuerySummaryList.Length.Should().BeGreaterThan(0, "Expected profiling summaries for duplicate connection"); + + var lastOriginal = sOriginal.QuerySummaryList.Last(); + var lastDup = sDup.QuerySummaryList.Last(); + + var keysOrig = new HashSet(lastOriginal.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + var keysDup = new HashSet(lastDup.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + + // strict check: both sets of metric keys must match + keysOrig.Should().BeEquivalentTo(keysDup); + } + finally + { + Connection.DisableProfiling(true); + } + } + [Fact] public void MultipleEnabledMetricsArePresentInMetricsDictionary() { From b62ea9b53f795fa90d0a9e75d693d115fe511501 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Tue, 19 May 2026 15:57:45 +0200 Subject: [PATCH 13/43] Add profilingOptions to duplicatedConnection initialization This commit updates the DuckDBConnection class to include the `profilingOptions` parameter when initializing the `duplicatedConnection` object. This ensures that profiling options are properly passed and set during the duplication process. --- DuckDB.NET.Data/DuckDBConnection.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index ecc09055..c50322c2 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -306,6 +306,7 @@ public DuckDBConnection Duplicate() inMemoryDuplication = true, connectionReference = connectionReference, isProfilingEnabled = isProfilingEnabled, + profilingOptions = profilingOptions, }; return duplicatedConnection; From ea5d5ff2588d0effa692279e100ac7bda3bf44a3 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Tue, 19 May 2026 17:02:44 +0200 Subject: [PATCH 14/43] Enhance time utilities and profiling logic Added new time-related utility methods in `TimerUtils.cs` for improved time tracking and conversions. Updated `PreparedStatement.cs` to ensure accurate query tracing by stopping the tracer after the last statement in a batch. Refactored `ConnectionStatistics.cs` to improve profiling summary calculations and reset logic. Simplified and cleaned up the sample program in `Program.cs` for better readability and functionality. --- DuckDB.NET.Data/Common/TimerUtils.cs | 19 +++++ .../PreparedStatement/PreparedStatement.cs | 12 +++- .../Statistics/ConnectionStatistics.cs | 9 ++- DuckDB.NET.Samples/Program.cs | 71 +++++-------------- 4 files changed, 55 insertions(+), 56 deletions(-) diff --git a/DuckDB.NET.Data/Common/TimerUtils.cs b/DuckDB.NET.Data/Common/TimerUtils.cs index 285db04a..40d1afdb 100644 --- a/DuckDB.NET.Data/Common/TimerUtils.cs +++ b/DuckDB.NET.Data/Common/TimerUtils.cs @@ -3,16 +3,35 @@ internal sealed class TimerUtils { + /// + /// Gets the current UTC time as the number of ticks elapsed since 12:00:00 midnight, January 1, 0001. + /// + /// A 64-bit integer representing the current UTC time in ticks, where one tick equals 100 nanoseconds. internal static long TimerCurrent() => DateTimeOffset.UtcNow.UtcTicks; + /// + /// Gets the current date and time in Coordinated Universal Time (UTC). + /// + /// A value that represents the current UTC date and time. internal static DateTimeOffset Now() => DateTimeOffset.UtcNow; + /// + /// Calculates the elapsed number of ticks between two tick count values. + /// + /// The starting tick count value, typically representing the beginning of a time interval. + /// The ending tick count value, typically representing the end of a time interval. + /// The number of ticks that have elapsed between the start and end tick counts, as an unsigned 32-bit integer. internal static uint CalculateTickCountElapsed(long startTick, long endTick) { return (uint)(endTick - startTick); } + /// + /// Converts a timer value expressed in ticks to its equivalent value in milliseconds. + /// + /// The timer value, in ticks, to convert to milliseconds. + /// The equivalent value in milliseconds as a 64-bit integer. internal static long TimerToMilliseconds(long timerValue) { long result = timerValue / TimeSpan.TicksPerMillisecond; diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 6fa918b2..f54ed0e1 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -43,7 +43,17 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c if (status.IsSuccess()) { using var preparedStatement = new PreparedStatement(statement); - yield return preparedStatement.Execute(parameters, useStreamingMode, connection); + var result = preparedStatement.Execute(parameters, useStreamingMode, connection); + + // Stop the query tracer after the last statement has been executed. + // This ensures that the total execution time for the entire batch of statements is accurately captured + // and not after the data retrieval of the last statement. + if (index == statementCount - 1) + { + queryTracer?.StopTimer(); + } + + yield return result; } else { diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 4d2c95ac..110b85d9 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -1,5 +1,7 @@ using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.PreparedStatement; using System.Collections.Concurrent; +using System.Linq; namespace DuckDB.NET.Data.Profiling.Statistics { @@ -116,10 +118,10 @@ private IEnumerable ReadSummaries() internal ProfilingSummary GetProfilingSummary() { return new ProfilingSummary( - startExecutionTime, - endExecutionTime, + new DateTimeOffset(openTimestamp, TimeSpan.Zero), + new DateTimeOffset(connectionTime, TimeSpan.Zero), TimerUtils.TimerToMilliseconds(connectionTime), - TimerUtils.TimerToMilliseconds(executionTime), + TimerUtils.TimerToMilliseconds(queryProfilers.Values.Sum(qp => qp.executionTime)), queryProfilers.Values.Count, [.. ReadSummaries()]); } @@ -131,6 +133,7 @@ internal void Reset() connectionTime = 0; startExecutionTime = default; endExecutionTime = default; + openTimestamp = 0; queryProfilers.Clear(); } diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 5b4c829b..bee41ac7 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -280,65 +280,32 @@ private static void Profiling() using var dBCommand = con.CreateCommand(); - //LoadTpch(con); + LoadTpch(con); - for (var selectPosition = 0; selectPosition < 3; selectPosition++) - { - - con.EnableProfiling(options); - - try - { - - var val = 200 + selectPosition; - string a = $"t_{id}_a"; - string b = $"t_{id}_b"; + con.EnableProfiling(options); - string cmd; - if (selectPosition == 0) - { - cmd = $"SELECT {val}; CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER);"; - } - else if (selectPosition == 1) - { - cmd = $"CREATE TABLE {a}(i INTEGER); SELECT {val}; CREATE TABLE {b}(i INTEGER);"; - } - else - { - cmd = $"CREATE TABLE {a}(i INTEGER); CREATE TABLE {b}(i INTEGER); SELECT {val};"; - } - - dBCommand.CommandText = cmd; - using var reader = dBCommand.ExecuteReader(); - do { } while (reader.NextResult()); - - var summary = con.RetrieveStatistics(); - var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); - Console.WriteLine($"newSummaries length: {newSummaries.Length}"); - } - finally - { - con.DisableProfiling(); - } - } - - //using var cmd = con.CreateCommand(); - //cmd.CommandText = "SELECT a:1;"; - //using var reader = cmd.ExecuteReader(); + using var cmd = con.CreateCommand(); + cmd.CommandText = "SELECT a:1;"; + using var reader = cmd.ExecuteReader(); + //while (reader.Read()) { }; + //while (reader.NextResult()) { }; + + var metrics = con.RetrieveStatistics(); + + cmd.CommandText = $"PRAGMA tpch(1);"; + cmd.ExecuteNonQuery(); - //cmd.CommandText = $"PRAGMA tpch(1);"; - //cmd.ExecuteNonQuery(); + metrics = con.RetrieveStatistics(); - //var metrics = con.RetrieveStatistics(); - //Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); + Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); - //cmd.CommandText = $"PRAGMA tpch(2); CREATE TABLE test (id int);"; - //cmd.ExecuteNonQuery(); + cmd.CommandText = $"PRAGMA tpch(2); CREATE TABLE test (id int);"; + cmd.ExecuteNonQuery(); - //metrics = con.RetrieveStatistics(); - //Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); + metrics = con.RetrieveStatistics(); + Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); - //PrintMetrics(metrics); + PrintMetrics(metrics); //PrintQueryResults(reader); From 6bd387435206e9671e63aeec5f2d884c04f6e08c Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 20 May 2026 11:19:28 +0200 Subject: [PATCH 15/43] Refactor profiling and add SemVer support Refactored profiling infrastructure to use `ProfilingStatementSummary` for detailed statement-level metrics. Updated `MetricType.cs`, `ConnectionStatistics.cs`, `QueryProfiler.cs`, and `StatementProfiler.cs` to align with the new structure. Enhanced test coverage in `ProfilingTests.cs` to validate changes. Added SemVer-style prerelease and build metadata support in `irion-package.ps1`, including a new `Split-SemVersion` function and updated version bumping logic. Transitioned versioning to `1.5.2-alpha.1` in `irion.version`. Updated documentation in `README.md` to reflect versioning changes. General cleanup and alignment across the codebase to improve consistency and clarity. --- DuckDB.NET.Data/Profiling/MetricType.cs | 2 +- .../Statistics/ConnectionStatistics.cs | 2 +- .../Statistics/ProfilingQuerySummary.cs | 2 +- .../Statistics/ProfilingStatementSummary.cs | 21 +++++ .../Profiling/Statistics/QueryProfiler.cs | 12 ++- .../Statistics/QueryProfilerTracer.cs | 2 +- .../Profiling/Statistics/StatementProfiler.cs | 24 ++++- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 26 ++--- README.md | 3 +- build/irion.version | 2 +- scripts/irion-package.ps1 | 94 ++++++++++++++++--- 11 files changed, 151 insertions(+), 39 deletions(-) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs diff --git a/DuckDB.NET.Data/Profiling/MetricType.cs b/DuckDB.NET.Data/Profiling/MetricType.cs index c9ffb9b2..511509a6 100644 --- a/DuckDB.NET.Data/Profiling/MetricType.cs +++ b/DuckDB.NET.Data/Profiling/MetricType.cs @@ -13,7 +13,7 @@ public static class DuckDBMetrics DuckDBMetricType.CpuTime, DuckDBMetricType.BlockedThreadTime, DuckDBMetricType.ResultSetSize, - DuckDBMetricType.RowsReturned, + //DuckDBMetricType.RowsReturned, DuckDBMetricType.CumulativeRowsScanned, DuckDBMetricType.TotalBytesRead, DuckDBMetricType.TotalBytesWritten, diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 110b85d9..4ac88b4b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -111,7 +111,7 @@ private IEnumerable ReadSummaries() { foreach (var queryProfiler in queryProfilers.Values) { - yield return queryProfiler.GetQuerySummary(); + yield return queryProfiler.GetSummary(); } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs index 2d7829d1..65a7cc9d 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs @@ -5,7 +5,7 @@ public readonly record struct ProfilingQuerySummary( DateTimeOffset EndTime, double ExecutionTimeMilliseconds, int StatementCount, - ProfilingInfoMetrics[] Infos) + ProfilingStatementSummary[] Infos) { public IDictionary ToDictionary() { diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs new file mode 100644 index 00000000..028beae0 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs @@ -0,0 +1,21 @@ +namespace DuckDB.NET.Data.Profiling.Statistics; + +public readonly record struct ProfilingStatementSummary( + DateTimeOffset StartTime, + DateTimeOffset EndTime, + double ExecutionTimeMilliseconds, + int Order, + ProfilingInfoMetrics Metrics) +{ + public IDictionary ToDictionary() + { + return new Dictionary(6) + { + ["StartTime"] = StartTime, + ["EndTime"] = EndTime, + ["ExecutionTime"] = ExecutionTimeMilliseconds, + ["Order"] = Order, + ["Infos"] = Metrics + }; + } +} diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index ee194aef..29d66513 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -21,9 +21,13 @@ internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeC this.statementProfilers = new StatementProfiler[statementCount]; } + internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); + internal DateTimeOffset StartTime => startExecutionTime; + internal DateTimeOffset EndTime => endExecutionTime; + internal int StatementCount => statementCount; + internal IntPtr QueryIdentifier => queryIdentifier; - internal int StatementCount => statementCount; internal bool RegisterStatementProfiler(StatementProfiler statistics, int queryIndex) { @@ -47,15 +51,15 @@ internal void ReleaseAndUpdateExecutionTimer() } } - private IEnumerable GetStatementInfo() + private IEnumerable GetStatementInfo() { foreach (var statementProfiler in statementProfilers) { - yield return statementProfiler?.Info ?? []; + yield return statementProfiler.GetSummary(); } } - internal ProfilingQuerySummary GetQuerySummary() + internal ProfilingQuerySummary GetSummary() { return new ProfilingQuerySummary( startExecutionTime, diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs index 1a70e59e..71220917 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs @@ -19,7 +19,7 @@ internal StatementProfilerTracer CreateStatementProfilerTracer(DuckDBPreparedSta throw new InvalidOperationException("Statement profiler tracer already exists for the given prepared statement."); } - var statementProfiler = new StatementProfiler(preparedStatement, statementIndex, duckDBNativeConnection); + var statementProfiler = new StatementProfiler(statementIndex, duckDBNativeConnection); statementTracers[preparedStatement] = new StatementProfilerTracer(statementProfiler); queryProfiler.RegisterStatementProfiler(statementProfiler, statementIndex); diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 0d641133..9fd217fd 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -12,17 +12,31 @@ internal sealed class StatementProfiler: ExecutionStatistics internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - private ProfilingInfoMetrics info; - private readonly DuckDBPreparedStatement preparedStatement; + private ProfilingInfoMetrics info = []; private readonly int queryIndex; - internal StatementProfiler(DuckDBPreparedStatement preparedStatement, int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) + internal StatementProfiler(int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { - this.preparedStatement = preparedStatement; this.queryIndex = queryIndex; } - public ProfilingInfoMetrics Info => info; + internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); + internal DateTimeOffset StartTime => startExecutionTime; + internal DateTimeOffset EndTime => endExecutionTime; + + internal ProfilingInfoMetrics Info => info; + + internal int QueryIndex => queryIndex; + + internal ProfilingStatementSummary GetSummary() + { + return new ProfilingStatementSummary( + StartTime: startExecutionTime, + EndTime: endExecutionTime, + ExecutionTimeMilliseconds: TimerUtils.TimerToMilliseconds(executionTime), + Order: queryIndex, + Metrics: info); + } internal override void StartTimer() { diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 1fe0b1f1..3ba52b28 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -36,11 +36,11 @@ public void MetricsDictionaryContainsEnabledMetrics() // If no new summaries were produced, profiling might not be available in this environment. var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); - var last = newSummaries.Last(); + var last = newSummaries.Last().Infos; // check that enabled metrics appear in at least one statement's metrics - var containsQueryName = last.Infos.Any(i => i != null && i.ContainsKey(DuckDBMetricType.QueryName)); - var containsCpuTime = last.Infos.Any(i => i != null && i.ContainsKey(DuckDBMetricType.CpuTime)); + var containsQueryName = last.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.QueryName)); + var containsCpuTime = last.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.CpuTime)); containsQueryName.Should().BeTrue(); containsCpuTime.Should().BeTrue(); @@ -190,8 +190,8 @@ public async Task FileBackedDuplicateReturnsSameMetricsAsOriginal() var last1 = s1.QuerySummaryList.Last(); var last2 = s2.QuerySummaryList.Last(); - var keys1 = new HashSet(last1.Infos.Where(i => i != null).SelectMany(i => i.Keys)); - var keys2 = new HashSet(last2.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + var keys1 = new HashSet(last1.Infos.SelectMany(i => i.Metrics.Keys)); + var keys2 = new HashSet(last2.Infos.SelectMany(i => i.Metrics.Keys)); // strict check: both sets of metric keys must match keys1.Should().BeEquivalentTo(keys2); @@ -240,8 +240,8 @@ public void InMemoryDuplicateReturnsSameMetricsAsOriginal() var lastOriginal = sOriginal.QuerySummaryList.Last(); var lastDup = sDup.QuerySummaryList.Last(); - var keysOrig = new HashSet(lastOriginal.Infos.Where(i => i != null).SelectMany(i => i.Keys)); - var keysDup = new HashSet(lastDup.Infos.Where(i => i != null).SelectMany(i => i.Keys)); + var keysOrig = new HashSet(lastOriginal.Infos.SelectMany(i => i.Metrics.Keys)); + var keysDup = new HashSet(lastDup.Infos.SelectMany(i => i.Metrics.Keys)); // strict check: both sets of metric keys must match keysOrig.Should().BeEquivalentTo(keysDup); @@ -290,7 +290,7 @@ public void MultipleEnabledMetricsArePresentInMetricsDictionary() last.Infos.Length.Should().BeGreaterThan(0, "Expected some profiling info when metrics are enabled"); // ensure at least one statement collected all requested metrics - bool found = last.Infos.Any(info => info != null && metricsToEnable.All(m => info.ContainsKey(m))); + bool found = last.Infos.Any(summary => metricsToEnable.All(m => summary.Metrics.ContainsKey(m))); found.Should().BeTrue("At least one statement should include all the enabled metrics (QueryName, CpuTime, Latency, TotalBytesRead)"); } @@ -357,7 +357,7 @@ public void SelectCoverageRespectsSelectPosition(int selectPosition) { for (int i = 0; i < last.StatementCount; i++) { - var info = i < last.Infos.Length ? last.Infos[i] : null; + var info = i < last.Infos.Length ? last.Infos[i].Metrics : null; if (i == selectPosition) { info.Should().NotBeNull(); @@ -372,8 +372,8 @@ public void SelectCoverageRespectsSelectPosition(int selectPosition) } else { - // If Infos length doesn't map 1:1 to statements, at least ensure exactly one statement collected metrics - var nonEmpty = last.Infos.Count(i => i != null && i.Count > 0); + // If Metrics length doesn't map 1:1 to statements, at least ensure exactly one statement collected metrics + var nonEmpty = last.Infos.Count(i => i.Metrics.Count > 0); nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); } } @@ -504,7 +504,7 @@ public void CoverageSelectOnlyProfilesOnlySelectStatements() { for (int i = 0; i < last.StatementCount; i++) { - var info = i < last.Infos.Length ? last.Infos[i] : null; + var info = i < last.Infos.Length ? last.Infos[i].Metrics : null; if (i == last.StatementCount - 1) { info.Should().NotBeNull(); @@ -519,7 +519,7 @@ public void CoverageSelectOnlyProfilesOnlySelectStatements() } else { - var nonEmpty = last.Infos.Count(i => i != null && i.Count > 0); + var nonEmpty = last.Infos.Count(i => i.Metrics.Count > 0); nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); } } diff --git a/README.md b/README.md index 8237a7c6..0686d1ad 100644 --- a/README.md +++ b/README.md @@ -100,12 +100,13 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.4.4.2 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2-alpha.1 ``` Bump version in `build/irion.version`: ```powershell +.\scripts\irion-package.ps1 -Command bump -Part prerelease # 1.4.4-alpha.1 -> 1.4.4-alpha.2 .\scripts\irion-package.ps1 -Command bump -Part revision # 1.4.4.1 -> 1.4.4.2 .\scripts\irion-package.ps1 -Command bump -Part build # 1.4.4.1 -> 1.4.5.0 .\scripts\irion-package.ps1 -Command bump -Part minor # 1.4.4.1 -> 1.5.0.0 diff --git a/build/irion.version b/build/irion.version index 3d19aff8..ad88bf4e 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.1 \ No newline at end of file +1.5.2-alpha.1 \ No newline at end of file diff --git a/scripts/irion-package.ps1 b/scripts/irion-package.ps1 index b3b0b46b..a230af6a 100644 --- a/scripts/irion-package.ps1 +++ b/scripts/irion-package.ps1 @@ -5,7 +5,7 @@ param( [string]$Version, - [ValidateSet("major", "minor", "build", "revision")] + [ValidateSet("major", "minor", "build", "revision", "prerelease")] [string]$Part = "revision", [string]$VersionFile = "build/irion.version", @@ -55,13 +55,47 @@ function Parse-VersionString { $trimmed = $VersionString.Trim() $parsed = $null - if (-not [Version]::TryParse($trimmed, [ref]$parsed)) { - throw "Invalid version '$VersionString'. Expected format like '1.4.4.1'." + # Allow SemVer-style prerelease or build metadata (e.g. 1.5.2-alpha.1 or 1.5.2+meta) + # by extracting the numeric core before any '-' (prerelease) or '+' (build metadata). + $numericPart = ($trimmed -split '[-+]')[0] + + if (-not [Version]::TryParse($numericPart, [ref]$parsed)) { + throw "Invalid version '$VersionString'. Expected numeric format like '1.4.4.1' or semver with prerelease like '1.4.4-alpha.1'." } return (Normalize-Version -ParsedVersion $parsed) } +function Split-SemVersion { + param([Parameter(Mandatory = $true)][string]$VersionString) + + $raw = $VersionString.Trim() + $sepIndex = -1 + for ($i = 0; $i -lt $raw.Length; $i++) { + if ($raw[$i] -eq '-' -or $raw[$i] -eq '+') { $sepIndex = $i; break } + } + + if ($sepIndex -ge 0) { + $numericPart = $raw.Substring(0, $sepIndex) + $suffix = $raw.Substring($sepIndex) # includes leading '-' or '+' + } + else { + $numericPart = $raw + $suffix = '' + } + + $parsed = $null + if (-not [Version]::TryParse($numericPart, [ref]$parsed)) { + throw "Invalid version '$VersionString'. Expected numeric format like '1.4.4.1' or semver with prerelease like '1.4.4-alpha.1'." + } + + return [PSCustomObject]@{ + Raw = $raw + Numeric = (Normalize-Version -ParsedVersion $parsed) + Suffix = $suffix + } +} + function Get-VersionFilePath { return (Resolve-RepoPath -Path $VersionFile) } @@ -69,13 +103,19 @@ function Get-VersionFilePath { function Get-NuGetPackageVersion { param([Parameter(Mandatory = $true)][string]$VersionString) - $version = Parse-VersionString -VersionString $VersionString + $parts = Split-SemVersion -VersionString $VersionString + $version = $parts.Numeric + $suffix = $parts.Suffix + + # NuGet supports prerelease suffixes (with '-') but not build metadata ('+' section), + # so drop '+'-prefixed suffixes and keep '-'-prefixed prerelease identifiers. + if ($suffix.StartsWith('+')) { $suffix = '' } if ($version.Revision -eq 0) { - return $version.ToString(3) + return ($version.ToString(3) + $suffix) } - return $version.ToString(4) + return ($version.ToString(4) + $suffix) } function Get-CurrentVersion { @@ -86,13 +126,16 @@ function Get-CurrentVersion { } $rawValue = Get-Content -Path $versionFilePath -Raw - return (Parse-VersionString -VersionString $rawValue).ToString(4) + return $rawValue.Trim() } function Save-Version { param([Parameter(Mandatory = $true)][string]$VersionToSave) - $normalizedVersion = (Parse-VersionString -VersionString $VersionToSave).ToString(4) + $trimmed = $VersionToSave.Trim() + # Validate semver but preserve the original string (including prerelease) in file + [void](Split-SemVersion -VersionString $trimmed) + $versionFilePath = Get-VersionFilePath $parentDir = Split-Path -Parent $versionFilePath @@ -100,8 +143,8 @@ function Save-Version { New-Item -Path $parentDir -ItemType Directory -Force | Out-Null } - Set-Content -Path $versionFilePath -Value $normalizedVersion -NoNewline - return $normalizedVersion + Set-Content -Path $versionFilePath -Value $trimmed -NoNewline + return $trimmed } function Get-BumpedVersion { @@ -110,13 +153,42 @@ function Get-BumpedVersion { [Parameter(Mandatory = $true)][string]$BumpPart ) - $version = Parse-VersionString -VersionString $CurrentVersion + $parts = Split-SemVersion -VersionString $CurrentVersion + $version = $parts.Numeric + $suffix = $parts.Suffix switch ($BumpPart) { "major" { return [Version]::new($version.Major + 1, 0, 0, 0).ToString(4) } "minor" { return [Version]::new($version.Major, $version.Minor + 1, 0, 0).ToString(4) } "build" { return [Version]::new($version.Major, $version.Minor, $version.Build + 1, 0).ToString(4) } "revision" { return [Version]::new($version.Major, $version.Minor, $version.Build, $version.Revision + 1).ToString(4) } + "prerelease" { + if ([string]::IsNullOrEmpty($suffix) -or $suffix.StartsWith('+')) { + throw "No prerelease suffix to bump for version '$CurrentVersion'." + } + + # strip leading '-' + $s = $suffix.Substring(1) + $lastDot = $s.LastIndexOf('.') + if ($lastDot -lt 0) { + # no numeric tail, append .1 + $newSuffix = "$s.1" + } + else { + $label = $s.Substring(0, $lastDot) + $tail = $s.Substring($lastDot + 1) + if ([int]::TryParse($tail, [ref]$null)) { + $num = [int]$tail + $newSuffix = "$label.$($num + 1)" + } + else { + # tail is not numeric; append .1 + $newSuffix = "$s.1" + } + } + + return ($version.ToString(4) + '-' + $newSuffix) + } default { throw "Unsupported bump part '$BumpPart'." } } } From d4a44a915fd2b3ab7677a10c94e78a1db1412a88 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 20 May 2026 15:39:57 +0200 Subject: [PATCH 16/43] Refactor profiling and timing utilities Refactored `TimerUtils` to use high-resolution timing with `Stopwatch.GetTimestamp()` and added `TimerToMilliseconds` for precise conversions. Simplified `CalculateTickCountElapsed` and removed redundant methods. Updated profiling configuration in `DuckDBConnection` to use `StringBuilder` for efficient query construction. Improved error handling and removed redundant checks. Replaced `Dictionary` with `ConcurrentDictionary` in `ConnectionStatistics` for thread safety. Added a fast path in `TryGetFor` to optimize lookups. Refactored `ProfilingInfo` and `StatementProfiler` to use raw metrics (`Dictionary`) instead of `ProfilingInfoMetrics`. Updated related methods and properties for consistency. Changed `TimerUtils.Now()` usage to `DateTimeOffset` for better precision --- DuckDB.NET.Bindings/DuckDBWrapperObjects.cs | 2 +- DuckDB.NET.Data/Common/TimerUtils.cs | 48 ++++++++--------- DuckDB.NET.Data/DuckDBConnection.cs | 51 +++++-------------- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 16 +++++- .../Statistics/ConnectionStatistics.cs | 12 ++++- .../Statistics/ProfilingInfoMetrics.cs | 2 +- .../Profiling/Statistics/QueryProfiler.cs | 4 +- .../Profiling/Statistics/StatementProfiler.cs | 15 +++--- build/irion.version | 2 +- 9 files changed, 73 insertions(+), 79 deletions(-) diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index 9e09ff13..42164ab6 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -174,7 +174,7 @@ public T GetValue() static T Cast(TSource value) => Unsafe.As(ref value); } - public IDictionary GetMapValue() where TKey : notnull + public Dictionary GetMapValue() where TKey : notnull { var result = new Dictionary(); diff --git a/DuckDB.NET.Data/Common/TimerUtils.cs b/DuckDB.NET.Data/Common/TimerUtils.cs index 40d1afdb..018b316a 100644 --- a/DuckDB.NET.Data/Common/TimerUtils.cs +++ b/DuckDB.NET.Data/Common/TimerUtils.cs @@ -1,41 +1,37 @@ -namespace DuckDB.NET.Data.Common +using System.Diagnostics; + +namespace DuckDB.NET.Data.Common { internal sealed class TimerUtils { + private static readonly long Frequency = Stopwatch.Frequency; /// - /// Gets the current UTC time as the number of ticks elapsed since 12:00:00 midnight, January 1, 0001. + /// Gets the current timestamp value for high-resolution timing operations. /// - /// A 64-bit integer representing the current UTC time in ticks, where one tick equals 100 nanoseconds. - internal static long TimerCurrent() => DateTimeOffset.UtcNow.UtcTicks; + /// The returned value can be used with Stopwatch.Frequency to calculate elapsed time intervals with high + /// precision. This method is intended for scenarios where precise timing is required, such as performance + /// measurements. + /// A long integer representing the current timestamp, as provided by the system's high-resolution performance counter. + internal static long TimerCurrent() => Stopwatch.GetTimestamp(); /// - /// Gets the current date and time in Coordinated Universal Time (UTC). + /// Converts a timer value to its equivalent duration in milliseconds. /// - /// A value that represents the current UTC date and time. - internal static DateTimeOffset Now() => DateTimeOffset.UtcNow; + /// The conversion uses the timer's frequency to determine the number of milliseconds represented by the + /// timer value. Ensure that the timer value and frequency are based on the same timer source. + /// The timer value to convert, typically representing elapsed timer ticks. + /// The equivalent duration in milliseconds calculated from the specified timer value. + internal static long TimerToMilliseconds(long timerValue) + => timerValue * 1000 / Frequency; /// /// Calculates the elapsed number of ticks between two tick count values. /// - /// The starting tick count value, typically representing the beginning of a time interval. - /// The ending tick count value, typically representing the end of a time interval. - /// The number of ticks that have elapsed between the start and end tick counts, as an unsigned 32-bit integer. - internal static uint CalculateTickCountElapsed(long startTick, long endTick) - { - - return (uint)(endTick - startTick); - } - - /// - /// Converts a timer value expressed in ticks to its equivalent value in milliseconds. - /// - /// The timer value, in ticks, to convert to milliseconds. - /// The equivalent value in milliseconds as a 64-bit integer. - internal static long TimerToMilliseconds(long timerValue) - { - long result = timerValue / TimeSpan.TicksPerMillisecond; - return result; - } + /// The starting tick count value. + /// The ending tick count value. + /// The difference between the ending and starting tick count values, representing the elapsed ticks. + internal static long CalculateTickCountElapsed(long startTick, long endTick) + => endTick - startTick; // keep as long, no truncation } } diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index c50322c2..193b3a07 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -5,6 +5,7 @@ using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Text; namespace DuckDB.NET.Data; @@ -472,48 +473,20 @@ private void LoadStatisticsProfile() if (ProfilingEnabled && profilingOptions is { } options) { - - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET enable_profiling = '{options.Format.ToDuckDBProfilingFormatString()}'", out _); - if (!state.IsSuccess()) - { - throw new DuckDBException($"Error setting '{"enable_profiling"}' to '{options.Format}'"); - } - - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_coverage = '{options.Coverage}'", out _); - if (!state.IsSuccess()) - { - throw new DuckDBException($"Error setting '{"profiling_coverage"}' to '{options.Coverage}'"); - } - - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_mode = '{options.Mode}'", out _); - if (!state.IsSuccess()) - { - throw new DuckDBException($"Error setting '{"profiling_mode"}' to '{options.Mode}'"); - } - + var sb = new StringBuilder(); + sb.Append($"SET enable_profiling = '{options.Format.ToDuckDBProfilingFormatString()}';"); + sb.Append($"SET profiling_coverage = '{options.Coverage}';"); + sb.Append($"SET profiling_mode = '{options.Mode}';"); + if (!string.IsNullOrEmpty(options.OutputPath)) - { - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET profiling_output = '{options.OutputPath}'", out _); - if (!state.IsSuccess()) - { - throw new DuckDBException($"Error setting '{"profiling_output"}' to '{options.OutputPath}'"); - } - } - - var enabledMetrics = options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); - state = NativeMethods.Query.DuckDBQuery(NativeConnection, $"SET custom_profiling_settings = '{enabledMetrics.ToDuckDBMetricString()}'", out _); - if (!state.IsSuccess()) - { - DuckDBException? innerEx = null; - - if (enabledMetrics.Contains(DuckDBMetricType.RowsReturned)) - { - innerEx = new DuckDBException($"The '{DuckDBMetricType.RowsReturned}' metric is not supported yet.", DuckDBErrorType.InvalidInput); - } + sb.Append($"SET profiling_output = '{options.OutputPath}';"); - throw new DuckDBException($"Error setting '{"custom_profiling_settings"}' to '{enabledMetrics.ToDuckDBMetricString()}'", innerEx); + var metrics = options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); + sb.Append($"SET custom_profiling_settings = '{metrics.ToDuckDBMetricString()}';"); - } + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sb.ToString(), out _); + if (!state.IsSuccess()) + throw new DuckDBException("Error configuring profiling settings."); } } diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 952980fb..024b38a8 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -37,7 +37,21 @@ internal ProfilingInfoMetrics GetMetrics() return []; } - return ProfilingInfoMetrics.FromMetricsDictionary(metricsValue.GetMapValue()); + return ProfilingInfoMetrics.FromRawMetrics(metricsValue.GetMapValue()); + } + + internal Dictionary GetRawMetrics() + { + if (duckDBProfilingInfoWrapper == null) + { + throw new InvalidOperationException("Profiling info is not prepared. Call Prepare() first."); + } + var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); + if (metricsValue.IsNull()) + { + return new Dictionary(); + } + return metricsValue.GetMapValue(); } private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int indent) diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 4ac88b4b..9e67898b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -18,7 +18,7 @@ private static readonly ConcurrentDictionary queryProfilers = []; + private readonly ConcurrentDictionary queryProfilers = new(); private bool isDisposed = false; // internal values that are exposed through properties @@ -71,7 +71,15 @@ internal void DisableQueryExecutionTracing() /// if statistics are found for the specified connection; otherwise, . internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) - => ByNativeConnection.TryGetValue(nativeConn, out stats); + { + // Fast path: if no connections have profiling enabled, skip the lookup entirely + if (ByNativeConnection.IsEmpty) + { + stats = null; + return false; + } + return ByNativeConnection.TryGetValue(nativeConn, out stats); + } /// /// Creates a new for the specified query if query execution tracing is enabled. diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs index 5f83335d..8e3d49fb 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs @@ -3,7 +3,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics; public sealed class ProfilingInfoMetrics : Dictionary { - internal static ProfilingInfoMetrics FromMetricsDictionary(IDictionary dict) + internal static ProfilingInfoMetrics FromRawMetrics(Dictionary dict) { ArgumentNullException.ThrowIfNull(dict); diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 29d66513..b3eb3394 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -43,7 +43,7 @@ internal void ReleaseAndUpdateExecutionTimer() { if (startExecutionTimestamp.HasValue) { - uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); executionTime += elapsed; endExecutionTime = startExecutionTime.AddTicks(elapsed); @@ -74,7 +74,7 @@ internal override void StartTimer() if (!startExecutionTimestamp.HasValue) { startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = TimerUtils.Now(); + startExecutionTime = new DateTimeOffset(startExecutionTimestamp.Value, TimeSpan.Zero); } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 9fd217fd..a9ba6a52 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -12,9 +12,12 @@ internal sealed class StatementProfiler: ExecutionStatistics internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - private ProfilingInfoMetrics info = []; + //private ProfilingInfoMetrics info = []; private readonly int queryIndex; + private Dictionary rawMetrics = new(); + + internal StatementProfiler(int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) { this.queryIndex = queryIndex; @@ -24,7 +27,7 @@ internal StatementProfiler(int queryIndex, DuckDBNativeConnection duckDBNativeCo internal DateTimeOffset StartTime => startExecutionTime; internal DateTimeOffset EndTime => endExecutionTime; - internal ProfilingInfoMetrics Info => info; + internal ProfilingInfoMetrics Info => ProfilingInfoMetrics.FromRawMetrics(rawMetrics); internal int QueryIndex => queryIndex; @@ -35,7 +38,7 @@ internal ProfilingStatementSummary GetSummary() EndTime: endExecutionTime, ExecutionTimeMilliseconds: TimerUtils.TimerToMilliseconds(executionTime), Order: queryIndex, - Metrics: info); + Metrics: Info); } internal override void StartTimer() @@ -43,7 +46,7 @@ internal override void StartTimer() if (!startExecutionTimestamp.HasValue) { startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = TimerUtils.Now(); + startExecutionTime = new DateTimeOffset(startExecutionTimestamp.Value, TimeSpan.Zero); } } @@ -56,7 +59,7 @@ internal void ReleaseAndUpdateExecutionTimer() { if (startExecutionTimestamp.HasValue) { - uint elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); executionTime += elapsed; endExecutionTime = startExecutionTime.AddTicks(elapsed); @@ -70,7 +73,7 @@ internal override void AcquireProfilingInfo() if (profile.TryPrepare()) { - info = profile.GetMetrics(); + rawMetrics = profile.GetRawMetrics(); } } diff --git a/build/irion.version b/build/irion.version index ad88bf4e..3598ed92 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2-alpha.1 \ No newline at end of file +1.5.2.0-alpha.2 \ No newline at end of file From 187014493656052582406fbb33171f03313efdac Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 20 May 2026 17:04:13 +0200 Subject: [PATCH 17/43] Refactor profiling and tracing system Replaced legacy profiling and tracing classes with a unified `ExecutionProfiler` base class and `IExecutionProfiler` interface, simplifying the design and reducing redundancy. Removed `ExecutionStatistics`, `ExecutionTracer`, and related classes, consolidating their functionality. Updated `QueryProfiler` and `StatementProfiler` to inherit from `ExecutionProfiler`, streamlining their implementation. Removed `QueryProfilerTracer` and `StatementProfilerTracer`, with their responsibilities absorbed into the profilers. Refactored `PreparedStatement` to directly use the new profiling system, improving initialization, timer management, and error handling. Enhanced profiling metrics acquisition and eliminated async context-specific tracing. Improved code readability, maintainability, and performance by consolidating profiling functionality and reducing complexity. Adjusted namespaces to reflect the new structure. --- .../PreparedStatement/PreparedStatement.cs | 94 ++++++++++--------- .../Statistics/ConnectionStatistics.cs | 7 +- .../Profiling/Statistics/ExecutionProfiler.cs | 33 +++++++ .../Statistics/ExecutionStatistics.cs | 7 -- .../Statistics/ExecutionStatisticsBase.cs | 37 -------- .../Profiling/Statistics/ExecutionTracer.cs | 81 ---------------- ...ionTracerBase.cs => IExecutionProfiler.cs} | 4 +- .../Profiling/Statistics/IExecutionTracer.cs | 7 -- .../Profiling/Statistics/QueryProfiler.cs | 34 +++---- .../Statistics/QueryProfilerTracer.cs | 41 -------- .../Profiling/Statistics/StatementProfiler.cs | 27 ++---- .../Statistics/StatementProfilerTracer.cs | 35 ------- 12 files changed, 111 insertions(+), 296 deletions(-) create mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs rename DuckDB.NET.Data/Profiling/Statistics/{IExecutionTracerBase.cs => IExecutionProfiler.cs} (68%) delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs delete mode 100644 DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index f54ed0e1..fcf574aa 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -7,6 +7,7 @@ namespace DuckDB.NET.Data.PreparedStatement; internal sealed class PreparedStatement : IDisposable { private readonly DuckDBPreparedStatement statement; + private StatementProfiler? statementProfiler; private PreparedStatement(DuckDBPreparedStatement statement) { @@ -20,15 +21,15 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c using (extractedStatements) { _ = ConnectionStatistics.TryGetFor(connection, out var stats); - // Initialize the query tracer for the entire batch of statements. The tracer will be responsible for tracking the execution of all statements within this batch. - using var queryTracer = stats?.CreateQueryProfilerTracer(extractedStatements.ToHandle(), statementCount); - queryTracer?.StartTimer(); + // Initialize the query profiler for the entire batch of statements. The profiler will be responsible for tracking the execution of all statements within this batch. + var queryProfiler = stats?.CreateQueryProfiler(extractedStatements.ToHandle(), statementCount); + queryProfiler?.StartTimer(); if (statementCount <= 0) { var error = NativeMethods.ExtractStatements.DuckDBExtractStatementsError(extractedStatements); - queryTracer?.SetState(DuckDBState.Error, DuckDBErrorType.Parser, error); + queryProfiler?.SetState(DuckDBState.Error, DuckDBErrorType.Parser, error); throw new DuckDBException(error); } @@ -37,20 +38,21 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c { var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, index, out var statement); - var statementTracer = queryTracer?.CreateStatementProfilerTracer(statement, index); - StatementProfilerTracer.Current = statementTracer; + var statementProfiler = queryProfiler?.CreateStatementProfiler(index); if (status.IsSuccess()) { using var preparedStatement = new PreparedStatement(statement); + preparedStatement.statementProfiler = statementProfiler; + var result = preparedStatement.Execute(parameters, useStreamingMode, connection); - // Stop the query tracer after the last statement has been executed. + // Stop the query profiler after the last statement has been executed. // This ensures that the total execution time for the entire batch of statements is accurately captured // and not after the data retrieval of the last statement. if (index == statementCount - 1) { - queryTracer?.StopTimer(); + queryProfiler?.StopTimer(); } yield return result; @@ -64,12 +66,10 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c errorMessage = "DuckDBQuery failed"; } - // Initialize the statement tracer for the current statement. This allows for detailed tracing of each individual statement within the batch - using (statementTracer) - { - statementTracer?.SetState(status, errorMessage); - } - StatementProfilerTracer.Current = null; + // Initialize the statement profiler for the current statement. This allows for detailed profiling of each individual statement within the batch + + statementProfiler?.SetState(status, errorMessage); + statementProfiler?.StopTimer(); throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); } @@ -79,45 +79,53 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBNativeConnection connection) { - // Tracing is discriminated by query identifier, which is a combination of the query text and the index of the statement in the case of multiple statements. - // This allows for more granular tracing of individual statements within a batch. - using var statementTracer = StatementProfilerTracer.Current; - statementTracer?.StartTimer(); + // Start the statement profiler timer to measure the execution time of this statement. + // The profiler will also capture any relevant metrics or state changes during execution. + var profiler = this.statementProfiler; + profiler?.StartTimer(); + + try + { - BindParameters(statement, parameterCollection); - var status = useStreamingMode - ? NativeMethods.PreparedStatements.DuckDBExecutePreparedStreaming(statement, out var queryResult) - : NativeMethods.PreparedStatements.DuckDBExecutePrepared(statement, out queryResult); + BindParameters(statement, parameterCollection); - if (!status.IsSuccess()) - { - var errorMessage = NativeMethods.Query.DuckDBResultError(ref queryResult); - var errorType = NativeMethods.Query.DuckDBResultErrorType(ref queryResult); - queryResult.Close(); + var status = useStreamingMode + ? NativeMethods.PreparedStatements.DuckDBExecutePreparedStreaming(statement, out var queryResult) + : NativeMethods.PreparedStatements.DuckDBExecutePrepared(statement, out queryResult); - if (string.IsNullOrEmpty(errorMessage)) + if (!status.IsSuccess()) { - errorMessage = "DuckDB execution failed"; - } + var errorMessage = NativeMethods.Query.DuckDBResultError(ref queryResult); + var errorType = NativeMethods.Query.DuckDBResultErrorType(ref queryResult); + queryResult.Close(); + + if (string.IsNullOrEmpty(errorMessage)) + { + errorMessage = "DuckDB execution failed"; + } - statementTracer?.SetState(status, errorType, errorMessage); + profiler?.SetState(status, errorType, errorMessage); - if (errorType == DuckDBErrorType.Interrupt) - { - throw new OperationCanceledException(); - } + if (errorType == DuckDBErrorType.Interrupt) + { + throw new OperationCanceledException(); + } - var innerException = UdfExceptionStore.Retrieve(connection); - throw innerException != null - ? new DuckDBException(errorMessage, innerException) - : new DuckDBException(errorMessage, errorType); - } + var innerException = UdfExceptionStore.Retrieve(connection); + throw innerException != null + ? new DuckDBException(errorMessage, innerException) + : new DuckDBException(errorMessage, errorType); + } - statementTracer?.AcquireMetrics(); - StatementProfilerTracer.Current = null; + profiler?.AcquireMetrics(); - return queryResult; + return queryResult; + } + finally + { + profiler?.StopTimer(); + } } private static void BindParameters(DuckDBPreparedStatement preparedStatement, DuckDBParameterCollection parameterCollection) diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 9e67898b..1ca6f79e 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -87,7 +87,7 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection /// A pointer that uniquely identifies the query for which the profiler tracer is created. /// The number of statements in the query to be profiled. Must be non-negative. /// A new instance of if query execution tracing is enabled; otherwise, . - internal QueryProfilerTracer? CreateQueryProfilerTracer(IntPtr queryIdentifier, int statementCount) + internal QueryProfiler? CreateQueryProfiler(IntPtr queryIdentifier, int statementCount) { if (!enableQueryExecutionTracing) { @@ -95,11 +95,8 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection } var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); - var tracer = new QueryProfilerTracer(queryProfiler, duckDBNativeConnection); - queryProfilers[queryIdentifier] = queryProfiler; - - return tracer; + return queryProfiler; } internal void UpdateStatistics() diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs new file mode 100644 index 00000000..6d417e5f --- /dev/null +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs @@ -0,0 +1,33 @@ +namespace DuckDB.NET.Data.Profiling.Statistics +{ + // Single base class — replaces ExecutionStatisticsBase + ExecutionStatistics + ExecutionTracer + internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeConnection) : IExecutionProfiler + { + protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; + protected DuckDBState state; + protected string? errorMessage; + private bool disposed; + + public abstract void StartTimer(); + public abstract void StopTimer(); + public virtual void AcquireMetrics() { } + public abstract void Reset(); + + public DuckDBState State => state; + public string ErrorMessage => errorMessage ?? string.Empty; + + public void SetState(DuckDBState state) => this.state = state; + + public void SetState(DuckDBState state, string message) + { + this.state = state; + this.errorMessage = message; + } + + public void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + this.state = state; + this.errorMessage = $"{errorType} error: {message}"; + } + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs deleted file mode 100644 index b17d4e0b..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatistics.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal abstract class ExecutionStatistics(DuckDBNativeConnection duckDBNativeConnection): ExecutionStatisticsBase(duckDBNativeConnection) - { - internal abstract void AcquireProfilingInfo(); - } -} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs deleted file mode 100644 index bb2d3ae3..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionStatisticsBase.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal abstract class ExecutionStatisticsBase(DuckDBNativeConnection duckDBNativeConnection) - { - protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; - - protected DuckDBState state; - protected string? errorMessage; - - internal abstract void Reset(); - - internal abstract void StartTimer(); - - internal abstract void StopTimer(); - - internal virtual void SetState(DuckDBState state) - { - this.state = state; - } - - internal virtual void SetState(DuckDBState state, string message) - { - this.state = state; - this.errorMessage = message; - } - - internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) - { - this.state = state; - this.errorMessage = string.Format("%s error: %s", errorType, message); - } - - internal virtual DuckDBState State => state; - - internal virtual string ErrorMessage => errorMessage ?? string.Empty; - } -} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs deleted file mode 100644 index 852a3c76..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionTracer.cs +++ /dev/null @@ -1,81 +0,0 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal abstract class ExecutionTracer : IExecutionTracerBase, IDisposable - { - protected readonly ExecutionStatisticsBase statistics; - private bool disposed; - - internal ExecutionTracer(ExecutionStatisticsBase statistics) - { - this.statistics = statistics; - } - - internal virtual void StartTimer() - { - statistics.StartTimer(); - } - - internal virtual void StopTimer() - { - statistics.StopTimer(); - } - - internal virtual void Dispose() - { - statistics.StopTimer(); - } - - internal virtual void SetState(DuckDBState state) - { - statistics.SetState(state); - } - - internal virtual void SetState(DuckDBState state, string message) - { - statistics.SetState(state, message); - } - - internal virtual void SetState(DuckDBState state, DuckDBErrorType errorType, string message) - { - statistics.SetState(state, errorType, message); - } - - internal virtual DuckDBState State => statistics.State; - - void IDisposable.Dispose() - { - if (!disposed) - { - Dispose(); - disposed = true; - } - } - - void IExecutionTracerBase.StartTimer() - { - StartTimer(); - } - - void IExecutionTracerBase.StopTimer() - { - StopTimer(); - } - - void IExecutionTracerBase.SetState(DuckDBState state) - { - SetState(state); - } - - void IExecutionTracerBase.SetState(DuckDBState state, string message) - { - SetState(state, message); - } - - void IExecutionTracerBase.SetState(DuckDBState state, DuckDBErrorType errorType, string message) - { - SetState(state, errorType, message); - } - - DuckDBState IExecutionTracerBase.State => State; - } -} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs b/DuckDB.NET.Data/Profiling/Statistics/IExecutionProfiler.cs similarity index 68% rename from DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs rename to DuckDB.NET.Data/Profiling/Statistics/IExecutionProfiler.cs index 5ad59e12..854fca3c 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracerBase.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/IExecutionProfiler.cs @@ -1,12 +1,14 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal interface IExecutionTracerBase : IDisposable + // Single interface — replaces both IExecutionTracerBase and IExecutionProfiler + internal interface IExecutionProfiler { void StartTimer(); void StopTimer(); void SetState(DuckDBState state); void SetState(DuckDBState state, string message); void SetState(DuckDBState state, DuckDBErrorType errorType, string message); + void AcquireMetrics(); DuckDBState State { get; } } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs deleted file mode 100644 index c9aaeede..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/IExecutionTracer.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal interface IExecutionTracer : IExecutionTracerBase - { - void AcquireMetrics(); - } -} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index b3eb3394..9432fa03 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -2,24 +2,16 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class QueryProfiler : ExecutionStatisticsBase + internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection) + : ExecutionProfiler(connection) { - // internal values that are not exposed through properties - internal long? startExecutionTimestamp; - private readonly StatementProfiler[] statementProfilers; + private readonly StatementProfiler[] statementProfilers = new StatementProfiler[statementCount]; - // internal values that are exposed through properties + internal long? startExecutionTimestamp; internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - private readonly IntPtr queryIdentifier; - private readonly int statementCount; - internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection duckDBNativeConnection) : base(duckDBNativeConnection) - { - this.statementCount = statementCount; - this.statementProfilers = new StatementProfiler[statementCount]; - } internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); internal DateTimeOffset StartTime => startExecutionTime; @@ -28,6 +20,12 @@ internal QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeC internal IntPtr QueryIdentifier => queryIdentifier; + internal StatementProfiler CreateStatementProfiler(int index) + { + var profiler = new StatementProfiler(index, duckDBNativeConnection); + statementProfilers[index] = profiler; + return profiler; + } internal bool RegisterStatementProfiler(StatementProfiler statistics, int queryIndex) { @@ -69,7 +67,7 @@ internal ProfilingQuerySummary GetSummary() [.. GetStatementInfo()]); } - internal override void StartTimer() + public override void StartTimer() { if (!startExecutionTimestamp.HasValue) { @@ -78,18 +76,12 @@ internal override void StartTimer() } } - internal override void StopTimer() + public override void StopTimer() { ReleaseAndUpdateExecutionTimer(); } - internal override void SetState(DuckDBState state, string message) - { - this.state = state; - this.errorMessage = message; - } - - internal override void Reset() + public override void Reset() { executionTime = 0; startExecutionTimestamp = null; diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs deleted file mode 100644 index 71220917..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfilerTracer.cs +++ /dev/null @@ -1,41 +0,0 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class QueryProfilerTracer : ExecutionTracer - { - private readonly Dictionary statementTracers = []; - private readonly QueryProfiler queryProfiler; - private readonly DuckDBNativeConnection duckDBNativeConnection; - - internal QueryProfilerTracer(QueryProfiler queryProfiler, DuckDBNativeConnection duckDBNativeConnection) : base(queryProfiler) - { - this.queryProfiler = queryProfiler; - this.duckDBNativeConnection = duckDBNativeConnection; - } - - internal StatementProfilerTracer CreateStatementProfilerTracer(DuckDBPreparedStatement preparedStatement, int statementIndex) - { - if (statementTracers.ContainsKey(preparedStatement)) - { - throw new InvalidOperationException("Statement profiler tracer already exists for the given prepared statement."); - } - - var statementProfiler = new StatementProfiler(statementIndex, duckDBNativeConnection); - statementTracers[preparedStatement] = new StatementProfilerTracer(statementProfiler); - - queryProfiler.RegisterStatementProfiler(statementProfiler, statementIndex); - - return statementTracers[preparedStatement]; - } - - internal StatementProfilerTracer GetOrCreateStatementProfilerTracer(DuckDBPreparedStatement preparedStatement, int? statementIndex = null) - { - if (!statementTracers.TryGetValue(preparedStatement, out var tracer)) - { - var index = statementIndex ?? throw new ArgumentNullException(nameof(statementIndex)); - return CreateStatementProfilerTracer(preparedStatement, index); - } - - return tracer; - } - } -} diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index a9ba6a52..823a1e75 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -3,33 +3,24 @@ namespace DuckDB.NET.Data.Profiling.Statistics { - internal sealed class StatementProfiler: ExecutionStatistics + internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection connection) + : ExecutionProfiler(connection) { - // internal values that are not exposed through properties - internal long? startExecutionTimestamp; + private Dictionary rawMetrics = new(); - // internal values that are exposed through properties + internal long? startExecutionTimestamp; internal long executionTime; internal DateTimeOffset startExecutionTime; internal DateTimeOffset endExecutionTime; - //private ProfilingInfoMetrics info = []; - private readonly int queryIndex; - - private Dictionary rawMetrics = new(); - internal StatementProfiler(int queryIndex, DuckDBNativeConnection duckDBNativeConnection): base(duckDBNativeConnection) - { - this.queryIndex = queryIndex; - } - internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); internal DateTimeOffset StartTime => startExecutionTime; internal DateTimeOffset EndTime => endExecutionTime; + internal int QueryIndex => queryIndex; internal ProfilingInfoMetrics Info => ProfilingInfoMetrics.FromRawMetrics(rawMetrics); - internal int QueryIndex => queryIndex; internal ProfilingStatementSummary GetSummary() { @@ -41,7 +32,7 @@ internal ProfilingStatementSummary GetSummary() Metrics: Info); } - internal override void StartTimer() + public override void StartTimer() { if (!startExecutionTimestamp.HasValue) { @@ -50,7 +41,7 @@ internal override void StartTimer() } } - internal override void StopTimer() + public override void StopTimer() { ReleaseAndUpdateExecutionTimer(); } @@ -67,7 +58,7 @@ internal void ReleaseAndUpdateExecutionTimer() } } - internal override void AcquireProfilingInfo() + public override void AcquireMetrics() { var profile = new ProfilingInfo(duckDBNativeConnection); @@ -77,7 +68,7 @@ internal override void AcquireProfilingInfo() } } - internal override void Reset() + public override void Reset() { executionTime = 0; startExecutionTimestamp = null; diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs deleted file mode 100644 index 391e446b..00000000 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfilerTracer.cs +++ /dev/null @@ -1,35 +0,0 @@ -using DuckDB.NET.Data.Common; -using System.Diagnostics; -using System.Threading; - -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class StatementProfilerTracer : ExecutionTracer, IExecutionTracer - { - private static readonly AsyncLocal currentTracer = new(); - - internal StatementProfilerTracer(StatementProfiler statistics): base(statistics) { } - - /// - /// Gets or sets the current for the current - /// execution context. This flows across async calls. - /// - public static StatementProfilerTracer? Current - { - get => currentTracer.Value; - set => currentTracer.Value = value; - } - - private StatementProfiler StatementProfiler => (StatementProfiler)statistics; - - internal void AcquireMetrics() - { - StatementProfiler.AcquireProfilingInfo(); - } - - void IExecutionTracer.AcquireMetrics() - { - AcquireMetrics(); - } - } -} \ No newline at end of file From 120bc733b2bcbe64f6701a4d893b57854c8bcf68 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 20 May 2026 17:09:10 +0200 Subject: [PATCH 18/43] Bump version to 1.5.2.0-alpha.3 --- build/irion.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/irion.version b/build/irion.version index 3598ed92..6378654d 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.0-alpha.2 \ No newline at end of file +1.5.2.0-alpha.3 \ No newline at end of file From e2f484b12db4d67bd2b2205726678db9139f8758 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 22 May 2026 14:54:14 +0200 Subject: [PATCH 19/43] Refactor profiling and enable XML docs generation Reorganized profiling-related classes into a new namespace `DuckDB.NET.Data.Profiling.Statistics.Summary` for better code structure. Enabled XML documentation file generation in `Bindings.csproj` and `Data.csproj` and suppressed warning 1591 for missing XML comments. Updated NuGet repository URL in `nuget.config` and bumped project version to `1.5.2.0-alpha.5`. No functional changes were made to profiling classes during the namespace move. --- DuckDB.NET.Bindings/Bindings.csproj | 5 +++++ DuckDB.NET.Data/Data.csproj | 5 +++++ DuckDB.NET.Data/DuckDBConnection.cs | 1 + DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs | 1 + DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs | 1 + DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs | 1 + .../Statistics/{ => Summary}/ProfilingQuerySummary.cs | 2 +- .../Statistics/{ => Summary}/ProfilingStatementSummary.cs | 2 +- .../Profiling/Statistics/{ => Summary}/ProfilingSummary.cs | 5 +---- DuckDB.NET.Samples/Program.cs | 1 + build/irion.version | 2 +- nuget.config | 2 +- 12 files changed, 20 insertions(+), 8 deletions(-) rename DuckDB.NET.Data/Profiling/Statistics/{ => Summary}/ProfilingQuerySummary.cs (90%) rename DuckDB.NET.Data/Profiling/Statistics/{ => Summary}/ProfilingStatementSummary.cs (90%) rename DuckDB.NET.Data/Profiling/Statistics/{ => Summary}/ProfilingSummary.cs (87%) diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index 82bad60b..eb7dc5fa 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -18,6 +18,11 @@ enable + + true + $(NoWarn);1591 + + $(Description) $(NoNativeText) diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index 6a8e00e8..8652de08 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -16,6 +16,11 @@ New features: enable + + true + $(NoWarn);1591 + + $(Description) $(NoNativeText) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 193b3a07..d3496744 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -2,6 +2,7 @@ using DuckDB.NET.Data.Connection; using DuckDB.NET.Data.Profiling; using DuckDB.NET.Data.Profiling.Statistics; +using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 1ca6f79e..b6ea459b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -1,5 +1,6 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.PreparedStatement; +using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.Collections.Concurrent; using System.Linq; diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 9432fa03..7a6c0e06 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.Profiling.Statistics.Summary; namespace DuckDB.NET.Data.Profiling.Statistics { diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 823a1e75..87450e2c 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.Diagnostics; namespace DuckDB.NET.Data.Profiling.Statistics diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs similarity index 90% rename from DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs rename to DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs index 65a7cc9d..a98482a7 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingQuerySummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs @@ -1,4 +1,4 @@ -namespace DuckDB.NET.Data.Profiling.Statistics; +namespace DuckDB.NET.Data.Profiling.Statistics.Summary; public readonly record struct ProfilingQuerySummary( DateTimeOffset StartTime, diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs similarity index 90% rename from DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs rename to DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs index 028beae0..4b5327f7 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingStatementSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs @@ -1,4 +1,4 @@ -namespace DuckDB.NET.Data.Profiling.Statistics; +namespace DuckDB.NET.Data.Profiling.Statistics.Summary; public readonly record struct ProfilingStatementSummary( DateTimeOffset StartTime, diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs similarity index 87% rename from DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs rename to DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs index 98d1ba3c..4c79bbed 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; - -namespace DuckDB.NET.Data.Profiling.Statistics; +namespace DuckDB.NET.Data.Profiling.Statistics.Summary; public readonly record struct ProfilingSummary( DateTimeOffset StartTime, diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index bee41ac7..5ff33669 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -2,6 +2,7 @@ using DuckDB.NET.Data; using DuckDB.NET.Data.Profiling; using DuckDB.NET.Data.Profiling.Statistics; +using DuckDB.NET.Data.Profiling.Statistics.Summary; using DuckDB.NET.Native; using DuckDB.NET.Test.Helpers; using System; diff --git a/build/irion.version b/build/irion.version index 6378654d..3d5eccb4 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.0-alpha.3 \ No newline at end of file +1.5.2.0-alpha.5 \ No newline at end of file diff --git a/nuget.config b/nuget.config index 7505d5b5..5f388a93 100644 --- a/nuget.config +++ b/nuget.config @@ -2,6 +2,6 @@ - + \ No newline at end of file From 096db612ce56114984b51efd48cc785a56c60d1a Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 22 May 2026 16:24:07 +0200 Subject: [PATCH 20/43] Improve error handling and add profiling test Enhanced error handling in `DuckDBConnection`: - Retrieve detailed error messages and types on failure. - Throw `OperationCanceledException` for interrupt errors. - Include inner exceptions from `UdfExceptionStore` if available. - Ensure proper cleanup by closing `queryResult`. Added `EnableProfilingWithoutOptionsUsesDefaults` test: - Verifies default behavior when enabling profiling. - Ensures statistics collection and proper cleanup. Updated version to `1.5.2.0-alpha.6`. --- DuckDB.NET.Data/DuckDBConnection.cs | 24 ++++++++- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 55 +++++++++++++++++++++ build/irion.version | 2 +- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index d3496744..aeb91d9f 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -485,9 +485,29 @@ private void LoadStatisticsProfile() var metrics = options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); sb.Append($"SET custom_profiling_settings = '{metrics.ToDuckDBMetricString()}';"); - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sb.ToString(), out _); + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sb.ToString(), out var queryResult); if (!state.IsSuccess()) - throw new DuckDBException("Error configuring profiling settings."); + { + var errorMessage = NativeMethods.Query.DuckDBResultError(ref queryResult); + var errorType = NativeMethods.Query.DuckDBResultErrorType(ref queryResult); + queryResult.Close(); + + if (string.IsNullOrEmpty(errorMessage)) + { + errorMessage = "Error configuring profiling settings"; + } + + if (errorType == DuckDBErrorType.Interrupt) + { + throw new OperationCanceledException(); + } + + var innerException = UdfExceptionStore.Retrieve(NativeConnection); + throw innerException != null + ? new DuckDBException(errorMessage, innerException) + : new DuckDBException(errorMessage, errorType); + } + queryResult.Close(); } } diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 3ba52b28..a602ca42 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -51,6 +51,61 @@ public void MetricsDictionaryContainsEnabledMetrics() } } + [Fact] + public void EnableProfilingWithoutOptionsUsesDefaults() + { + // enable profiling with explicit empty options - should use defaults + var defaultMetrics = DuckDBMetrics.DefaultMetrics.ToList(); + _ = defaultMetrics.Remove(DuckDBMetricType.QueryName); + var options = new ProfilingOptions { Coverage = DuckDBProfilingCoverage.All, EnabledMetrics = new DuckDBMetricTypeCollection(defaultMetrics) }; + + var con = new DuckDBConnection("data source=:memory:"); + con.Open(); + + con.EnableProfiling(options); + + using var cmd = con.CreateCommand(); + + try + { + // ensure clean start + con.ResetStatistics(); + + // run a simple query to produce profiling data + cmd.CommandText = "SELECT 1;"; + using (var r = cmd.ExecuteReader()) { } + + // profiling should be enabled + con.ProfilingEnabled.Should().BeTrue(); + + var summary = con.RetrieveStatistics(); + summary.QueryCount.Should().BeGreaterThan(0); + + + // enable profiling without passing any options - should reset to defaults if already enabled + con.EnableProfiling(); + + con.ResetStatistics(); + + cmd.CommandText = "SET user = 'test';"; + using (var r = cmd.ExecuteReader()) { } + + // profiling should be enabled + con.ProfilingEnabled.Should().BeTrue(); + + summary = con.RetrieveStatistics(); + summary.QueryCount.Should().BeGreaterThan(0); + + // with default options, there should be no metrics collected for Set statements + summary.QuerySummaryList.Last().Infos.Length.Should().Be(1); + summary.QuerySummaryList.Last().Infos[0].Metrics.Count.Should().Be(0); + } + finally + { + con.DisableProfiling(true); + } + } + [Fact] public async Task FileBackedConnectionSharesProfilingStateWhenEnabledOnFirst() { diff --git a/build/irion.version b/build/irion.version index 3d5eccb4..91370f8b 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.0-alpha.5 \ No newline at end of file +1.5.2.0-alpha.6 \ No newline at end of file From cd2447b85cdc8d92d75c5adf48c0e70da03cb25c Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 22 May 2026 16:31:07 +0200 Subject: [PATCH 21/43] Bump version to 1.5.2.2-alpha.6 in README and irion.version --- README.md | 10 +++++----- build/irion.version | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0686d1ad..9d88d5b0 100644 --- a/README.md +++ b/README.md @@ -100,17 +100,17 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2-alpha.1 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-alpha.5 ``` Bump version in `build/irion.version`: ```powershell .\scripts\irion-package.ps1 -Command bump -Part prerelease # 1.4.4-alpha.1 -> 1.4.4-alpha.2 -.\scripts\irion-package.ps1 -Command bump -Part revision # 1.4.4.1 -> 1.4.4.2 -.\scripts\irion-package.ps1 -Command bump -Part build # 1.4.4.1 -> 1.4.5.0 -.\scripts\irion-package.ps1 -Command bump -Part minor # 1.4.4.1 -> 1.5.0.0 -.\scripts\irion-package.ps1 -Command bump -Part major # 1.4.4.1 -> 2.0.0.0 +# .\scripts\irion-package.ps1 -Command bump -Part revision # 1.4.4.1 -> 1.4.4.2 +# .\scripts\irion-package.ps1 -Command bump -Part build # 1.4.4.1 -> 1.4.5.0 +# .\scripts\irion-package.ps1 -Command bump -Part minor # 1.4.4.1 -> 1.5.0.0 +# .\scripts\irion-package.ps1 -Command bump -Part major # 1.4.4.1 -> 2.0.0.0 ``` Check latest published versions on the configured feed: diff --git a/build/irion.version b/build/irion.version index 91370f8b..0e3c9793 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.0-alpha.6 \ No newline at end of file +1.5.2.2-alpha.6 \ No newline at end of file From e22ba0e58c81d22cb253e974e0708c754889bd9a Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 25 May 2026 14:46:52 +0200 Subject: [PATCH 22/43] Refactor and enhance profiling and execution tracing Refactored `TimerUtils`, `ConnectionStatistics`, and `ExecutionProfiler` to improve readability, maintainability, and extensibility. Enhanced `PreparedStatement`, `QueryProfiler`, and `StatementProfiler` with better timer handling, state tracking, and error message reporting. Added `State` and `Message` fields to `ProfilingQuerySummary` and `ProfilingStatementSummary` for improved diagnostics. Introduced new profiling tests to validate success and error scenarios. Updated version to `1.5.2.2-beta.1` and adjusted documentation in `README.md` to reflect the new release. --- DuckDB.NET.Data/Common/TimerUtils.cs | 69 +++-- .../PreparedStatement/PreparedStatement.cs | 4 +- .../Statistics/ConnectionStatistics.cs | 258 +++++++++--------- .../Profiling/Statistics/ExecutionProfiler.cs | 119 ++++++-- .../Profiling/Statistics/QueryProfiler.cs | 128 ++++----- .../Profiling/Statistics/StatementProfiler.cs | 108 +++----- .../Summary/ProfilingQuerySummary.cs | 8 +- .../Summary/ProfilingStatementSummary.cs | 8 +- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 89 ++++++ README.md | 2 +- build/irion.version | 2 +- 11 files changed, 463 insertions(+), 332 deletions(-) diff --git a/DuckDB.NET.Data/Common/TimerUtils.cs b/DuckDB.NET.Data/Common/TimerUtils.cs index 018b316a..30bb2221 100644 --- a/DuckDB.NET.Data/Common/TimerUtils.cs +++ b/DuckDB.NET.Data/Common/TimerUtils.cs @@ -1,37 +1,48 @@ using System.Diagnostics; -namespace DuckDB.NET.Data.Common +namespace DuckDB.NET.Data.Common; + +internal sealed class TimerUtils { - internal sealed class TimerUtils - { - private static readonly long Frequency = Stopwatch.Frequency; + private static readonly long Frequency = Stopwatch.Frequency; - /// - /// Gets the current timestamp value for high-resolution timing operations. - /// - /// The returned value can be used with Stopwatch.Frequency to calculate elapsed time intervals with high - /// precision. This method is intended for scenarios where precise timing is required, such as performance - /// measurements. - /// A long integer representing the current timestamp, as provided by the system's high-resolution performance counter. - internal static long TimerCurrent() => Stopwatch.GetTimestamp(); + /// + /// Gets the current timestamp value for high-resolution timing operations. + /// + /// The returned value can be used with Stopwatch.Frequency to calculate elapsed time intervals with high + /// precision. This method is intended for scenarios where precise timing is required, such as performance + /// measurements. + /// A long integer representing the current timestamp, as provided by the system's high-resolution performance counter. + internal static long TimerCurrent() => Stopwatch.GetTimestamp(); - /// - /// Converts a timer value to its equivalent duration in milliseconds. - /// - /// The conversion uses the timer's frequency to determine the number of milliseconds represented by the - /// timer value. Ensure that the timer value and frequency are based on the same timer source. - /// The timer value to convert, typically representing elapsed timer ticks. - /// The equivalent duration in milliseconds calculated from the specified timer value. - internal static long TimerToMilliseconds(long timerValue) - => timerValue * 1000 / Frequency; + /// + /// Converts a timer value to its equivalent duration in milliseconds. + /// + /// The conversion uses the timer's frequency to determine the number of milliseconds represented by the + /// timer value. Ensure that the timer value and frequency are based on the same timer source. + /// The timer value to convert, typically representing elapsed timer ticks. + /// The equivalent duration in milliseconds calculated from the specified timer value. + internal static long TimerToMilliseconds(long timerValue) + => timerValue * 1000 / Frequency; - /// - /// Calculates the elapsed number of ticks between two tick count values. - /// - /// The starting tick count value. - /// The ending tick count value. - /// The difference between the ending and starting tick count values, representing the elapsed ticks. - internal static long CalculateTickCountElapsed(long startTick, long endTick) - => endTick - startTick; // keep as long, no truncation + /// + /// Converts a timer tick duration (as returned by ) to a . + /// + /// The timer tick duration to convert. + /// A representing the duration of the provided timer ticks. + internal static TimeSpan TimerToTimeSpan(long timerValue) + { + // Convert stopwatch ticks to DateTime/TimeSpan ticks. TimeSpan.TicksPerSecond = 10_000_000. + long timeSpanTicks = timerValue * TimeSpan.TicksPerSecond / Frequency; + return TimeSpan.FromTicks(timeSpanTicks); } + + /// + /// Calculates the elapsed number of ticks between two tick count values. + /// + /// The starting tick count value. + /// The ending tick count value. + /// The difference between the ending and starting tick count values, representing the elapsed ticks. + internal static long CalculateTickCountElapsed(long startTick, long endTick) + => endTick - startTick; // keep as long, no truncation } diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index fcf574aa..e97c2ec7 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -30,6 +30,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c var error = NativeMethods.ExtractStatements.DuckDBExtractStatementsError(extractedStatements); queryProfiler?.SetState(DuckDBState.Error, DuckDBErrorType.Parser, error); + queryProfiler?.StopTimer(); throw new DuckDBException(error); } @@ -69,7 +70,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c // Initialize the statement profiler for the current statement. This allows for detailed profiling of each individual statement within the batch statementProfiler?.SetState(status, errorMessage); - statementProfiler?.StopTimer(); + queryProfiler?.StopTimer(index); throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); } @@ -87,7 +88,6 @@ private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool try { - BindParameters(statement, parameterCollection); var status = useStreamingMode diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index b6ea459b..ab36a6b5 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -1,162 +1,160 @@ using DuckDB.NET.Data.Common; -using DuckDB.NET.Data.PreparedStatement; using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.Collections.Concurrent; using System.Linq; -namespace DuckDB.NET.Data.Profiling.Statistics +namespace DuckDB.NET.Data.Profiling.Statistics; + +internal sealed class ConnectionStatistics : IDisposable { - internal sealed class ConnectionStatistics : IDisposable + + // internal values that are not exposed through properties + private static readonly ConcurrentDictionary ByNativeConnection += new(ReferenceEqualityComparer.Instance); + + + internal long closeTimestamp; + internal long openTimestamp; + internal long? startExecutionTimestamp; + private bool enableQueryExecutionTracing; + private readonly DuckDBNativeConnection duckDBNativeConnection; + private readonly ConcurrentDictionary queryProfilers = new(); + private bool isDisposed = false; + + // internal values that are exposed through properties + internal long executionTime; + internal long connectionTime; + internal DateTimeOffset startExecutionTime; + internal DateTimeOffset endExecutionTime; + + /// + /// Initializes a new instance of the class. + /// + /// The native DuckDB connection. + /// Indicates whether query execution tracing is enabled. + internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) { + this.enableQueryExecutionTracing = enableQueryExecutionTracing; + this.duckDBNativeConnection = duckDBNativeConnection; - // internal values that are not exposed through properties - private static readonly ConcurrentDictionary ByNativeConnection - = new(ReferenceEqualityComparer.Instance); - - - internal long closeTimestamp; - internal long openTimestamp; - internal long? startExecutionTimestamp; - private bool enableQueryExecutionTracing; - private readonly DuckDBNativeConnection duckDBNativeConnection; - private readonly ConcurrentDictionary queryProfilers = new(); - private bool isDisposed = false; - - // internal values that are exposed through properties - internal long executionTime; - internal long connectionTime; - internal DateTimeOffset startExecutionTime; - internal DateTimeOffset endExecutionTime; - - /// - /// Initializes a new instance of the class. - /// - /// The native DuckDB connection. - /// Indicates whether query execution tracing is enabled. - internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) - { - this.enableQueryExecutionTracing = enableQueryExecutionTracing; - this.duckDBNativeConnection = duckDBNativeConnection; + // Add or update the mapping for the native connection to this ConnectionStatistics instance + ByNativeConnection.AddOrUpdate(duckDBNativeConnection, this, (_, _) => this); + } - // Add or update the mapping for the native connection to this ConnectionStatistics instance - ByNativeConnection.AddOrUpdate(duckDBNativeConnection, this, (_, _) => this); - } + /// + /// Enables tracing of query execution for diagnostic or debugging purposes. + /// + /// After calling this method, additional diagnostic information about query execution may be collected + /// or logged. This can assist in troubleshooting or performance analysis. Tracing remains enabled until explicitly + /// disabled + internal void EnableQueryExecutionTracing() + { + this.enableQueryExecutionTracing = true; + } - /// - /// Enables tracing of query execution for diagnostic or debugging purposes. - /// - /// After calling this method, additional diagnostic information about query execution may be collected - /// or logged. This can assist in troubleshooting or performance analysis. Tracing remains enabled until explicitly - /// disabled - internal void EnableQueryExecutionTracing() - { - this.enableQueryExecutionTracing = true; - } + /// + /// Disables tracing of query execution for the current instance. + /// + /// After calling this method, query execution tracing will no longer be performed until explicitly + /// re-enabled. This may affect the ability to diagnose or audit query behavior. + internal void DisableQueryExecutionTracing() + { + this.enableQueryExecutionTracing = false; + } - /// - /// Disables tracing of query execution for the current instance. - /// - /// After calling this method, query execution tracing will no longer be performed until explicitly - /// re-enabled. This may affect the ability to diagnose or audit query behavior. - internal void DisableQueryExecutionTracing() + /// + /// Attempts to retrieve a associated with the specified connection. + /// + /// The native DuckDB connection for which to obtain statistics. + /// When this method returns, contains the statistics for the specified connection if found; otherwise, . + /// if statistics are found for the specified connection; otherwise, . + internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) + { + // Fast path: if no connections have profiling enabled, skip the lookup entirely + if (ByNativeConnection.IsEmpty) { - this.enableQueryExecutionTracing = false; + stats = null; + return false; } + return ByNativeConnection.TryGetValue(nativeConn, out stats); + } - /// - /// Attempts to retrieve a associated with the specified connection. - /// - /// The native DuckDB connection for which to obtain statistics. - /// When this method returns, contains the statistics for the specified connection if found; otherwise, . - /// if statistics are found for the specified connection; otherwise, . - internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out ConnectionStatistics? stats) + /// + /// Creates a new for the specified query if query execution tracing is enabled. + /// + /// A pointer that uniquely identifies the query for which the profiler tracer is created. + /// The number of statements in the query to be profiled. Must be non-negative. + /// A new instance of if query execution tracing is enabled; otherwise, . + internal QueryProfiler? CreateQueryProfiler(IntPtr queryIdentifier, int statementCount) + { + if (!enableQueryExecutionTracing) { - // Fast path: if no connections have profiling enabled, skip the lookup entirely - if (ByNativeConnection.IsEmpty) - { - stats = null; - return false; - } - return ByNativeConnection.TryGetValue(nativeConn, out stats); + return null; } - /// - /// Creates a new for the specified query if query execution tracing is enabled. - /// - /// A pointer that uniquely identifies the query for which the profiler tracer is created. - /// The number of statements in the query to be profiled. Must be non-negative. - /// A new instance of if query execution tracing is enabled; otherwise, . - internal QueryProfiler? CreateQueryProfiler(IntPtr queryIdentifier, int statementCount) - { - if (!enableQueryExecutionTracing) - { - return null; - } - - var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); - queryProfilers[queryIdentifier] = queryProfiler; - return queryProfiler; - } + var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); + queryProfilers[queryIdentifier] = queryProfiler; + return queryProfiler; + } - internal void UpdateStatistics() + internal void UpdateStatistics() + { + // update connection time + if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) { - // update connection time - if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) - { - connectionTime = closeTimestamp - openTimestamp; - } - else - { - connectionTime = long.MaxValue; - } + connectionTime = closeTimestamp - openTimestamp; } - - private IEnumerable ReadSummaries() + else { - foreach (var queryProfiler in queryProfilers.Values) - { - yield return queryProfiler.GetSummary(); - } + connectionTime = long.MaxValue; } + } - internal ProfilingSummary GetProfilingSummary() + private IEnumerable ReadSummaries() + { + foreach (var queryProfiler in queryProfilers.Values) { - return new ProfilingSummary( - new DateTimeOffset(openTimestamp, TimeSpan.Zero), - new DateTimeOffset(connectionTime, TimeSpan.Zero), - TimerUtils.TimerToMilliseconds(connectionTime), - TimerUtils.TimerToMilliseconds(queryProfilers.Values.Sum(qp => qp.executionTime)), - queryProfilers.Values.Count, - [.. ReadSummaries()]); + yield return queryProfiler.GetSummary(); } + } - internal void Reset() - { - executionTime = 0; - startExecutionTimestamp = null; - connectionTime = 0; - startExecutionTime = default; - endExecutionTime = default; - openTimestamp = 0; - - queryProfilers.Clear(); - } + internal ProfilingSummary GetProfilingSummary() + { + return new ProfilingSummary( + new DateTimeOffset(openTimestamp, TimeSpan.Zero), + new DateTimeOffset(connectionTime, TimeSpan.Zero), + TimerUtils.TimerToMilliseconds(connectionTime), + TimerUtils.TimerToMilliseconds(queryProfilers.Values.Sum(qp => qp.ExecutionTime)), + queryProfilers.Values.Count, + [.. ReadSummaries()]); + } - // call on connection close/dispose - private void RemoveMapping() - { - ByNativeConnection.TryRemove(duckDBNativeConnection, out _); - } + internal void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + connectionTime = 0; + startExecutionTime = default; + endExecutionTime = default; + openTimestamp = 0; + + queryProfilers.Clear(); + } - public void Dispose() + // call on connection close/dispose + private void RemoveMapping() + { + ByNativeConnection.TryRemove(duckDBNativeConnection, out _); + } + + public void Dispose() + { + if (!isDisposed) { - if (!isDisposed) - { - RemoveMapping(); - isDisposed = true; - } + RemoveMapping(); + isDisposed = true; } } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs index 6d417e5f..ac7bd345 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs @@ -1,33 +1,108 @@ -namespace DuckDB.NET.Data.Profiling.Statistics -{ - // Single base class — replaces ExecutionStatisticsBase + ExecutionStatistics + ExecutionTracer - internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeConnection) : IExecutionProfiler - { - protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; - protected DuckDBState state; - protected string? errorMessage; - private bool disposed; +using DuckDB.NET.Data.Common; + +namespace DuckDB.NET.Data.Profiling.Statistics; - public abstract void StartTimer(); - public abstract void StopTimer(); - public virtual void AcquireMetrics() { } - public abstract void Reset(); +internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeConnection) : IExecutionProfiler +{ + protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; + protected DuckDBState state; + protected string? errorMessage; - public DuckDBState State => state; - public string ErrorMessage => errorMessage ?? string.Empty; + protected long? startExecutionTimestamp; + protected long executionTime; + protected DateTimeOffset startExecutionTime; + protected DateTimeOffset endExecutionTime; - public void SetState(DuckDBState state) => this.state = state; + internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); + internal DateTimeOffset StartTime => startExecutionTime; + internal DateTimeOffset EndTime => endExecutionTime; - public void SetState(DuckDBState state, string message) + /// + /// Starts the timer, initiating the timing operation. + /// + public virtual void StartTimer() + { + if (!startExecutionTimestamp.HasValue) { - this.state = state; - this.errorMessage = message; + startExecutionTimestamp = TimerUtils.TimerCurrent(); + startExecutionTime = DateTimeOffset.UtcNow; } + } - public void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + /// + /// Stops the currently running timer, if active. + /// + /// Call this method to halt the timer and record its elapsed time. If the timer is not running, calling + /// this method has no effect. + public virtual void StopTimer() + { + ReleaseAndUpdateExecutionTimer(); + } + + /// + /// Releases the current execution timer and updates the accumulated execution time and end time for the operation. + /// + protected virtual void ReleaseAndUpdateExecutionTimer() + { + if (startExecutionTimestamp.HasValue) { - this.state = state; - this.errorMessage = $"{errorType} error: {message}"; + long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + executionTime += elapsed; + + // Convert the elapsed high-resolution ticks to a TimeSpan and apply to the wall-clock start time. + var elapsedSpan = TimerUtils.TimerToTimeSpan(elapsed); + endExecutionTime = startExecutionTime.Add(elapsedSpan); + + startExecutionTimestamp = null; } } + + /// + /// Collects and updates performance or usage metrics for the current instance. + /// + public virtual void AcquireMetrics() { } + + /// + /// Resets the object to its initial state. + /// + public abstract void Reset(); + + /// + /// Gets the current state of the current operation, indicating success or error + /// + public DuckDBState State => state; + + /// + /// Gets the error message associated with the current operation. + /// + public string ErrorMessage => errorMessage ?? string.Empty; + + /// + /// Sets the current state of the DuckDB connection or operation. + /// + /// The new state to assign to the DuckDB instance. + public void SetState(DuckDBState state) => this.state = state; + + /// + /// Sets the current state and associated error message for the DuckDB instance. + /// + /// The new state to assign to the DuckDB instance. + /// The error message to associate with the specified state. Can be null or empty if no error message is required. + public void SetState(DuckDBState state, string message) + { + this.state = state; + this.errorMessage = message; + } + + /// + /// Sets the current state and error information for the DuckDB connection or operation. + /// + /// The new state to assign to the DuckDB connection or operation. + /// The type of error associated with the state change. Used to categorize the error message. + /// A descriptive message providing details about the error or state change. Cannot be null. + public void SetState(DuckDBState state, DuckDBErrorType errorType, string message) + { + this.state = state; + this.errorMessage = $"{errorType} error: {message}"; + } } \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 7a6c0e06..4705197d 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -1,93 +1,71 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.Profiling.Statistics.Summary; +using System.Linq; -namespace DuckDB.NET.Data.Profiling.Statistics -{ - internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection) - : ExecutionProfiler(connection) - { - private readonly StatementProfiler[] statementProfilers = new StatementProfiler[statementCount]; +namespace DuckDB.NET.Data.Profiling.Statistics; - internal long? startExecutionTimestamp; - internal long executionTime; - internal DateTimeOffset startExecutionTime; - internal DateTimeOffset endExecutionTime; +internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection) +: ExecutionProfiler(connection) +{ + private readonly StatementProfiler[] statementProfilers = new StatementProfiler[statementCount]; - internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); - internal DateTimeOffset StartTime => startExecutionTime; - internal DateTimeOffset EndTime => endExecutionTime; - internal int StatementCount => statementCount; + internal int StatementCount => statementCount; - internal IntPtr QueryIdentifier => queryIdentifier; + internal IntPtr QueryIdentifier => queryIdentifier; - internal StatementProfiler CreateStatementProfiler(int index) - { - var profiler = new StatementProfiler(index, duckDBNativeConnection); - statementProfilers[index] = profiler; - return profiler; - } - - internal bool RegisterStatementProfiler(StatementProfiler statistics, int queryIndex) - { - if (statementProfilers[queryIndex] == null) - { - statementProfilers[queryIndex] = statistics; - return true; - } - return false; - } - - internal void ReleaseAndUpdateExecutionTimer() - { - if (startExecutionTimestamp.HasValue) - { - long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); - executionTime += elapsed; - endExecutionTime = startExecutionTime.AddTicks(elapsed); + internal StatementProfiler CreateStatementProfiler(int index) + { + var profiler = new StatementProfiler(index, duckDBNativeConnection); + statementProfilers[index] = profiler; + return profiler; + } - startExecutionTimestamp = null; - } - } + /// + /// Creates a summary of the profiling query, including execution times, statement information, and state. + /// + /// A object containing details about the query's execution, statements, state, and + /// any error message. + internal ProfilingQuerySummary GetSummary() + { + var childWithError = statementProfilers.Where(sp => sp?.State == DuckDBState.Error).FirstOrDefault(); - private IEnumerable GetStatementInfo() - { - foreach (var statementProfiler in statementProfilers) - { - yield return statementProfiler.GetSummary(); - } - } + state = state == DuckDBState.Error ? state : (childWithError?.State ?? DuckDBState.Success); + errorMessage = !string.IsNullOrEmpty(ErrorMessage) ? ErrorMessage : (childWithError?.ErrorMessage ?? string.Empty); - internal ProfilingQuerySummary GetSummary() - { - return new ProfilingQuerySummary( - startExecutionTime, - endExecutionTime, - TimerUtils.TimerToMilliseconds(executionTime), - statementCount, - [.. GetStatementInfo()]); - } + return new ProfilingQuerySummary( + startExecutionTime, + endExecutionTime, + TimerUtils.TimerToMilliseconds(executionTime), + statementCount, + [.. statementProfilers.Select(sp => sp.GetSummary())], + state, + errorMessage + ); + } - public override void StartTimer() + /// + /// Stops the timer associated with the specified statement index. + /// + /// The zero-based index of the statement whose timer should be stopped. + /// Thrown if statementIndex is less than zero or greater than the highest valid statement profiler index. + public void StopTimer(int statementIndex) + { + if (statementIndex < 0 || statementIndex > statementProfilers.Length - 1) { - if (!startExecutionTimestamp.HasValue) - { - startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = new DateTimeOffset(startExecutionTimestamp.Value, TimeSpan.Zero); - } + throw new ArgumentOutOfRangeException(nameof(statementIndex), $"Index {statementIndex} is out of range for statement profilers."); } + statementProfilers[statementIndex]?.StopTimer(); - public override void StopTimer() - { - ReleaseAndUpdateExecutionTimer(); - } + StopTimer(); + } - public override void Reset() - { - executionTime = 0; - startExecutionTimestamp = null; - startExecutionTime = default; - endExecutionTime = default; - } + /// + public override void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + startExecutionTime = default; + endExecutionTime = default; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 87450e2c..efc475b1 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -1,80 +1,52 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.Profiling.Statistics.Summary; -using System.Diagnostics; -namespace DuckDB.NET.Data.Profiling.Statistics +namespace DuckDB.NET.Data.Profiling.Statistics; + +internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection connection) +: ExecutionProfiler(connection) { - internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection connection) - : ExecutionProfiler(connection) + private Dictionary rawMetrics = []; + + /// + /// Gets the profiling metrics associated with the current operation. + /// + internal ProfilingInfoMetrics Info => ProfilingInfoMetrics.FromRawMetrics(rawMetrics); + + /// + /// Creates a summary of the profiling statement, including execution timing, order, metrics, state, and any associated + /// message. + /// + /// A instance containing the collected profiling data for the statement. + internal ProfilingStatementSummary GetSummary() { - private Dictionary rawMetrics = new(); - - internal long? startExecutionTimestamp; - internal long executionTime; - internal DateTimeOffset startExecutionTime; - internal DateTimeOffset endExecutionTime; - - - internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); - internal DateTimeOffset StartTime => startExecutionTime; - internal DateTimeOffset EndTime => endExecutionTime; - - internal int QueryIndex => queryIndex; - internal ProfilingInfoMetrics Info => ProfilingInfoMetrics.FromRawMetrics(rawMetrics); - - - internal ProfilingStatementSummary GetSummary() - { - return new ProfilingStatementSummary( - StartTime: startExecutionTime, - EndTime: endExecutionTime, - ExecutionTimeMilliseconds: TimerUtils.TimerToMilliseconds(executionTime), - Order: queryIndex, - Metrics: Info); - } - - public override void StartTimer() - { - if (!startExecutionTimestamp.HasValue) - { - startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = new DateTimeOffset(startExecutionTimestamp.Value, TimeSpan.Zero); - } - } - - public override void StopTimer() - { - ReleaseAndUpdateExecutionTimer(); - } - - internal void ReleaseAndUpdateExecutionTimer() - { - if (startExecutionTimestamp.HasValue) - { - long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); - executionTime += elapsed; - endExecutionTime = startExecutionTime.AddTicks(elapsed); + return new ProfilingStatementSummary( + StartTime: startExecutionTime, + EndTime: endExecutionTime, + ExecutionTimeMilliseconds: TimerUtils.TimerToMilliseconds(executionTime), + Order: queryIndex, + Metrics: Info, + State: state, + Message: ErrorMessage); + } - startExecutionTimestamp = null; - } - } + /// + public override void AcquireMetrics() + { + var profile = new ProfilingInfo(duckDBNativeConnection); - public override void AcquireMetrics() + if (profile.TryPrepare()) { - var profile = new ProfilingInfo(duckDBNativeConnection); - - if (profile.TryPrepare()) - { - rawMetrics = profile.GetRawMetrics(); - } + rawMetrics = profile.GetRawMetrics(); } + } - public override void Reset() - { - executionTime = 0; - startExecutionTimestamp = null; - startExecutionTime = default; - endExecutionTime = default; - } + /// + public override void Reset() + { + executionTime = 0; + startExecutionTimestamp = null; + startExecutionTime = default; + endExecutionTime = default; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs index a98482a7..1421d4de 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs @@ -5,7 +5,9 @@ public readonly record struct ProfilingQuerySummary( DateTimeOffset EndTime, double ExecutionTimeMilliseconds, int StatementCount, - ProfilingStatementSummary[] Infos) + ProfilingStatementSummary[] Infos, + DuckDBState State, + string Message) { public IDictionary ToDictionary() { @@ -15,7 +17,9 @@ public IDictionary ToDictionary() ["EndTime"] = EndTime, ["ExecutionTime"] = ExecutionTimeMilliseconds, ["StatementCount"] = StatementCount, - ["Infos"] = Infos + ["Infos"] = Infos, + ["State"] = State, + ["Message"] = Message }; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs index 4b5327f7..bfb32ce8 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs @@ -5,7 +5,9 @@ public readonly record struct ProfilingStatementSummary( DateTimeOffset EndTime, double ExecutionTimeMilliseconds, int Order, - ProfilingInfoMetrics Metrics) + ProfilingInfoMetrics Metrics, + DuckDBState State, + string Message) { public IDictionary ToDictionary() { @@ -15,7 +17,9 @@ public IDictionary ToDictionary() ["EndTime"] = EndTime, ["ExecutionTime"] = ExecutionTimeMilliseconds, ["Order"] = Order, - ["Infos"] = Metrics + ["Infos"] = Metrics, + ["State"] = State, + ["Message"] = Message }; } } diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index a602ca42..824aebfe 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -1,11 +1,100 @@ using DuckDB.NET.Data.Profiling; using DuckDB.NET.Test.Helpers; +using DuckDB.NET.Native; namespace DuckDB.NET.Test.Profiling; public class ProfilingTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) { + [Fact] + public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + // Expect overall query to be successful + last.State.Should().Be(DuckDBState.Success); + last.Message.Should().BeNullOrEmpty(); + + // Expect constituent statements to be successful as well + if (last.Infos.Length > 0) + { + var stmt = last.Infos.Last(); + stmt.State.Should().Be(DuckDBState.Success); + stmt.Message.Should().BeNullOrEmpty(); + } + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void ProfilingSummariesContainCorrectStateAndMessageOnError() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + // Execute a statement that will error + Command.CommandText = "SELECT * FROM __this_table_does_not_exist__"; + try + { + using (var r = Command.ExecuteReader()) { } + } + catch (Exception) + { + // swallow - we expect an error to be thrown by the command execution + } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + // Overall query should be marked as error + last.State.Should().Be(DuckDBState.Error); + last.Message.Should().NotBeNullOrWhiteSpace(); + + // Ensure at least one statement reports error state/message + var anyError = last.Infos.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); + anyError.Should().BeTrue("At least one statement should report an error state and message"); + } + finally + { + Connection.DisableProfiling(); + } + } + [Fact] public void MetricsDictionaryContainsEnabledMetrics() { diff --git a/README.md b/README.md index 9d88d5b0..5acfd884 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-alpha.5 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-beta.1 ``` Bump version in `build/irion.version`: diff --git a/build/irion.version b/build/irion.version index 0e3c9793..3e6f5d7b 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-alpha.6 \ No newline at end of file +1.5.2.2-beta.1 \ No newline at end of file From fd36e5ce1a04aa63604f9b46c0a90d78648edfc5 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 27 May 2026 11:26:04 +0200 Subject: [PATCH 23/43] Add advanced profiling options and metrics threshold Enhanced profiling capabilities with new `ProfilingOptions`: - Added properties for format, mode, coverage, output path, enabled metrics, and `MetricsThresholdMS` for conditional metrics collection based on execution time. - Updated `ConnectionStatistics`, `ExecutionProfiler`, `QueryProfiler`, and `StatementProfiler` to support `ProfilingOptions` and respect the metrics threshold. Added new tests in `ProfilingTests` to validate behavior: - Metrics are collected when execution exceeds the threshold. - Metrics are skipped when execution is below the threshold. Bumped version to `1.5.2.2-beta.2`. --- DuckDB.NET.Data/DuckDBConnection.cs | 2 +- DuckDB.NET.Data/Profiling/ProfilingOptions.cs | 23 ++++++ .../Statistics/ConnectionStatistics.cs | 7 +- .../Profiling/Statistics/ExecutionProfiler.cs | 4 +- .../Profiling/Statistics/QueryProfiler.cs | 6 +- .../Profiling/Statistics/StatementProfiler.cs | 9 ++- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 75 +++++++++++++++++++ build/irion.version | 2 +- 8 files changed, 118 insertions(+), 10 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index aeb91d9f..f3dae961 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -416,7 +416,7 @@ private void InitProfiling() if (ProfilingEnabled) { - profilingInfo = new ConnectionStatistics(NativeConnection, isProfilingEnabled); + profilingInfo = new ConnectionStatistics(NativeConnection, isProfilingEnabled, this.profilingOptions); LoadStatisticsProfile(); } } diff --git a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs index 8e934fb8..033e6d37 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs @@ -2,11 +2,34 @@ { public readonly struct ProfilingOptions { + /// + /// The profiling output format to use when generating profiling results. + /// public DuckDBProfilingFormat Format { get; init; } + + /// + /// The profiling mode used for DuckDB query execution. + /// public DuckDBProfilingMode Mode { get; init; } + + /// + /// The profiling coverage settings used for DuckDB query analysis. + /// public DuckDBProfilingCoverage Coverage { get; init; } + + /// + /// The output directory path where generated files will be placed. + /// public string OutputPath { get; init; } + /// + /// The collection of enabled metrics for profiling. + /// public DuckDBMetricTypeCollection EnabledMetrics { get; init; } + + /// + /// The threshold value used for metrics acquisition, in milliseconds. + /// + public int MetricsThresholdMS { get; init; } = 0; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index ab36a6b5..4a86a62b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -18,6 +18,7 @@ private static readonly ConcurrentDictionary queryProfilers = new(); private bool isDisposed = false; @@ -32,10 +33,12 @@ private static readonly ConcurrentDictionary /// The native DuckDB connection. /// Indicates whether query execution tracing is enabled. - internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false) + /// The profiling options for the connection. + internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false, ProfilingOptions? profilingOptions = default) { this.enableQueryExecutionTracing = enableQueryExecutionTracing; this.duckDBNativeConnection = duckDBNativeConnection; + this.profilingOptions = profilingOptions; // Add or update the mapping for the native connection to this ConnectionStatistics instance ByNativeConnection.AddOrUpdate(duckDBNativeConnection, this, (_, _) => this); @@ -94,7 +97,7 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection return null; } - var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection); + var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection, profilingOptions); queryProfilers[queryIdentifier] = queryProfiler; return queryProfiler; } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs index ac7bd345..b87025a2 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs @@ -2,9 +2,11 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeConnection) : IExecutionProfiler +internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeConnection, ProfilingOptions? profilingOptions) : IExecutionProfiler { protected readonly DuckDBNativeConnection duckDBNativeConnection = duckDBNativeConnection; + protected readonly ProfilingOptions? profilingOptions = profilingOptions; + protected DuckDBState state; protected string? errorMessage; diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 4705197d..348264a9 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -4,8 +4,8 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection) -: ExecutionProfiler(connection) +internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection, ProfilingOptions? profilingOptions) +: ExecutionProfiler(connection, profilingOptions) { private readonly StatementProfiler[] statementProfilers = new StatementProfiler[statementCount]; @@ -16,7 +16,7 @@ internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, internal StatementProfiler CreateStatementProfiler(int index) { - var profiler = new StatementProfiler(index, duckDBNativeConnection); + var profiler = new StatementProfiler(index, duckDBNativeConnection, profilingOptions); statementProfilers[index] = profiler; return profiler; } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index efc475b1..60d0420e 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -3,8 +3,8 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection connection) -: ExecutionProfiler(connection) +internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection connection, ProfilingOptions? profilingOptions) +: ExecutionProfiler(connection, profilingOptions) { private Dictionary rawMetrics = []; @@ -33,6 +33,11 @@ internal ProfilingStatementSummary GetSummary() /// public override void AcquireMetrics() { + long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp ?? 0, TimerUtils.TimerCurrent()); + + if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThresholdMS ?? 0)) + return; + var profile = new ProfilingInfo(duckDBNativeConnection); if (profile.TryPrepare()) diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 824aebfe..14922140 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -49,6 +49,81 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() } } + [Fact] + public void MetricsCollectedWhenOverMetricsThreshold() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName, DuckDBMetricType.CpuTime }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + MetricsThresholdMS = 1 // very small threshold so a heavy query will exceed it + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + // run a heavier query to exceed the threshold + // use DuckDB's range table function with a single argument (0..n-1) + Command.CommandText = "SELECT SUM(i) FROM range(1000000) AS t(i);"; + using (var r = Command.ExecuteReader()) { } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + last.Infos.Should().NotBeNull(); + + // When execution is over the metrics threshold, at least one statement should contain enabled metrics + var hasMetrics = last.Infos.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.QueryName) || i.Metrics.ContainsKey(DuckDBMetricType.CpuTime)); + hasMetrics.Should().BeTrue("Metrics should be collected when execution time exceeds MetricsThresholdMS"); + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void MetricsNotCollectedWhenUnderMetricsThreshold() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + MetricsThresholdMS = 10000 // high threshold to prevent metrics collection for a short query + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + last.Infos.Should().NotBeNull(); + + // When execution is under the metrics threshold, no metrics should be extracted + last.Infos.All(i => i.Metrics.Count == 0).Should().BeTrue("Metrics should not be collected when execution time is below the configured MetricsThresholdMS"); + } + finally + { + Connection.DisableProfiling(); + } + } + [Fact] public void ProfilingSummariesContainCorrectStateAndMessageOnError() { diff --git a/build/irion.version b/build/irion.version index 3e6f5d7b..9953bafa 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-beta.1 \ No newline at end of file +1.5.2.2-beta.2 \ No newline at end of file From 2fb41dc10635180f04ac206484d098c8a61d58d3 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 27 May 2026 11:27:53 +0200 Subject: [PATCH 24/43] removed default from ProfilingOptions.MetricsThresholdMS --- DuckDB.NET.Data/Profiling/ProfilingOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs index 033e6d37..b5575d29 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs @@ -30,6 +30,6 @@ public readonly struct ProfilingOptions /// /// The threshold value used for metrics acquisition, in milliseconds. /// - public int MetricsThresholdMS { get; init; } = 0; + public int MetricsThresholdMS { get; init; } } } From dd87b1380d7795b37b8cab8f5794adcb13b5d985 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Wed, 27 May 2026 17:55:01 +0200 Subject: [PATCH 25/43] Add metrics threshold check to StatementProfiler The `StatementProfiler` class was updated to include a conditional check for a metrics threshold before acquiring metrics. If `profilingOptions?.MetricsThresholdMS` is set, the elapsed execution time is calculated, and metrics are only acquired if the execution time meets or exceeds the threshold. This prevents unnecessary metric acquisition for short executions. Additionally, the version number in `irion.version` was updated from `1.5.2.2-beta.2` to `1.5.2.2-beta.3` to reflect the new beta release. --- .../Profiling/Statistics/StatementProfiler.cs | 10 +++++++--- build/irion.version | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 60d0420e..99e7ac43 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -33,10 +33,14 @@ internal ProfilingStatementSummary GetSummary() /// public override void AcquireMetrics() { - long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp ?? 0, TimerUtils.TimerCurrent()); + // If a metrics threshold is set, check if the elapsed execution time meets the threshold before acquiring metrics. + if ((profilingOptions?.MetricsThresholdMS ?? 0) > 0) + { + long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp ?? 0, TimerUtils.TimerCurrent()); - if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThresholdMS ?? 0)) - return; + if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThresholdMS ?? 0)) + return; + } var profile = new ProfilingInfo(duckDBNativeConnection); diff --git a/build/irion.version b/build/irion.version index 9953bafa..12a23287 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-beta.2 \ No newline at end of file +1.5.2.2-beta.3 \ No newline at end of file From 3130829ca4c17fa8cd567db3e1a895881820602e Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Thu, 28 May 2026 09:22:09 +0200 Subject: [PATCH 26/43] Refactor profiling system in DuckDBConnection Refactored `profilingInfo` to `statistics` for improved naming consistency and alignment with the updated profiling architecture. Introduced the `ExecutionProfiler` base class to centralize common profiling logic, including timer management and timestamp handling. Updated `ConnectionStatistics`, `QueryProfiler`, and `StatementProfiler` to inherit from `ExecutionProfiler`, simplifying code and reducing duplication. Renamed `MetricsThresholdMS` to `MetricsThreshold` in `ProfilingOptions` for clarity. Updated related test cases and sample code to reflect the new profiling architecture. Bumped version to `1.5.2.2-beta.4`. --- DuckDB.NET.Data/DuckDBConnection.cs | 25 +++++------ DuckDB.NET.Data/Profiling/ProfilingOptions.cs | 5 ++- .../Statistics/ConnectionStatistics.cs | 42 +++++++------------ .../Profiling/Statistics/ExecutionProfiler.cs | 26 ++++++------ .../Profiling/Statistics/QueryProfiler.cs | 10 ++--- .../Profiling/Statistics/StatementProfiler.cs | 16 +++---- DuckDB.NET.Samples/Program.cs | 3 +- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 4 +- build/irion.version | 2 +- 9 files changed, 62 insertions(+), 71 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index f3dae961..6c4f496a 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -21,7 +21,7 @@ public partial class DuckDBConnection : DbConnection private static readonly StateChangeEventArgs FromOpenToClosedEventArgs = new(ConnectionState.Open, ConnectionState.Closed); // Statistics support - internal ConnectionStatistics? profilingInfo; + private ConnectionStatistics? statistics; private bool isProfilingEnabled; private ProfilingOptions? profilingOptions; @@ -278,7 +278,7 @@ protected override void Dispose(bool disposing) Close(); } - profilingInfo?.Dispose(); + statistics?.Dispose(); } base.Dispose(disposing); @@ -340,10 +340,10 @@ public DuckDBQueryProgress GetQueryProgress() /// A containing the collected statistics. public ProfilingSummary RetrieveStatistics() { - if (profilingInfo != null) + if (statistics != null) { UpdateStatistics(); - return profilingInfo.GetProfilingSummary(); + return statistics.GetProfilingSummary(); } else { @@ -380,7 +380,7 @@ public void EnableProfiling(ProfilingOptions? options = null) public void DisableProfiling(bool resetStatistics = false) { // stop - profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); + statistics?.StopTimer(); DisableProfiling(); if (resetStatistics) @@ -400,7 +400,7 @@ public void ResetStatistics() { if (ProfilingEnabled) { - profilingInfo?.Reset(); + statistics?.Reset(); } } @@ -416,7 +416,7 @@ private void InitProfiling() if (ProfilingEnabled) { - profilingInfo = new ConnectionStatistics(NativeConnection, isProfilingEnabled, this.profilingOptions); + statistics = new ConnectionStatistics(NativeConnection, isProfilingEnabled, this.profilingOptions); LoadStatisticsProfile(); } } @@ -434,7 +434,7 @@ public void EditProfilingOptions(ProfilingOptions options) } this.profilingOptions = options; - profilingInfo?.Reset(); + statistics?.Reset(); LoadStatisticsProfile(); } @@ -455,7 +455,7 @@ private void DisableProfiling() } } - profilingInfo?.DisableQueryExecutionTracing(); + statistics?.DisableQueryExecutionTracing(); isProfilingEnabled = false; } @@ -470,7 +470,7 @@ private void DisableProfiling() /// Thrown if an error occurs while setting profiling configuration options. private void LoadStatisticsProfile() { - profilingInfo?.openTimestamp = TimerUtils.TimerCurrent(); + statistics?.StartTimer(); if (ProfilingEnabled && profilingOptions is { } options) { @@ -516,10 +516,11 @@ private void UpdateStatistics() if (ConnectionState.Open == State) { // update timestamp - profilingInfo?.closeTimestamp = TimerUtils.TimerCurrent(); + statistics?.StopTimer(); } + // delegate the rest of the work to the SqlStatistics class - profilingInfo?.UpdateStatistics(); + statistics?.UpdateStatistics(); } #endregion } diff --git a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs index b5575d29..4eaa8b8d 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs @@ -28,8 +28,9 @@ public readonly struct ProfilingOptions public DuckDBMetricTypeCollection EnabledMetrics { get; init; } /// - /// The threshold value used for metrics acquisition, in milliseconds. + /// The threshold (in milliseconds) used for metrics acquisition. /// - public int MetricsThresholdMS { get; init; } + /// Execution times below this threshold may not trigger metrics collection. + public int MetricsThreshold { get; init; } } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 4a86a62b..b8a4686d 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -5,28 +5,18 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal sealed class ConnectionStatistics : IDisposable +internal sealed class ConnectionStatistics : ExecutionProfiler { // internal values that are not exposed through properties - private static readonly ConcurrentDictionary ByNativeConnection -= new(ReferenceEqualityComparer.Instance); + private static readonly ConcurrentDictionary ByNativeConnection = new(); - - internal long closeTimestamp; - internal long openTimestamp; - internal long? startExecutionTimestamp; private bool enableQueryExecutionTracing; - private readonly DuckDBNativeConnection duckDBNativeConnection; - private readonly ProfilingOptions? profilingOptions; private readonly ConcurrentDictionary queryProfilers = new(); private bool isDisposed = false; // internal values that are exposed through properties - internal long executionTime; internal long connectionTime; - internal DateTimeOffset startExecutionTime; - internal DateTimeOffset endExecutionTime; /// /// Initializes a new instance of the class. @@ -35,10 +25,9 @@ private static readonly ConcurrentDictionaryIndicates whether query execution tracing is enabled. /// The profiling options for the connection. internal ConnectionStatistics(DuckDBNativeConnection duckDBNativeConnection, bool enableQueryExecutionTracing = false, ProfilingOptions? profilingOptions = default) + : base(duckDBNativeConnection, profilingOptions) { this.enableQueryExecutionTracing = enableQueryExecutionTracing; - this.duckDBNativeConnection = duckDBNativeConnection; - this.profilingOptions = profilingOptions; // Add or update the mapping for the native connection to this ConnectionStatistics instance ByNativeConnection.AddOrUpdate(duckDBNativeConnection, this, (_, _) => this); @@ -85,11 +74,11 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection } /// - /// Creates a new for the specified query if query execution tracing is enabled. + /// Creates a new for the specified query if query execution tracing is enabled. /// - /// A pointer that uniquely identifies the query for which the profiler tracer is created. + /// A pointer that uniquely identifies the query for which the profiler is created. /// The number of statements in the query to be profiled. Must be non-negative. - /// A new instance of if query execution tracing is enabled; otherwise, . + /// A new instance of if query execution tracing is enabled; otherwise, . internal QueryProfiler? CreateQueryProfiler(IntPtr queryIdentifier, int statementCount) { if (!enableQueryExecutionTracing) @@ -105,9 +94,9 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection internal void UpdateStatistics() { // update connection time - if (closeTimestamp >= openTimestamp && long.MaxValue > closeTimestamp - openTimestamp) + if (endTime >= startTime && long.MaxValue > (endTime - startTime).Ticks) { - connectionTime = closeTimestamp - openTimestamp; + connectionTime = (endTime - startTime).Ticks; } else { @@ -117,7 +106,7 @@ internal void UpdateStatistics() private IEnumerable ReadSummaries() { - foreach (var queryProfiler in queryProfilers.Values) + foreach (var queryProfiler in queryProfilers.Values.OrderBy(qp => qp.StartTime)) { yield return queryProfiler.GetSummary(); } @@ -126,22 +115,21 @@ private IEnumerable ReadSummaries() internal ProfilingSummary GetProfilingSummary() { return new ProfilingSummary( - new DateTimeOffset(openTimestamp, TimeSpan.Zero), - new DateTimeOffset(connectionTime, TimeSpan.Zero), + startTime, + endTime, TimerUtils.TimerToMilliseconds(connectionTime), TimerUtils.TimerToMilliseconds(queryProfilers.Values.Sum(qp => qp.ExecutionTime)), queryProfilers.Values.Count, [.. ReadSummaries()]); } - internal void Reset() + public override void Reset() { executionTime = 0; - startExecutionTimestamp = null; + startTimestamp = null; connectionTime = 0; - startExecutionTime = default; - endExecutionTime = default; - openTimestamp = 0; + startTime = default; + endTime = default; queryProfilers.Clear(); } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs index b87025a2..78639d16 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs @@ -10,24 +10,24 @@ internal abstract class ExecutionProfiler(DuckDBNativeConnection duckDBNativeCon protected DuckDBState state; protected string? errorMessage; - protected long? startExecutionTimestamp; + protected long? startTimestamp; protected long executionTime; - protected DateTimeOffset startExecutionTime; - protected DateTimeOffset endExecutionTime; + protected DateTimeOffset startTime; + protected DateTimeOffset endTime; internal long ExecutionTime => TimerUtils.TimerToMilliseconds(executionTime); - internal DateTimeOffset StartTime => startExecutionTime; - internal DateTimeOffset EndTime => endExecutionTime; + internal DateTimeOffset StartTime => startTime; + internal DateTimeOffset EndTime => endTime; /// /// Starts the timer, initiating the timing operation. /// public virtual void StartTimer() { - if (!startExecutionTimestamp.HasValue) + if (!startTimestamp.HasValue) { - startExecutionTimestamp = TimerUtils.TimerCurrent(); - startExecutionTime = DateTimeOffset.UtcNow; + startTimestamp = TimerUtils.TimerCurrent(); + startTime = DateTimeOffset.UtcNow; } } @@ -46,16 +46,16 @@ public virtual void StopTimer() /// protected virtual void ReleaseAndUpdateExecutionTimer() { - if (startExecutionTimestamp.HasValue) + if (startTimestamp.HasValue) { - long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp.Value, TimerUtils.TimerCurrent()); + long elapsed = TimerUtils.CalculateTickCountElapsed(startTimestamp.Value, TimerUtils.TimerCurrent()); executionTime += elapsed; // Convert the elapsed high-resolution ticks to a TimeSpan and apply to the wall-clock start time. var elapsedSpan = TimerUtils.TimerToTimeSpan(elapsed); - endExecutionTime = startExecutionTime.Add(elapsedSpan); + endTime = startTime.Add(elapsedSpan); - startExecutionTimestamp = null; + startTimestamp = null; } } @@ -67,7 +67,7 @@ public virtual void AcquireMetrics() { } /// /// Resets the object to its initial state. /// - public abstract void Reset(); + public virtual void Reset() { } /// /// Gets the current state of the current operation, indicating success or error diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 348264a9..aee5f6cc 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -34,8 +34,8 @@ internal ProfilingQuerySummary GetSummary() errorMessage = !string.IsNullOrEmpty(ErrorMessage) ? ErrorMessage : (childWithError?.ErrorMessage ?? string.Empty); return new ProfilingQuerySummary( - startExecutionTime, - endExecutionTime, + startTime, + endTime, TimerUtils.TimerToMilliseconds(executionTime), statementCount, [.. statementProfilers.Select(sp => sp.GetSummary())], @@ -64,8 +64,8 @@ public void StopTimer(int statementIndex) public override void Reset() { executionTime = 0; - startExecutionTimestamp = null; - startExecutionTime = default; - endExecutionTime = default; + startTimestamp = null; + startTime = default; + endTime = default; } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 99e7ac43..ac51936f 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -21,8 +21,8 @@ internal sealed class StatementProfiler(int queryIndex, DuckDBNativeConnection c internal ProfilingStatementSummary GetSummary() { return new ProfilingStatementSummary( - StartTime: startExecutionTime, - EndTime: endExecutionTime, + StartTime: startTime, + EndTime: endTime, ExecutionTimeMilliseconds: TimerUtils.TimerToMilliseconds(executionTime), Order: queryIndex, Metrics: Info, @@ -34,11 +34,11 @@ internal ProfilingStatementSummary GetSummary() public override void AcquireMetrics() { // If a metrics threshold is set, check if the elapsed execution time meets the threshold before acquiring metrics. - if ((profilingOptions?.MetricsThresholdMS ?? 0) > 0) + if ((profilingOptions?.MetricsThreshold ?? 0) > 0) { - long elapsed = TimerUtils.CalculateTickCountElapsed(startExecutionTimestamp ?? 0, TimerUtils.TimerCurrent()); + long elapsed = TimerUtils.CalculateTickCountElapsed(startTimestamp ?? 0, TimerUtils.TimerCurrent()); - if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThresholdMS ?? 0)) + if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThreshold ?? 0)) return; } @@ -54,8 +54,8 @@ public override void AcquireMetrics() public override void Reset() { executionTime = 0; - startExecutionTimestamp = null; - startExecutionTime = default; - endExecutionTime = default; + startTimestamp = null; + startTime = default; + endTime = default; } } diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 5ff33669..414a188a 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -269,7 +269,8 @@ private static void Profiling() DuckDBMetricType.SystemPeakBufferMemory, DuckDBMetricType.SystemPeakTempDirSize, DuckDBMetricType.WriteToWalLatency, - ] + ], + MetricsThreshold = 100 }; con.Open(); diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 14922140..e0f85b7a 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -58,7 +58,7 @@ public void MetricsCollectedWhenOverMetricsThreshold() EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName, DuckDBMetricType.CpuTime }, Format = DuckDBProfilingFormat.Json, Mode = DuckDBProfilingMode.Standard, - MetricsThresholdMS = 1 // very small threshold so a heavy query will exceed it + MetricsThreshold = 1 // very small threshold so a heavy query will exceed it }; Connection.EnableProfiling(options); @@ -97,7 +97,7 @@ public void MetricsNotCollectedWhenUnderMetricsThreshold() EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, Format = DuckDBProfilingFormat.Json, Mode = DuckDBProfilingMode.Standard, - MetricsThresholdMS = 10000 // high threshold to prevent metrics collection for a short query + MetricsThreshold = 10000 // high threshold to prevent metrics collection for a short query }; Connection.EnableProfiling(options); diff --git a/build/irion.version b/build/irion.version index 12a23287..88b06027 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-beta.3 \ No newline at end of file +1.5.2.2-beta.4 \ No newline at end of file From 12af03b5642d957a76bbe64cccb8003787a3e41d Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 1 Jun 2026 09:21:51 +0200 Subject: [PATCH 27/43] Update profiling modes and version for release candidate Updated the `DuckDBProfilingMode` enumeration in `DuckDBNativeObjects.cs`: - Added a new profiling mode, `All`. Updated the version in `irion.version` from `1.5.2.2-beta.4` to `1.5.2.2-rc.1`, marking the transition from beta to release candidate. --- DuckDB.NET.Bindings/DuckDBNativeObjects.cs | 3 ++- build/irion.version | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DuckDB.NET.Bindings/DuckDBNativeObjects.cs b/DuckDB.NET.Bindings/DuckDBNativeObjects.cs index 66a71de9..6ce5311a 100644 --- a/DuckDB.NET.Bindings/DuckDBNativeObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBNativeObjects.cs @@ -342,7 +342,8 @@ public enum DuckDBProfilingFormat public enum DuckDBProfilingMode { Standard, - Detailed + Detailed, + All } public enum DuckDBProfilingCoverage diff --git a/build/irion.version b/build/irion.version index 88b06027..4d42b2c6 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-beta.4 \ No newline at end of file +1.5.2.2-rc.1 \ No newline at end of file From 2ecc98495067d782c66e76a602f4923aac0d0466 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 1 Jun 2026 09:49:51 +0200 Subject: [PATCH 28/43] Refactor profiling logic in DuckDBConnection Updated `DisableProfiling` and `EnableProfiling` methods to use `CALL` SQL procedures for enabling/disabling profiling with named parameters, replacing `PRAGMA` commands. Modified `GetProfilingInfo` to return a nullable wrapper for better error handling. Removed `GetMetrics` and `PrintProfilingNode` methods from `ProfilingInfo` to simplify the class. Updated `GetRawMetrics` to return an empty list when no metrics are available. Performed general cleanup by removing unused or commented-out code related to profiling metrics and node printing. --- DuckDB.NET.Data/DuckDBConnection.cs | 18 ++++--- .../Profiling/DuckDBProfilingInfoWrapper.cs | 2 +- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 53 +------------------ 3 files changed, 12 insertions(+), 61 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 6c4f496a..e57597eb 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -448,7 +448,7 @@ private void DisableProfiling() { if (ProfilingEnabled && State == ConnectionState.Open) { - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, "PRAGMA disable_profiling; PRAGMA disable_profile;", out _); + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, "CALL disable_profiling();", out _); if (!state.IsSuccess()) { throw new DuckDBException("Error disabling profiling."); @@ -475,15 +475,17 @@ private void LoadStatisticsProfile() if (ProfilingEnabled && profilingOptions is { } options) { var sb = new StringBuilder(); - sb.Append($"SET enable_profiling = '{options.Format.ToDuckDBProfilingFormatString()}';"); - sb.Append($"SET profiling_coverage = '{options.Coverage}';"); - sb.Append($"SET profiling_mode = '{options.Mode}';"); - + + sb.AppendLine("CALL enable_profiling("); + sb.AppendLine($" format := '{options.Format.ToDuckDBProfilingFormatString()}',"); + sb.AppendLine($" coverage := '{options.Coverage}',"); + sb.AppendLine($" mode := '{options.Mode}',"); + if (!string.IsNullOrEmpty(options.OutputPath)) - sb.Append($"SET profiling_output = '{options.OutputPath}';"); + sb.AppendLine($" save_location := '{options.OutputPath}',"); - var metrics = options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics); - sb.Append($"SET custom_profiling_settings = '{metrics.ToDuckDBMetricString()}';"); + sb.AppendLine($" metrics := '{(options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics)).ToDuckDBMetricString()}'"); + sb.AppendLine(");"); var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sb.ToString(), out var queryResult); if (!state.IsSuccess()) diff --git a/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs index c112a5b7..d6e4d97a 100644 --- a/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs +++ b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs @@ -12,7 +12,7 @@ public DuckDBProfilingInfoWrapper(DuckDBProfilingInfo handle) this.handle = handle; } - public static DuckDBProfilingInfoWrapper GetProfilingInfo(DuckDBNativeConnection connection) + public static DuckDBProfilingInfoWrapper? GetProfilingInfo(DuckDBNativeConnection connection) { ArgumentNullException.ThrowIfNull(connection); diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 024b38a8..276073d4 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -24,22 +24,6 @@ internal bool TryPrepare() return true; } - internal ProfilingInfoMetrics GetMetrics() - { - if (duckDBProfilingInfoWrapper == null) - { - throw new InvalidOperationException("Profiling info is not prepared. Call Prepare() first."); - } - - var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); - if (metricsValue.IsNull()) - { - return []; - } - - return ProfilingInfoMetrics.FromRawMetrics(metricsValue.GetMapValue()); - } - internal Dictionary GetRawMetrics() { if (duckDBProfilingInfoWrapper == null) @@ -49,46 +33,11 @@ internal Dictionary GetRawMetrics() var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); if (metricsValue.IsNull()) { - return new Dictionary(); + return []; } return metricsValue.GetMapValue(); } - private static void PrintProfilingNode(DuckDBProfilingInfoWrapper node, int indent) - { - // Example: print the 'name' and 'extra_info' keys if present - var queryNameValue = node.GetValue("QUERY_NAME"); - Console.WriteLine($"{new string(' ', indent * 2)}QueryName: {queryNameValue.GetValue()}"); - var nameValue = node.GetValue("CPU_TIME"); - ////var extraInfoValue = node.GetValue("extra_info"); - - Console.WriteLine($"{new string(' ', indent * 2)}CpuTime: {nameValue.GetValue()}"); - //if (!extraInfoValue.IsNull()) - //{ - // Console.WriteLine($"{new string(' ', indent * 2 + 2)}Extra: {extraInfoValue.GetValue()}"); - //} - - // Print metrics if needed - var metrics = node.GetMetrics(); - if (!metrics.IsNull()) - { - var dic = metrics.GetMapValue(); - Console.WriteLine($"{new string(' ', indent * 2 + 2)}Metrics:"); - foreach (var kvp in dic) - { - Console.WriteLine($"{new string(' ', indent * 3 + 2)} {kvp.Key}:{kvp.Value}"); - } - } - - //// Recurse into children - //var childCount = node.GetChildCount(); - //for (ulong i = 0; i < childCount; i++) - //{ - // using var child = node.GetChild(i); - // PrintProfilingNode(child, indent + 1); - //} - } - public void Dispose() { if (!isDisposed) From a361c1f559c921b17356c2e484003ffcd635ee38 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 1 Jun 2026 09:53:41 +0200 Subject: [PATCH 29/43] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5acfd884..f170b620 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-beta.1 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.1 ``` Bump version in `build/irion.version`: From fc507224fb1476afdbbbf7212518142c77c8d8d9 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 09:33:42 +0200 Subject: [PATCH 30/43] removed trailing whitespace --- DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs | 2 +- DuckDB.NET.Data/Connection/FileReference.cs | 2 +- DuckDB.NET.Data/DuckDBConnection.cs | 2 +- DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs | 2 +- DuckDB.NET.Data/Profiling/ProfilingOptions.cs | 2 +- DuckDB.NET.Samples/Program.cs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs index 25b9cda5..df14d9e3 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Value.cs @@ -272,7 +272,7 @@ public static DuckDBValue DuckDBCreateMapValue(DuckDBLogicalType logicalType, Du { var duckDBValue = DuckDBCreateMapValue(logicalType, keys.Select(item => item.DangerousGetHandle()).ToArray(), values.Select(item => item.DangerousGetHandle()).ToArray(), count); - duckDBValue.SetChildValues(values); + duckDBValue.SetChildValues([.. keys, .. values]); return duckDBValue; } diff --git a/DuckDB.NET.Data/Connection/FileReference.cs b/DuckDB.NET.Data/Connection/FileReference.cs index 2c5b4450..541e7553 100644 --- a/DuckDB.NET.Data/Connection/FileReference.cs +++ b/DuckDB.NET.Data/Connection/FileReference.cs @@ -15,7 +15,7 @@ internal class FileReference(string filename) public long ConnectionCount { get; private set; } //don't need a long, but it is slightly faster on 64 bit systems /// - /// Gets a value indicating whether profiling is enabled for the current context. + /// Gets a value indicating whether profiling is enabled for the current context. /// Is used to ensure that the DuckDBDatabase is created with the correct profiling mode when a connection is opened on the same database /// public bool IsProfilingEnabled { get; internal set; } diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index e57597eb..d858549b 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -334,7 +334,7 @@ public DuckDBQueryProgress GetQueryProgress() public bool ProfilingEnabled => isProfilingEnabled; /// - /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. + /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. /// If profiling is not enabled, this method will return an empty summary. /// /// A containing the collected statistics. diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index e97c2ec7..5b310c2b 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -49,7 +49,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c var result = preparedStatement.Execute(parameters, useStreamingMode, connection); // Stop the query profiler after the last statement has been executed. - // This ensures that the total execution time for the entire batch of statements is accurately captured + // This ensures that the total execution time for the entire batch of statements is accurately captured // and not after the data retrieval of the last statement. if (index == statementCount - 1) { diff --git a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs index 4eaa8b8d..36dbef1b 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingOptions.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingOptions.cs @@ -28,7 +28,7 @@ public readonly struct ProfilingOptions public DuckDBMetricTypeCollection EnabledMetrics { get; init; } /// - /// The threshold (in milliseconds) used for metrics acquisition. + /// The threshold (in milliseconds) used for metrics acquisition. /// /// Execution times below this threshold may not trigger metrics collection. public int MetricsThreshold { get; init; } diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 414a188a..8eb64411 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -345,7 +345,7 @@ private static string PrintStatementMetrics(ProfilingInfoMetrics[] profilingSumm private static string PrintStatementMetrics(ProfilingInfoMetrics profilingSummary) { - return string.Join("", profilingSummary.ToDictionary().ToList().Select(kv => $"\n\t\t{kv.Key}: {kv.Value}")); + return string.Join("", profilingSummary.ToDictionary().ToList().Select(kv => $"\n\t\t{kv.Key}: {kv.Value}")); } private static void PrintQueryResults(DbDataReader queryResult) From 30f81a0ef82f2deb6341caec3072cca1ef01589a Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 09:35:41 +0200 Subject: [PATCH 31/43] removed trailing whitespace --- Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj index 039fac69..c7efbb18 100644 --- a/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj +++ b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj @@ -3,7 +3,6 @@ net10.0;net8 Exe - Irion.DuckDB.NET.Benchmark Irion.DuckDB.NET.Benchmark Irion.DuckDB.NET.Benchmark From 60ea2f8406c4266b2fb9fe2c487a2e846daa020c Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 11:57:33 +0200 Subject: [PATCH 32/43] Refactor query profiling to use integer-based IDs Replaced pointer-based query identifiers with thread-safe, integer-based IDs in `ConnectionStatistics`. Updated the `CreateQueryProfiler` method in both `PreparedStatement` and `ConnectionStatistics` to reflect this change. Introduced a new `nextQueryId` field with `Interlocked.Increment` to ensure thread-safe ID generation. Added the `GetNextQueryId` method, optimized with `[MethodImpl(MethodImplOptions.AggressiveInlining)]`. Updated `QueryProfiler` creation logic to use the new integer IDs and adjusted the `queryProfilers` dictionary accordingly. Simplified the `CreateQueryProfiler` method signature by removing the `queryIdentifier` parameter. --- .../PreparedStatement/PreparedStatement.cs | 2 +- .../Statistics/ConnectionStatistics.cs | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 5b310c2b..6d151cb0 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -22,7 +22,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c { _ = ConnectionStatistics.TryGetFor(connection, out var stats); // Initialize the query profiler for the entire batch of statements. The profiler will be responsible for tracking the execution of all statements within this batch. - var queryProfiler = stats?.CreateQueryProfiler(extractedStatements.ToHandle(), statementCount); + var queryProfiler = stats?.CreateQueryProfiler(statementCount); queryProfiler?.StartTimer(); if (statementCount <= 0) diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index b8a4686d..151253ac 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -2,6 +2,8 @@ using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.Collections.Concurrent; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; namespace DuckDB.NET.Data.Profiling.Statistics; @@ -14,6 +16,7 @@ internal sealed class ConnectionStatistics : ExecutionProfiler private bool enableQueryExecutionTracing; private readonly ConcurrentDictionary queryProfilers = new(); private bool isDisposed = false; + private int nextQueryId = 0; // internal values that are exposed through properties internal long connectionTime; @@ -76,21 +79,28 @@ internal static bool TryGetFor(DuckDBNativeConnection nativeConn, out Connection /// /// Creates a new for the specified query if query execution tracing is enabled. /// - /// A pointer that uniquely identifies the query for which the profiler is created. /// The number of statements in the query to be profiled. Must be non-negative. /// A new instance of if query execution tracing is enabled; otherwise, . - internal QueryProfiler? CreateQueryProfiler(IntPtr queryIdentifier, int statementCount) + internal QueryProfiler? CreateQueryProfiler(int statementCount) { if (!enableQueryExecutionTracing) { return null; } - var queryProfiler = new QueryProfiler(queryIdentifier, statementCount, duckDBNativeConnection, profilingOptions); - queryProfilers[queryIdentifier] = queryProfiler; + var id = GetNextQueryId(); + + var queryProfiler = new QueryProfiler(id, statementCount, duckDBNativeConnection, profilingOptions); + queryProfilers[id] = queryProfiler; return queryProfiler; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetNextQueryId() + { + return Interlocked.Increment(ref nextQueryId); + } + internal void UpdateStatistics() { // update connection time From 740e80fbcfd2559e0f05390c5557b82b84d98d79 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 11:59:12 +0200 Subject: [PATCH 33/43] Refactor queryProfilers key type to use int Changed the key type of the `queryProfilers` dictionary from `IntPtr` to `int` to simplify query profiler identification and improve consistency with other system components. --- DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 151253ac..9514a6d7 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -14,7 +14,7 @@ internal sealed class ConnectionStatistics : ExecutionProfiler private static readonly ConcurrentDictionary ByNativeConnection = new(); private bool enableQueryExecutionTracing; - private readonly ConcurrentDictionary queryProfilers = new(); + private readonly ConcurrentDictionary queryProfilers = new(); private bool isDisposed = false; private int nextQueryId = 0; From 5ac6ff6250662394c5b045ee56e7cbab9969a2e9 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 12:33:30 +0200 Subject: [PATCH 34/43] Refactor profiling and improve error handling Refactored `PreparedStatement` to enhance error handling and ensure proper resource cleanup with a `try-finally` block. Updated `ConnectionStatistics` and `QueryProfiler` to improve timer management and profiling accuracy. Renamed `Infos` to `StatementSummaries` and `Metrics` for clarity and consistency. Enhanced `ProfilingTests` with new test cases for query and statement failure scenarios. Updated existing tests to validate the new profiling structure and ensure correctness of metrics, error states, and timing data. Improved profiling logic for accurate timing and error reporting at both query and statement levels. Cleaned up code, removed redundant comments, and ensured consistent naming conventions for better readability and maintainability. --- .../PreparedStatement/PreparedStatement.cs | 63 ++++--- .../Statistics/ConnectionStatistics.cs | 2 + .../Profiling/Statistics/QueryProfiler.cs | 14 +- .../Summary/ProfilingQuerySummary.cs | 6 +- .../Summary/ProfilingStatementSummary.cs | 4 +- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 171 +++++++++++++++--- 6 files changed, 195 insertions(+), 65 deletions(-) diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 6d151cb0..55f48c85 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,5 +1,6 @@ using DuckDB.NET.Data.Connection; using DuckDB.NET.Data.Profiling.Statistics; +using Microsoft.VisualBasic; using System.Linq; namespace DuckDB.NET.Data.PreparedStatement; @@ -35,45 +36,51 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c throw new DuckDBException(error); } - for (int index = 0; index < statementCount; index++) - { - var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, index, out var statement); - - var statementProfiler = queryProfiler?.CreateStatementProfiler(index); + try { - if (status.IsSuccess()) + for (int index = 0; index < statementCount; index++) { - using var preparedStatement = new PreparedStatement(statement); - preparedStatement.statementProfiler = statementProfiler; + var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, index, out var statement); - var result = preparedStatement.Execute(parameters, useStreamingMode, connection); + var statementProfiler = queryProfiler?.CreateStatementProfiler(index); - // Stop the query profiler after the last statement has been executed. - // This ensures that the total execution time for the entire batch of statements is accurately captured - // and not after the data retrieval of the last statement. - if (index == statementCount - 1) + if (status.IsSuccess()) { - queryProfiler?.StopTimer(); - } + using var preparedStatement = new PreparedStatement(statement); + preparedStatement.statementProfiler = statementProfiler; - yield return result; - } - else - { - var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(statement); + var result = preparedStatement.Execute(parameters, useStreamingMode, connection); - if (string.IsNullOrEmpty(errorMessage)) - { - errorMessage = "DuckDBQuery failed"; + // Stop the query profiler after the last statement has been executed. + // This ensures that the total execution time for the entire batch of statements is accurately captured + // and not after the data retrieval of the last statement. + if (index == statementCount - 1) + { + queryProfiler?.StopTimer(); + } + + yield return result; } + else + { + var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(statement); - // Initialize the statement profiler for the current statement. This allows for detailed profiling of each individual statement within the batch + if (string.IsNullOrEmpty(errorMessage)) + { + errorMessage = "DuckDBQuery failed"; + } - statementProfiler?.SetState(status, errorMessage); - queryProfiler?.StopTimer(index); + // Initialize the statement profiler for the current statement. This allows for detailed profiling of each individual statement within the batch + statementProfiler?.SetState(status, errorMessage); - throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); - } + throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); + } + } + } + finally + { + // Ensure that the query profiler is stopped in case of any exceptions during statement preparation or execution. + queryProfiler?.StopTimer(); } } } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 9514a6d7..94638832 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -142,6 +142,8 @@ public override void Reset() endTime = default; queryProfilers.Clear(); + + StartTimer(); } // call on connection close/dispose diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index aee5f6cc..161382e9 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -45,19 +45,17 @@ [.. statementProfilers.Select(sp => sp.GetSummary())], } /// - /// Stops the timer associated with the specified statement index. + /// Stops all the statement profilers associated with this query profiler. /// - /// The zero-based index of the statement whose timer should be stopped. - /// Thrown if statementIndex is less than zero or greater than the highest valid statement profiler index. - public void StopTimer(int statementIndex) + public override void StopTimer() { - if (statementIndex < 0 || statementIndex > statementProfilers.Length - 1) + + foreach (var profiler in statementProfilers) { - throw new ArgumentOutOfRangeException(nameof(statementIndex), $"Index {statementIndex} is out of range for statement profilers."); + profiler?.StopTimer(); } - statementProfilers[statementIndex]?.StopTimer(); - StopTimer(); + base.StopTimer(); } /// diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs index 1421d4de..ad52c64d 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingQuerySummary.cs @@ -5,19 +5,19 @@ public readonly record struct ProfilingQuerySummary( DateTimeOffset EndTime, double ExecutionTimeMilliseconds, int StatementCount, - ProfilingStatementSummary[] Infos, + ProfilingStatementSummary[] StatementSummaries, DuckDBState State, string Message) { public IDictionary ToDictionary() { - return new Dictionary(6) + return new Dictionary(7) { ["StartTime"] = StartTime, ["EndTime"] = EndTime, ["ExecutionTime"] = ExecutionTimeMilliseconds, ["StatementCount"] = StatementCount, - ["Infos"] = Infos, + ["StatementSummaries"] = StatementSummaries, ["State"] = State, ["Message"] = Message }; diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs index bfb32ce8..d2f8c98b 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingStatementSummary.cs @@ -11,13 +11,13 @@ public readonly record struct ProfilingStatementSummary( { public IDictionary ToDictionary() { - return new Dictionary(6) + return new Dictionary(7) { ["StartTime"] = StartTime, ["EndTime"] = EndTime, ["ExecutionTime"] = ExecutionTimeMilliseconds, ["Order"] = Order, - ["Infos"] = Metrics, + ["Metrics"] = Metrics, ["State"] = State, ["Message"] = Message }; diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index e0f85b7a..718d84ff 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -36,9 +36,9 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() last.Message.Should().BeNullOrEmpty(); // Expect constituent statements to be successful as well - if (last.Infos.Length > 0) + if (last.StatementSummaries.Length > 0) { - var stmt = last.Infos.Last(); + var stmt = last.StatementSummaries.Last(); stmt.State.Should().Be(DuckDBState.Success); stmt.Message.Should().BeNullOrEmpty(); } @@ -49,6 +49,129 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() } } + [Fact] + public void QueryAndStatementStatusAndTimingWhenPrepareFails() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + var id = Guid.NewGuid().ToString("N"); + var tbl = $"pfp_{id}"; + + // batch where one statement will fail at prepare time (syntax error) + Command.CommandText = $"CREATE TABLE {tbl}(i INTEGER); BAD SYNTAX HERE; SELECT 1;"; + + try + { + using (var r = Command.ExecuteReader()) { do { } while (r.NextResult()); } + } + catch (Exception) + { + // swallow - prepare/execute is expected to throw + } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + + // Query-level state should reflect the failure + last.State.Should().Be(DuckDBState.Error); + last.Message.Should().NotBeNullOrWhiteSpace(); + + // Timing at query-level should be well-formed + last.ExecutionTimeMilliseconds.Should().BeGreaterOrEqualTo(0); + (last.StartTime <= last.EndTime).Should().BeTrue("StartTime should be less than or equal to EndTime"); + + last.StatementSummaries.Should().NotBeNull(); + last.StatementSummaries.Length.Should().Be(0, "No statements should be recorded when prepare fails"); + + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void QueryAndStatementStatusAndTimingWhenStatementFails() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + var id = Guid.NewGuid().ToString("N"); + var tbl = $"pf_{id}"; + + // batch with multiple statements where the last one will fail + Command.CommandText = $"CREATE TABLE {tbl}(i INTEGER); INSERT INTO {tbl} VALUES (1); SELECT i FROM {tbl}; SELECT * FROM __this_table_does_not_exist__;"; + + try + { + using (var r = Command.ExecuteReader()) { do { } while (r.NextResult()); } + } + catch (Exception) + { + // swallow - we expect an error for the failing statement + } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + + // Query-level state should reflect the failure + last.State.Should().Be(DuckDBState.Error); + last.Message.Should().NotBeNullOrWhiteSpace(); + + // Timing at query-level should be well-formed + last.ExecutionTimeMilliseconds.Should().BeGreaterOrEqualTo(0); + (last.StartTime <= last.EndTime).Should().BeTrue("StartTime should be less than or equal to EndTime"); + + // Ensure at least one statement reports error and at least one reports success + last.StatementSummaries.Should().NotBeNull(); + last.StatementSummaries.Length.Should().BeGreaterThan(0); + + var anyError = last.StatementSummaries.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); + anyError.Should().BeTrue("At least one statement should report an error state and message when one statement in a query fails"); + + var anySuccess = last.StatementSummaries.Any(i => i.State == DuckDBState.Success); + anySuccess.Should().BeTrue("At least one statement prior to the failing statement should have succeeded"); + + // Check statement timing sanity + foreach (var stmt in last.StatementSummaries) + { + stmt.ExecutionTimeMilliseconds.Should().BeGreaterOrEqualTo(0); + (stmt.StartTime <= stmt.EndTime).Should().BeTrue("Statement StartTime should be less than or equal to EndTime"); + } + } + finally + { + Connection.DisableProfiling(); + } + } + [Fact] public void MetricsCollectedWhenOverMetricsThreshold() { @@ -76,10 +199,10 @@ public void MetricsCollectedWhenOverMetricsThreshold() summary.QuerySummaryList.Length.Should().BeGreaterThan(0); var last = summary.QuerySummaryList.Last(); - last.Infos.Should().NotBeNull(); + last.StatementSummaries.Should().NotBeNull(); // When execution is over the metrics threshold, at least one statement should contain enabled metrics - var hasMetrics = last.Infos.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.QueryName) || i.Metrics.ContainsKey(DuckDBMetricType.CpuTime)); + var hasMetrics = last.StatementSummaries.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.QueryName) || i.Metrics.ContainsKey(DuckDBMetricType.CpuTime)); hasMetrics.Should().BeTrue("Metrics should be collected when execution time exceeds MetricsThresholdMS"); } finally @@ -113,10 +236,10 @@ public void MetricsNotCollectedWhenUnderMetricsThreshold() summary.QuerySummaryList.Length.Should().BeGreaterThan(0); var last = summary.QuerySummaryList.Last(); - last.Infos.Should().NotBeNull(); + last.StatementSummaries.Should().NotBeNull(); // When execution is under the metrics threshold, no metrics should be extracted - last.Infos.All(i => i.Metrics.Count == 0).Should().BeTrue("Metrics should not be collected when execution time is below the configured MetricsThresholdMS"); + last.StatementSummaries.All(i => i.Metrics.Count == 0).Should().BeTrue("Metrics should not be collected when execution time is below the configured MetricsThresholdMS"); } finally { @@ -161,7 +284,7 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnError() last.Message.Should().NotBeNullOrWhiteSpace(); // Ensure at least one statement reports error state/message - var anyError = last.Infos.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); + var anyError = last.StatementSummaries.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); anyError.Should().BeTrue("At least one statement should report an error state and message"); } finally @@ -200,7 +323,7 @@ public void MetricsDictionaryContainsEnabledMetrics() // If no new summaries were produced, profiling might not be available in this environment. var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); - var last = newSummaries.Last().Infos; + var last = newSummaries.Last().StatementSummaries; // check that enabled metrics appear in at least one statement's metrics var containsQueryName = last.Any(i => i.Metrics.ContainsKey(DuckDBMetricType.QueryName)); @@ -261,8 +384,8 @@ public void EnableProfilingWithoutOptionsUsesDefaults() summary.QueryCount.Should().BeGreaterThan(0); // with default options, there should be no metrics collected for Set statements - summary.QuerySummaryList.Last().Infos.Length.Should().Be(1); - summary.QuerySummaryList.Last().Infos[0].Metrics.Count.Should().Be(0); + summary.QuerySummaryList.Last().StatementSummaries.Length.Should().Be(1); + summary.QuerySummaryList.Last().StatementSummaries[0].Metrics.Count.Should().Be(0); } finally { @@ -409,8 +532,8 @@ public async Task FileBackedDuplicateReturnsSameMetricsAsOriginal() var last1 = s1.QuerySummaryList.Last(); var last2 = s2.QuerySummaryList.Last(); - var keys1 = new HashSet(last1.Infos.SelectMany(i => i.Metrics.Keys)); - var keys2 = new HashSet(last2.Infos.SelectMany(i => i.Metrics.Keys)); + var keys1 = new HashSet(last1.StatementSummaries.SelectMany(i => i.Metrics.Keys)); + var keys2 = new HashSet(last2.StatementSummaries.SelectMany(i => i.Metrics.Keys)); // strict check: both sets of metric keys must match keys1.Should().BeEquivalentTo(keys2); @@ -459,8 +582,8 @@ public void InMemoryDuplicateReturnsSameMetricsAsOriginal() var lastOriginal = sOriginal.QuerySummaryList.Last(); var lastDup = sDup.QuerySummaryList.Last(); - var keysOrig = new HashSet(lastOriginal.Infos.SelectMany(i => i.Metrics.Keys)); - var keysDup = new HashSet(lastDup.Infos.SelectMany(i => i.Metrics.Keys)); + var keysOrig = new HashSet(lastOriginal.StatementSummaries.SelectMany(i => i.Metrics.Keys)); + var keysDup = new HashSet(lastDup.StatementSummaries.SelectMany(i => i.Metrics.Keys)); // strict check: both sets of metric keys must match keysOrig.Should().BeEquivalentTo(keysDup); @@ -505,11 +628,11 @@ public void MultipleEnabledMetricsArePresentInMetricsDictionary() var newSummaries = summary.QuerySummaryList.Skip(beforeCount).ToArray(); newSummaries.Length.Should().BeGreaterThan(0, "Expected new query summaries after executing batch"); var last = newSummaries.Last(); - last.Infos.Should().NotBeNull(); - last.Infos.Length.Should().BeGreaterThan(0, "Expected some profiling info when metrics are enabled"); + last.StatementSummaries.Should().NotBeNull(); + last.StatementSummaries.Length.Should().BeGreaterThan(0, "Expected some profiling info when metrics are enabled"); // ensure at least one statement collected all requested metrics - bool found = last.Infos.Any(summary => metricsToEnable.All(m => summary.Metrics.ContainsKey(m))); + bool found = last.StatementSummaries.Any(summary => metricsToEnable.All(m => summary.Metrics.ContainsKey(m))); found.Should().BeTrue("At least one statement should include all the enabled metrics (QueryName, CpuTime, Latency, TotalBytesRead)"); } @@ -570,13 +693,13 @@ public void SelectCoverageRespectsSelectPosition(int selectPosition) var last = newSummaries.Last(); last.StatementCount.Should().Be(3); - last.Infos.Should().NotBeNull(); + last.StatementSummaries.Should().NotBeNull(); - if (last.Infos.Length >= last.StatementCount) + if (last.StatementSummaries.Length >= last.StatementCount) { for (int i = 0; i < last.StatementCount; i++) { - var info = i < last.Infos.Length ? last.Infos[i].Metrics : null; + var info = i < last.StatementSummaries.Length ? last.StatementSummaries[i].Metrics : null; if (i == selectPosition) { info.Should().NotBeNull(); @@ -592,7 +715,7 @@ public void SelectCoverageRespectsSelectPosition(int selectPosition) else { // If Metrics length doesn't map 1:1 to statements, at least ensure exactly one statement collected metrics - var nonEmpty = last.Infos.Count(i => i.Metrics.Count > 0); + var nonEmpty = last.StatementSummaries.Count(i => i.Metrics.Count > 0); nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); } } @@ -719,11 +842,11 @@ public void CoverageSelectOnlyProfilesOnlySelectStatements() last.StatementCount.Should().BeGreaterOrEqualTo(3); // For SELECT coverage, expect only the SELECT (last) statement to have metrics collected - if (last.Infos.Length >= last.StatementCount) + if (last.StatementSummaries.Length >= last.StatementCount) { for (int i = 0; i < last.StatementCount; i++) { - var info = i < last.Infos.Length ? last.Infos[i].Metrics : null; + var info = i < last.StatementSummaries.Length ? last.StatementSummaries[i].Metrics : null; if (i == last.StatementCount - 1) { info.Should().NotBeNull(); @@ -738,7 +861,7 @@ public void CoverageSelectOnlyProfilesOnlySelectStatements() } else { - var nonEmpty = last.Infos.Count(i => i.Metrics.Count > 0); + var nonEmpty = last.StatementSummaries.Count(i => i.Metrics.Count > 0); nonEmpty.Should().Be(1, "Expected exactly one statement to have metrics when coverage=Select"); } } From 0faffe4a1a11e72a6766a4c42b811c9626fb5cd0 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 14:13:26 +0200 Subject: [PATCH 35/43] Improve TimerUtils and profiling consistency Enhanced TimerUtils with sentinel handling for infinite durations and added a new `TimeSpanTicksToMilliseconds` method. Refactored ConnectionStatistics and ExecutionProfiler for accurate and consistent time conversions. Improved QueryProfiler and StatementProfiler with null safety and defensive checks. Expanded ProfilingTests with new test cases to validate time conversion consistency and profiling accuracy. Refactored existing tests for clarity and maintainability. Updated ProfilingTests namespaces and imports to align with project structure. Performed general code cleanup, improving readability and replacing redundant code with reusable methods. --- DuckDB.NET.Data/Common/TimerUtils.cs | 30 ++- .../Statistics/ConnectionStatistics.cs | 13 +- .../Profiling/Statistics/ExecutionProfiler.cs | 7 +- .../Profiling/Statistics/QueryProfiler.cs | 2 +- .../Profiling/Statistics/StatementProfiler.cs | 13 +- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 243 +++++++++++++----- 6 files changed, 228 insertions(+), 80 deletions(-) diff --git a/DuckDB.NET.Data/Common/TimerUtils.cs b/DuckDB.NET.Data/Common/TimerUtils.cs index 30bb2221..f18f3b40 100644 --- a/DuckDB.NET.Data/Common/TimerUtils.cs +++ b/DuckDB.NET.Data/Common/TimerUtils.cs @@ -23,7 +23,15 @@ internal sealed class TimerUtils /// The timer value to convert, typically representing elapsed timer ticks. /// The equivalent duration in milliseconds calculated from the specified timer value. internal static long TimerToMilliseconds(long timerValue) - => timerValue * 1000 / Frequency; + { + // sentinel handling: preserve "infinite" indicator + if (timerValue == long.MaxValue) + { + return long.MaxValue; + } + + return timerValue * 1000 / Frequency; + } /// /// Converts a timer tick duration (as returned by ) to a . @@ -32,11 +40,31 @@ internal static long TimerToMilliseconds(long timerValue) /// A representing the duration of the provided timer ticks. internal static TimeSpan TimerToTimeSpan(long timerValue) { + // sentinel handling: preserve "infinite" indicator + if (timerValue == long.MaxValue) + { + return TimeSpan.MaxValue; + } + // Convert stopwatch ticks to DateTime/TimeSpan ticks. TimeSpan.TicksPerSecond = 10_000_000. long timeSpanTicks = timerValue * TimeSpan.TicksPerSecond / Frequency; return TimeSpan.FromTicks(timeSpanTicks); } + /// + /// Converts a TimeSpan tick count (100-nanosecond ticks) to milliseconds as a double. + /// Handles sentinel long.MaxValue as an infinite duration. + /// + internal static double TimeSpanTicksToMilliseconds(long timeSpanTicks) + { + if (timeSpanTicks == long.MaxValue) + { + return double.MaxValue; + } + + return TimeSpan.FromTicks(timeSpanTicks).TotalMilliseconds; + } + /// /// Calculates the elapsed number of ticks between two tick count values. /// diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index 94638832..f4924d82 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -124,11 +124,16 @@ private IEnumerable ReadSummaries() internal ProfilingSummary GetProfilingSummary() { + var connectionTimeMs = TimerUtils.TimeSpanTicksToMilliseconds(connectionTime); + + // QueryProfiler.ExecutionTime is already exposed in milliseconds via ExecutionTime property + var executionTimeMs = queryProfilers.Values.Sum(qp => (double)qp.ExecutionTime); + return new ProfilingSummary( - startTime, - endTime, - TimerUtils.TimerToMilliseconds(connectionTime), - TimerUtils.TimerToMilliseconds(queryProfilers.Values.Sum(qp => qp.ExecutionTime)), + startTime, + endTime, + connectionTimeMs, + executionTimeMs, queryProfilers.Values.Count, [.. ReadSummaries()]); } diff --git a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs index 78639d16..b568e73a 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ExecutionProfiler.cs @@ -51,9 +51,10 @@ protected virtual void ReleaseAndUpdateExecutionTimer() long elapsed = TimerUtils.CalculateTickCountElapsed(startTimestamp.Value, TimerUtils.TimerCurrent()); executionTime += elapsed; - // Convert the elapsed high-resolution ticks to a TimeSpan and apply to the wall-clock start time. - var elapsedSpan = TimerUtils.TimerToTimeSpan(elapsed); - endTime = startTime.Add(elapsedSpan); + // Convert the total accumulated high-resolution ticks to a TimeSpan and apply to the wall-clock start time. + // Use the cumulative executionTime so that multiple start/stop segments are represented correctly. + var totalSpan = TimerUtils.TimerToTimeSpan(executionTime); + endTime = startTime.Add(totalSpan); startTimestamp = null; } diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 161382e9..76cd3800 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -38,7 +38,7 @@ internal ProfilingQuerySummary GetSummary() endTime, TimerUtils.TimerToMilliseconds(executionTime), statementCount, - [.. statementProfilers.Select(sp => sp.GetSummary())], + [.. statementProfilers.Select(sp => sp?.GetSummary() ?? default)], state, errorMessage ); diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index ac51936f..80b41c7c 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -34,11 +34,18 @@ internal ProfilingStatementSummary GetSummary() public override void AcquireMetrics() { // If a metrics threshold is set, check if the elapsed execution time meets the threshold before acquiring metrics. - if ((profilingOptions?.MetricsThreshold ?? 0) > 0) + var threshold = profilingOptions?.MetricsThreshold ?? 0; + if (threshold > 0) { - long elapsed = TimerUtils.CalculateTickCountElapsed(startTimestamp ?? 0, TimerUtils.TimerCurrent()); + // defensive: if timer wasn't started, skip acquiring metrics based on threshold + if (!startTimestamp.HasValue) + { + return; + } + + long elapsed = TimerUtils.CalculateTickCountElapsed(startTimestamp.Value, TimerUtils.TimerCurrent()); - if (TimerUtils.TimerToMilliseconds(elapsed) < (profilingOptions?.MetricsThreshold ?? 0)) + if (TimerUtils.TimerToMilliseconds(elapsed) < threshold) return; } diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 718d84ff..3f094253 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -1,53 +1,160 @@ -using DuckDB.NET.Data.Profiling; +using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.Profiling; using DuckDB.NET.Test.Helpers; -using DuckDB.NET.Native; +using System; +using System.Linq; +using System.Threading; namespace DuckDB.NET.Test.Profiling; public class ProfilingTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) { - [Fact] - public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() + [Fact] + public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() + { + var options = new ProfilingOptions { - var options = new ProfilingOptions - { - Coverage = DuckDBProfilingCoverage.All, - EnabledMetrics = new DuckDBMetricTypeCollection(), - Format = DuckDBProfilingFormat.Json, - Mode = DuckDBProfilingMode.Standard - }; + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; - Connection.EnableProfiling(options); + Connection.EnableProfiling(options); - try - { - Connection.ResetStatistics(); + try + { + Connection.ResetStatistics(); - Command.CommandText = "SELECT 1;"; - using (var r = Command.ExecuteReader()) { } + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } - var summary = Connection.RetrieveStatistics(); - summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); - var last = summary.QuerySummaryList.Last(); - // Expect overall query to be successful - last.State.Should().Be(DuckDBState.Success); - last.Message.Should().BeNullOrEmpty(); + var last = summary.QuerySummaryList.Last(); + // Expect overall query to be successful + last.State.Should().Be(DuckDBState.Success); + last.Message.Should().BeNullOrEmpty(); - // Expect constituent statements to be successful as well - if (last.StatementSummaries.Length > 0) - { - var stmt = last.StatementSummaries.Last(); - stmt.State.Should().Be(DuckDBState.Success); - stmt.Message.Should().BeNullOrEmpty(); - } + // Expect constituent statements to be successful as well + if (last.StatementSummaries.Length > 0) + { + var stmt = last.StatementSummaries.Last(); + stmt.State.Should().Be(DuckDBState.Success); + stmt.Message.Should().BeNullOrEmpty(); } - finally + } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void ProfilingSummary_ConversionsAreConsistent() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + // run two queries with measurable time + Command.CommandText = "SELECT SUM(i) FROM range(200000) AS t(i);"; + using (var r = Command.ExecuteReader()) { } + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThanOrEqualTo(1); + + // connection-level: wall-clock vs stored connection time (milliseconds) + var wallConnMs = (summary.EndTime - summary.StartTime).TotalMilliseconds; + Math.Abs(wallConnMs - summary.ConnectionTimeMilliseconds).Should().BeLessThan(250, "Connection time conversion should match wall-clock duration"); + + // execution time (total) should equal summed query execution times + var sumQueryExecMs = summary.QuerySummaryList.Sum(q => q.ExecutionTimeMilliseconds); + Math.Abs(sumQueryExecMs - summary.ExecutionTimeMilliseconds).Should().BeLessThan(200, "Total execution time should match sum of query execution times"); + + // per-query: check wall-clock vs measured execution time and statement sum + foreach (var q in summary.QuerySummaryList) { - Connection.DisableProfiling(); + var wallMs = (q.EndTime - q.StartTime).TotalMilliseconds; + Math.Abs(wallMs - q.ExecutionTimeMilliseconds).Should().BeLessThan(250, "Query execution conversion should match wall-clock duration"); + + var sumStmtMs = q.StatementSummaries.Sum(s => s.ExecutionTimeMilliseconds); + Math.Abs(sumStmtMs - q.ExecutionTimeMilliseconds).Should().BeLessThan(200, "Query execution time should match sum of statement execution times"); } } + finally + { + Connection.DisableProfiling(); + } + } + + [Fact] + public void TimerUtils_ConversionsAreConsistent() + { + // measure a short delay using Stopwatch ticks and verify conversions + long start = System.Diagnostics.Stopwatch.GetTimestamp(); + Thread.Sleep(120); + long end = System.Diagnostics.Stopwatch.GetTimestamp(); + + long elapsed = end - start; + + var msFromTimer = TimerUtils.TimerToMilliseconds(elapsed); + var span = TimerUtils.TimerToTimeSpan(elapsed); + + // both conversions should be approximately the same + Math.Abs(((long)span.TotalMilliseconds) - msFromTimer).Should().BeLessOrEqualTo(2); + + // and should be close to the real sleep duration + msFromTimer.Should().BeGreaterOrEqualTo(100); + } + + [Fact] + public void Profiler_StartStop_ProducesConsistentTiming() + { + var options = new ProfilingOptions { Coverage = DuckDBProfilingCoverage.All }; + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + // run a query that takes measurable time and is visible through public RetrieveStatistics API + Command.CommandText = "SELECT SUM(i) FROM range(200000) AS t(i);"; + using (var r = Command.ExecuteReader()) { } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + + // execution time should be non-zero and reasonable + last.ExecutionTimeMilliseconds.Should().BeGreaterThan(0); + + var wallMs = (last.EndTime - last.StartTime).TotalMilliseconds; + + // the wall-clock duration and the measured execution time should be reasonably close + Math.Abs(wallMs - last.ExecutionTimeMilliseconds).Should().BeLessThan(200, "Timer conversion should be consistent with wall-clock duration"); + } + finally + { + Connection.DisableProfiling(); + } + } [Fact] public void QueryAndStatementStatusAndTimingWhenPrepareFails() @@ -247,51 +354,51 @@ public void MetricsNotCollectedWhenUnderMetricsThreshold() } } - [Fact] - public void ProfilingSummariesContainCorrectStateAndMessageOnError() + [Fact] + public void ProfilingSummariesContainCorrectStateAndMessageOnError() + { + var options = new ProfilingOptions { - var options = new ProfilingOptions - { - Coverage = DuckDBProfilingCoverage.All, - EnabledMetrics = new DuckDBMetricTypeCollection(), - Format = DuckDBProfilingFormat.Json, - Mode = DuckDBProfilingMode.Standard - }; + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection(), + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); - Connection.EnableProfiling(options); + try + { + Connection.ResetStatistics(); + // Execute a statement that will error + Command.CommandText = "SELECT * FROM __this_table_does_not_exist__"; try { - Connection.ResetStatistics(); - - // Execute a statement that will error - Command.CommandText = "SELECT * FROM __this_table_does_not_exist__"; - try - { - using (var r = Command.ExecuteReader()) { } - } - catch (Exception) - { - // swallow - we expect an error to be thrown by the command execution - } - - var summary = Connection.RetrieveStatistics(); - summary.QuerySummaryList.Length.Should().BeGreaterThan(0); - - var last = summary.QuerySummaryList.Last(); - // Overall query should be marked as error - last.State.Should().Be(DuckDBState.Error); - last.Message.Should().NotBeNullOrWhiteSpace(); - - // Ensure at least one statement reports error state/message - var anyError = last.StatementSummaries.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); - anyError.Should().BeTrue("At least one statement should report an error state and message"); + using (var r = Command.ExecuteReader()) { } } - finally + catch (Exception) { - Connection.DisableProfiling(); + // swallow - we expect an error to be thrown by the command execution } + + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + + var last = summary.QuerySummaryList.Last(); + // Overall query should be marked as error + last.State.Should().Be(DuckDBState.Error); + last.Message.Should().NotBeNullOrWhiteSpace(); + + // Ensure at least one statement reports error state/message + var anyError = last.StatementSummaries.Any(i => i.State == DuckDBState.Error && !string.IsNullOrWhiteSpace(i.Message)); + anyError.Should().BeTrue("At least one statement should report an error state and message"); + } + finally + { + Connection.DisableProfiling(); } + } [Fact] public void MetricsDictionaryContainsEnabledMetrics() From 7b2f5fa3c790a0438dae606bc7e82ff515869ab6 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Fri, 5 Jun 2026 18:17:47 +0200 Subject: [PATCH 36/43] Refactor and extend DuckDB profiling and metrics support - Move MetricsExtensions to DuckDBMetricsExtensions and update usages - Add ProfilingOptionsExtensions for SQL generation from options - Introduce DuckDBMetrics with static metric collections - Use IsProfilingEnabled and improve profiling state management - Refine profiling lifecycle and statistics reset/disposal - Enhance tests for profiling output path and JSON validation - General code cleanup and improved exception handling --- DuckDB.NET.Data/DuckDBConnection.cs | 132 ++++++++++-------- ...tensions.cs => DuckDBMetricsExtensions.cs} | 4 +- .../Extensions/ProfilingOptionsExtensions.cs | 34 +++++ .../PreparedStatement/PreparedStatement.cs | 2 +- DuckDB.NET.Data/Profiling/DuckDBMetrics.cs | 48 +++++++ DuckDB.NET.Data/Profiling/MetricType.cs | 28 ---- DuckDB.NET.Data/Profiling/ProfilingInfo.cs | 14 +- .../Statistics/ConnectionStatistics.cs | 15 +- .../Statistics/ProfilingInfoMetrics.cs | 2 +- .../Profiling/Statistics/StatementProfiler.cs | 2 +- DuckDB.NET.Test/Profiling/ProfilingTests.cs | 127 ++++++++++++++++- 11 files changed, 308 insertions(+), 100 deletions(-) rename DuckDB.NET.Data/Extensions/{MetricsExtensions.cs => DuckDBMetricsExtensions.cs} (96%) create mode 100644 DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs create mode 100644 DuckDB.NET.Data/Profiling/DuckDBMetrics.cs diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index d858549b..24760dfb 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -1,12 +1,10 @@ -using DuckDB.NET.Data.Common; -using DuckDB.NET.Data.Connection; +using DuckDB.NET.Data.Connection; using DuckDB.NET.Data.Profiling; using DuckDB.NET.Data.Profiling.Statistics; using DuckDB.NET.Data.Profiling.Statistics.Summary; using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; -using System.Text; namespace DuckDB.NET.Data; @@ -22,7 +20,7 @@ public partial class DuckDBConnection : DbConnection // Statistics support private ConnectionStatistics? statistics; - private bool isProfilingEnabled; + private bool profilingEnabled; private ProfilingOptions? profilingOptions; #region Protected Properties @@ -87,18 +85,6 @@ public override string DataSource public DuckDBDatabase NativeDatabase => connectionReference?.FileReferenceCounter.Database ?? throw new InvalidOperationException("The DuckDBConnection must be open to access the native database."); - - /// - /// Sets the minimum execution time, in milliseconds, required for a query plan to be collected for analysis. - /// - /// The minimum duration, in milliseconds, that a query must run before its plan is collected. Must be a - /// non-negative integer. - /// Thrown if the connection is not open. - public void QueryPlanCollectionThreshold(int threshold) - { - throw new NotImplementedException(); - } - public override string ServerVersion => NativeMethods.Startup.DuckDBLibraryVersion(); public override ConnectionState State => connectionState; @@ -140,17 +126,16 @@ public override void Open() connectionState = ConnectionState.Open; - // If profiling is not enabled, profilingOptions will be null and we need to initialize it with the - // connectionReference.FileReferenceCounter to ensure that if profiling is later enabled for this connection, - // it will use the correct options, same for isProfilingEnabled - if (profilingOptions.IsNull()) + // If profilingOptions has no explicit value for this connection, inherit the file-backed DB settings. + // Otherwise persist the explicit options to the shared FileReference so other connections see them. + if (!profilingOptions.HasValue) { - isProfilingEnabled = connectionReference.FileReferenceCounter.IsProfilingEnabled; + profilingEnabled = connectionReference.FileReferenceCounter.IsProfilingEnabled; profilingOptions = connectionReference.FileReferenceCounter.ProfilingOptions; } else { - connectionReference.FileReferenceCounter.IsProfilingEnabled = isProfilingEnabled; + connectionReference.FileReferenceCounter.IsProfilingEnabled = profilingEnabled; connectionReference.FileReferenceCounter.ProfilingOptions = profilingOptions.Value; } @@ -307,7 +292,7 @@ public DuckDBConnection Duplicate() parsedConnection = ParsedConnection, inMemoryDuplication = true, connectionReference = connectionReference, - isProfilingEnabled = isProfilingEnabled, + profilingEnabled = profilingEnabled, profilingOptions = profilingOptions, }; @@ -331,7 +316,7 @@ public DuckDBQueryProgress GetQueryProgress() #region profiling - public bool ProfilingEnabled => isProfilingEnabled; + public bool IsProfilingEnabled => profilingEnabled; /// /// Retrieves a summary of the collected profiling statistics, including connection time, execution time, and any relevant metrics. @@ -347,7 +332,8 @@ public ProfilingSummary RetrieveStatistics() } else { - return new ConnectionStatistics(NativeConnection, isProfilingEnabled).GetProfilingSummary(); + // Profiling not enabled for this connection + return new ProfilingSummary(); } } @@ -359,11 +345,14 @@ public ProfilingSummary RetrieveStatistics() /// An optional set of profiling options to configure profiling behavior. If null, default options are used. public void EnableProfiling(ProfilingOptions? options = null) { - isProfilingEnabled = true; + profilingEnabled = true; this.profilingOptions = options ?? new ProfilingOptions(); // use provided options or default options if null - connectionReference?.FileReferenceCounter?.IsProfilingEnabled = true; - connectionReference?.FileReferenceCounter?.ProfilingOptions = this.profilingOptions.Value; + if (connectionReference?.FileReferenceCounter is { } fileRefCounter) + { + fileRefCounter.IsProfilingEnabled = true; + fileRefCounter.ProfilingOptions = this.profilingOptions.Value; + } if (State == ConnectionState.Open) { @@ -381,14 +370,13 @@ public void DisableProfiling(bool resetStatistics = false) { // stop statistics?.StopTimer(); - DisableProfiling(); if (resetStatistics) { ResetStatistics(); } - connectionReference?.FileReferenceCounter?.IsProfilingEnabled = false; + DisableProfiling(); } /// @@ -398,7 +386,7 @@ public void DisableProfiling(bool resetStatistics = false) /// profiling data before starting a new measurement period. public void ResetStatistics() { - if (ProfilingEnabled) + if (IsProfilingEnabled) { statistics?.Reset(); } @@ -410,13 +398,22 @@ public void ResetStatistics() /// This method prepares internal data structures to collect and store profiling information /// related to the connection's activity. It should be called before attempting to access profiling or statistics /// data for the connection. - private void InitProfiling() + private void InitProfiling() { EnsureConnectionOpen(); - if (ProfilingEnabled) + if (IsProfilingEnabled) { - statistics = new ConnectionStatistics(NativeConnection, isProfilingEnabled, this.profilingOptions); + // Dispose any existing statistics instance to ensure fresh state and remove previous mapping + // from the global ConnectionStatistics cache. + try + { + statistics?.Dispose(); + } + finally + { + statistics = new ConnectionStatistics(NativeConnection, profilingEnabled, this.profilingOptions); + } LoadStatisticsProfile(); } } @@ -428,12 +425,18 @@ private void InitProfiling() /// Thrown if profiling is not enabled when attempting to edit profiling options. public void EditProfilingOptions(ProfilingOptions options) { - if (!ProfilingEnabled) + if (!IsProfilingEnabled) { throw new InvalidOperationException("Profiling must be enabled to edit profiling options."); } + this.profilingOptions = options; + if (connectionReference?.FileReferenceCounter is { } fileRefCounter) + { + fileRefCounter.ProfilingOptions = options; + } + statistics?.Reset(); LoadStatisticsProfile(); } @@ -446,17 +449,29 @@ public void EditProfilingOptions(ProfilingOptions options) /// Thrown if an error occurs while disabling profiling on the database connection. private void DisableProfiling() { - if (ProfilingEnabled && State == ConnectionState.Open) + if (IsProfilingEnabled && State == ConnectionState.Open) { - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, "CALL disable_profiling();", out _); - if (!state.IsSuccess()) + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, "CALL disable_profiling();", out var queryResult); + EnsureStateIsSuccess(state, queryResult, "Error disabling profiling."); + + if (this.profilingOptions is { OutputPath: not null }) { - throw new DuckDBException("Error disabling profiling."); + // If output path is set, we need to disable profiling at the end of each session to ensure the file is properly flushed and closed. + state = NativeMethods.Query.DuckDBQuery(NativeConnection, "SET profiling_output = ''", out var queryResult2); + EnsureStateIsSuccess(state, queryResult2, "Error disabling profiling."); } } + this.profilingOptions = null; + + if (connectionReference?.FileReferenceCounter is { } fileRefCounter) + { + fileRefCounter.IsProfilingEnabled = false; + fileRefCounter.ProfilingOptions = null; + } + statistics?.DisableQueryExecutionTracing(); - isProfilingEnabled = false; + profilingEnabled = false; } /// @@ -472,31 +487,29 @@ private void LoadStatisticsProfile() { statistics?.StartTimer(); - if (ProfilingEnabled && profilingOptions is { } options) + if (IsProfilingEnabled && profilingOptions is { } options) { - var sb = new StringBuilder(); + // Build SQL and emit lightweight diagnostics to help reproduce issues when tests run together + var sql = ProfilingOptionsExtensions.ToDuckDBProfilingOptionString(options); - sb.AppendLine("CALL enable_profiling("); - sb.AppendLine($" format := '{options.Format.ToDuckDBProfilingFormatString()}',"); - sb.AppendLine($" coverage := '{options.Coverage}',"); - sb.AppendLine($" mode := '{options.Mode}',"); - - if (!string.IsNullOrEmpty(options.OutputPath)) - sb.AppendLine($" save_location := '{options.OutputPath}',"); - - sb.AppendLine($" metrics := '{(options.EnabledMetrics ?? new DuckDBMetricTypeCollection(DuckDBMetrics.DefaultMetrics)).ToDuckDBMetricString()}'"); - sb.AppendLine(");"); + var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sql, out var queryResult); + EnsureStateIsSuccess(state, queryResult, "Error configuring profiling settings."); + } + } - var state = NativeMethods.Query.DuckDBQuery(NativeConnection, sb.ToString(), out var queryResult); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureStateIsSuccess(DuckDBState state, DuckDBResult result, string fallbackMessage) + { + try + { if (!state.IsSuccess()) { - var errorMessage = NativeMethods.Query.DuckDBResultError(ref queryResult); - var errorType = NativeMethods.Query.DuckDBResultErrorType(ref queryResult); - queryResult.Close(); + var errorMessage = NativeMethods.Query.DuckDBResultError(ref result); + var errorType = NativeMethods.Query.DuckDBResultErrorType(ref result); if (string.IsNullOrEmpty(errorMessage)) { - errorMessage = "Error configuring profiling settings"; + errorMessage = fallbackMessage; } if (errorType == DuckDBErrorType.Interrupt) @@ -509,7 +522,10 @@ private void LoadStatisticsProfile() ? new DuckDBException(errorMessage, innerException) : new DuckDBException(errorMessage, errorType); } - queryResult.Close(); + } + finally + { + result.Close(); } } diff --git a/DuckDB.NET.Data/Extensions/MetricsExtensions.cs b/DuckDB.NET.Data/Extensions/DuckDBMetricsExtensions.cs similarity index 96% rename from DuckDB.NET.Data/Extensions/MetricsExtensions.cs rename to DuckDB.NET.Data/Extensions/DuckDBMetricsExtensions.cs index 4486a309..26bbf06f 100644 --- a/DuckDB.NET.Data/Extensions/MetricsExtensions.cs +++ b/DuckDB.NET.Data/Extensions/DuckDBMetricsExtensions.cs @@ -6,7 +6,7 @@ namespace DuckDB.NET.Data.Extensions { - internal static class MetricsExtensions + internal static class DuckDBMetricsExtensions { // Cache the snake_case conversions of DuckDBProfilingFormat enum values for efficient lookup. private static readonly FrozenDictionary ProfilingFormatCache = @@ -27,7 +27,7 @@ internal static class MetricsExtensions ProfilingFormatCache.ToFrozenDictionary(kv => kv.Value, kv => kv.Key); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static string ToDuckDBMetricString(this DuckDBMetricTypeCollection metrics) + internal static string ToDuckDBMetricTypeCollectionString(this DuckDBMetricTypeCollection metrics) { return $"{{{string.Join(",", metrics.Select(m => $"\"{m.ToDuckDBMetricTypeString()}\": \"TRUE\""))}}}"; } diff --git a/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs b/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs new file mode 100644 index 00000000..d30de09b --- /dev/null +++ b/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs @@ -0,0 +1,34 @@ +using DuckDB.NET.Data.Profiling; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace DuckDB.NET.Data.Extensions +{ + internal static class ProfilingOptionsExtensions + { + public static string ToDuckDBProfilingOptionString(this ProfilingOptions options) + { + var sb = new StringBuilder(); + + sb.AppendLine("CALL enable_profiling("); + sb.AppendLine($" format := '{options.Format.ToDuckDBProfilingFormatString()}',"); + sb.AppendLine($" coverage := '{options.Coverage}',"); + sb.AppendLine($" mode := '{options.Mode}'"); + + if (!string.IsNullOrEmpty(options.OutputPath)) + sb.AppendLine($" ,save_location := '{options.OutputPath.Replace("'", "''")}'"); + + if (options.EnabledMetrics is { Count: > 0 }) + sb.AppendLine($" ,metrics := '{(options.EnabledMetrics).ToDuckDBMetricTypeCollectionString()}'"); + else + sb.AppendLine($" ,metrics := '{DuckDBMetrics.DefaultMetrics.ToDuckDBMetricTypeCollectionString()}'"); + + sb.AppendLine(");"); + + return sb.ToString(); + } + } +} diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 55f48c85..22539cf8 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -75,7 +75,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); } - } + } } finally { diff --git a/DuckDB.NET.Data/Profiling/DuckDBMetrics.cs b/DuckDB.NET.Data/Profiling/DuckDBMetrics.cs new file mode 100644 index 00000000..a9e65ad4 --- /dev/null +++ b/DuckDB.NET.Data/Profiling/DuckDBMetrics.cs @@ -0,0 +1,48 @@ +using System.Linq; + +namespace DuckDB.NET.Data.Profiling +{ + public static class DuckDBMetrics + { + public static readonly DuckDBMetricTypeCollection AllMetrics = new DuckDBMetricTypeCollection(Enum.GetValues().Cast()); + + public static readonly DuckDBMetricTypeCollection DefaultMetrics = new DuckDBMetricTypeCollection( + [ + // Basic query metadata + DuckDBMetricType.QueryName, + DuckDBMetricType.Latency, + DuckDBMetricType.CpuTime, + DuckDBMetricType.BlockedThreadTime, + + // Result and cardinality + DuckDBMetricType.ResultSetSize, + // DuckDBMetricType.RowsReturned, // Not enabled due to known issue + DuckDBMetricType.CumulativeCardinality, + DuckDBMetricType.CumulativeRowsScanned, + DuckDBMetricType.ExtraInfo, + + // Operator-level metrics + DuckDBMetricType.OperatorCardinality, + DuckDBMetricType.OperatorName, + DuckDBMetricType.OperatorRowsScanned, + DuckDBMetricType.OperatorTiming, + DuckDBMetricType.OperatorType, + + // Memory / IO / storage metrics + DuckDBMetricType.TotalBytesRead, + DuckDBMetricType.TotalBytesWritten, + DuckDBMetricType.TotalMemoryAllocated, + DuckDBMetricType.SystemPeakBufferMemory, + DuckDBMetricType.SystemPeakTempDirSize, + + // WAL / attach / checkpoint metrics + DuckDBMetricType.AttachLoadStorageLatency, + DuckDBMetricType.AttachReplayWalLatency, + DuckDBMetricType.CheckpointLatency, + DuckDBMetricType.CommitLocalStorageLatency, + DuckDBMetricType.WaitingToAttachLatency, + DuckDBMetricType.WalReplayEntryCount, + DuckDBMetricType.WriteToWalLatency + ]); + } +} \ No newline at end of file diff --git a/DuckDB.NET.Data/Profiling/MetricType.cs b/DuckDB.NET.Data/Profiling/MetricType.cs index 511509a6..e69de29b 100644 --- a/DuckDB.NET.Data/Profiling/MetricType.cs +++ b/DuckDB.NET.Data/Profiling/MetricType.cs @@ -1,28 +0,0 @@ -using System.Linq; - -namespace DuckDB.NET.Data.Profiling -{ - public static class DuckDBMetrics - { - public static IEnumerable AllMetrics => Enum.GetValues().Cast(); - - public static IEnumerable DefaultMetrics => - [ - DuckDBMetricType.QueryName, - DuckDBMetricType.Latency, - DuckDBMetricType.CpuTime, - DuckDBMetricType.BlockedThreadTime, - DuckDBMetricType.ResultSetSize, - //DuckDBMetricType.RowsReturned, - DuckDBMetricType.CumulativeRowsScanned, - DuckDBMetricType.TotalBytesRead, - DuckDBMetricType.TotalBytesWritten, - DuckDBMetricType.TotalMemoryAllocated, - DuckDBMetricType.SystemPeakBufferMemory, - DuckDBMetricType.SystemPeakTempDirSize, - DuckDBMetricType.WriteToWalLatency - ]; - - } - -} diff --git a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs index 276073d4..2e919912 100644 --- a/DuckDB.NET.Data/Profiling/ProfilingInfo.cs +++ b/DuckDB.NET.Data/Profiling/ProfilingInfo.cs @@ -11,6 +11,10 @@ internal ProfilingInfo(DuckDBNativeConnection connection) this.connection = connection; } + /// + /// Tries to prepare the profiling info by retrieving it from the connection. Returns true if successful, false otherwise. + /// + /// internal bool TryPrepare() { var profilingInfo = DuckDBProfilingInfoWrapper.GetProfilingInfo(connection); @@ -24,13 +28,19 @@ internal bool TryPrepare() return true; } + /// + /// Gets the raw metrics as a dictionary of string key-value pairs. Throws an exception if the profiling info is not prepared. + /// + /// + /// internal Dictionary GetRawMetrics() { if (duckDBProfilingInfoWrapper == null) { - throw new InvalidOperationException("Profiling info is not prepared. Call Prepare() first."); + throw new InvalidOperationException("Profiling info is not prepared. Call TryPrepare() first."); } - var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); + + using var metricsValue = duckDBProfilingInfoWrapper.GetMetrics(); if (metricsValue.IsNull()) { return []; diff --git a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs index f4924d82..288201dc 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ConnectionStatistics.cs @@ -7,7 +7,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal sealed class ConnectionStatistics : ExecutionProfiler +internal sealed class ConnectionStatistics : ExecutionProfiler, IDisposable { // internal values that are not exposed through properties @@ -101,6 +101,19 @@ private int GetNextQueryId() return Interlocked.Increment(ref nextQueryId); } + /// + /// Stops all active timers for the connection and its associated query profilers, recording their elapsed times. + /// + public override void StopTimer() + { + foreach (var profiler in queryProfilers.Values) + { + profiler?.StopTimer(); + } + + base.StopTimer(); + } + internal void UpdateStatistics() { // update connection time diff --git a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs index 8e3d49fb..645cb9aa 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/ProfilingInfoMetrics.cs @@ -11,7 +11,7 @@ internal static ProfilingInfoMetrics FromRawMetrics(Dictionary d foreach (var kvp in dict) { - if (MetricsExtensions.TryParseDuckDBMetricType(kvp.Key, out var metricType)) + if (DuckDBMetricsExtensions.TryParseDuckDBMetricType(kvp.Key, out var metricType)) { result.Add(metricType, kvp.Value); } diff --git a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs index 80b41c7c..a30be188 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/StatementProfiler.cs @@ -49,7 +49,7 @@ public override void AcquireMetrics() return; } - var profile = new ProfilingInfo(duckDBNativeConnection); + using var profile = new ProfilingInfo(duckDBNativeConnection); if (profile.TryPrepare()) { diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 3f094253..766aadf4 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -1,8 +1,8 @@ using DuckDB.NET.Data.Common; +using DuckDB.NET.Data.Extensions; using DuckDB.NET.Data.Profiling; using DuckDB.NET.Test.Helpers; -using System; -using System.Linq; +using System.Text.Json; using System.Threading; namespace DuckDB.NET.Test.Profiling; @@ -52,6 +52,121 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() } } + [Fact] + public void EnableProfiling_UsesOutputPathConfiguration() + { + // create a temp directory to be used as profiling output + var tempLocation = Path.Combine(Path.GetTempPath(), "duckdb_profile_output_" + Guid.NewGuid().ToString("N"), "profile.json"); + var tempDir = Path.GetDirectoryName(tempLocation)!; + Directory.CreateDirectory(Path.GetDirectoryName(tempLocation)!); + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + try + { + Connection.EnableProfiling(options); + + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + // If DuckDB writes files to the provided OutputPath, the directory should remain accessible + // We can't reliably assert that DuckDB created files in all environments, but we verify the option was accepted + var summary = Connection.RetrieveStatistics(); + summary.QuerySummaryList.Length.Should().BeGreaterThan(0); + } + finally + { + Connection.DisableProfiling(); + try { Directory.Delete(tempDir, true); } catch { } + } + } + + [Fact] + public void EnableProfiling_WritesJsonToOutputPath_WithQueryNameProperty() + { + // create a temp directory to be used as profiling output + var tempLocation = Path.Combine(Path.GetTempPath(), "duckdb_profile_output_" + Guid.NewGuid().ToString("N"), "profile.json"); + var tempDir = Path.GetDirectoryName(tempLocation)!; + Directory.CreateDirectory(tempDir); + + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + OutputPath = tempLocation + }; + + try + { + + Connection.EnableProfiling(options); + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + // give DuckDB a small amount of time to flush files + var jsonFiles = Array.Empty(); + for (int i = 0; i < 10; i++) + { + jsonFiles = Directory.EnumerateFiles(tempDir, "*.json", SearchOption.AllDirectories).ToArray(); + if (jsonFiles.Length > 0) break; + Thread.Sleep(200); + } + + jsonFiles.Length.Should().BeGreaterThan(0, "Expected at least one JSON file in the profiling output path"); + + // parse the first JSON file and assert it contains a property named "QueryName" somewhere + var json = File.ReadAllText(jsonFiles[0]); + using var doc = JsonDocument.Parse(json); + + bool found = false; + void Search(JsonElement el) + { + if (found) return; + if (el.ValueKind == JsonValueKind.Object) + { + foreach (var prop in el.EnumerateObject()) + { + if (string.Equals(prop.Name, DuckDBMetricsExtensions.ToDuckDBMetricTypeString(DuckDBMetricType.QueryName), StringComparison.OrdinalIgnoreCase)) + { + found = true; + return; + } + Search(prop.Value); + if (found) return; + } + } + else if (el.ValueKind == JsonValueKind.Array) + { + foreach (var item in el.EnumerateArray()) + { + Search(item); + if (found) return; + } + } + } + + Search(doc.RootElement); + found.Should().BeTrue("Expected the profiling JSON to contain a 'QueryName' property when QueryName metric is enabled"); + } + finally + { + Connection.DisableProfiling(); + try { Directory.Delete(tempDir, true); } catch { } + } + } + [Fact] public void ProfilingSummary_ConversionsAreConsistent() { @@ -470,7 +585,7 @@ public void EnableProfilingWithoutOptionsUsesDefaults() using (var r = cmd.ExecuteReader()) { } // profiling should be enabled - con.ProfilingEnabled.Should().BeTrue(); + con.IsProfilingEnabled.Should().BeTrue(); var summary = con.RetrieveStatistics(); summary.QueryCount.Should().BeGreaterThan(0); @@ -485,7 +600,7 @@ public void EnableProfilingWithoutOptionsUsesDefaults() using (var r = cmd.ExecuteReader()) { } // profiling should be enabled - con.ProfilingEnabled.Should().BeTrue(); + con.IsProfilingEnabled.Should().BeTrue(); summary = con.RetrieveStatistics(); summary.QueryCount.Should().BeGreaterThan(0); @@ -532,7 +647,7 @@ public async Task FileBackedConnectionSharesProfilingStateWhenEnabledOnFirst() conn2.Open(); // conn2 should have profiling enabled because it shares the file-backed DB state - conn2.ProfilingEnabled.Should().BeTrue(); + conn2.IsProfilingEnabled.Should().BeTrue(); // run a query on conn2 and ensure it has its own statistics recorded using var cmd2 = conn2.CreateCommand(); @@ -622,7 +737,7 @@ public async Task FileBackedDuplicateReturnsSameMetricsAsOriginal() { conn2.Open(); // conn2 should have profiling enabled implicitly for file-backed DB when conn1 enabled it - conn2.ProfilingEnabled.Should().BeTrue(); + conn2.IsProfilingEnabled.Should().BeTrue(); using (var c2 = conn2.CreateCommand()) { From cb15d0ddb38645243565474a1fb0a0be5a2f6754 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 09:24:19 +0200 Subject: [PATCH 37/43] Improve profiling output handling and add related tests - Disable profiling file output when OutputPath is empty - Add tests for clearing statistics, stopping file writes, and OutputPath removal - Ensure profiling configuration changes are reflected in output behavior and statistics --- .../Extensions/ProfilingOptionsExtensions.cs | 4 + DuckDB.NET.Test/Profiling/ProfilingTests.cs | 180 ++++++++++++++++++ 2 files changed, 184 insertions(+) diff --git a/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs b/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs index d30de09b..bba66011 100644 --- a/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs +++ b/DuckDB.NET.Data/Extensions/ProfilingOptionsExtensions.cs @@ -28,6 +28,10 @@ public static string ToDuckDBProfilingOptionString(this ProfilingOptions options sb.AppendLine(");"); + // If no output path is provided, disable profiling output to file + if (string.IsNullOrEmpty(options.OutputPath)) + sb.AppendLine("SET profiling_output = '';"); + return sb.ToString(); } } diff --git a/DuckDB.NET.Test/Profiling/ProfilingTests.cs b/DuckDB.NET.Test/Profiling/ProfilingTests.cs index 766aadf4..a747ab27 100644 --- a/DuckDB.NET.Test/Profiling/ProfilingTests.cs +++ b/DuckDB.NET.Test/Profiling/ProfilingTests.cs @@ -52,6 +52,113 @@ public void ProfilingSummariesContainCorrectStateAndMessageOnSuccess() } } + [Fact] + public void DisableProfiling_WithResetTrue_ClearsStatistics() + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EnableProfiling(options); + + try + { + // run a first batch to collect stats + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var before = Connection.RetrieveStatistics(); + before.QueryCount.Should().BeGreaterThan(0, "Expected some profiling data before disabling with reset"); + + // disable and reset statistics + Connection.DisableProfiling(true); + + var after = Connection.RetrieveStatistics(); + after.QueryCount.Should().Be(0, "Expected QueryCount to be zero after DisableProfiling(true)"); + } + finally + { + try { Connection.DisableProfiling(true); } catch { } + } + } + + [Fact] + public void DisableProfiling_StopsWritingFiles() + { + // create a temp directory to be used as profiling output + var tempLocation = Path.Combine(Path.GetTempPath(), "duckdb_profile_output_" + Guid.NewGuid().ToString("N"), "profile.json"); + var tempDir = Path.GetDirectoryName(tempLocation)!; + Directory.CreateDirectory(tempDir); + + try + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + OutputPath = tempLocation + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + // give DuckDB a small amount of time to flush files + var jsonFiles = Array.Empty(); + for (int i = 0; i < 10; i++) + { + jsonFiles = [.. Directory.EnumerateFiles(tempDir, "*.json", SearchOption.AllDirectories)]; + if (jsonFiles.Length > 0) break; + Thread.Sleep(200); + } + + jsonFiles.Length.Should().BeGreaterThan(0, "Expected at least one JSON file in the profiling output path after enabling with OutputPath"); + + var beforeCount = jsonFiles.Length; + + // Disable profiling + Connection.DisableProfiling(); + + Connection.ResetStatistics(); + + // run another query; no new files should be produced + Command.CommandText = "SELECT 2;"; + using (var r = Command.ExecuteReader()) { } + + // wait a short while and re-check files + var jsonFilesAfter = Array.Empty(); + for (int i = 0; i < 10; i++) + { + jsonFilesAfter = [.. Directory.EnumerateFiles(tempDir, "*.json", SearchOption.AllDirectories)]; + if (jsonFilesAfter.Length != beforeCount) break; + Thread.Sleep(200); + } + + // After disabling profiling, directory should not gain additional JSON files + jsonFilesAfter.Length.Should().Be(beforeCount, "No additional profiling JSON files should be created after disabling profiling"); + } + finally + { + try { Connection.DisableProfiling(); } catch { } + } + } + finally + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + [Fact] public void EnableProfiling_UsesOutputPathConfiguration() { @@ -167,6 +274,79 @@ void Search(JsonElement el) } } + [Fact] + public void EditingProfilingOptions_RemovesOutputPath_WhenCleared() + { + var tempLocation = Path.Combine(Path.GetTempPath(), "duckdb_profile_output_" + Guid.NewGuid().ToString("N"), "profile.json"); + var tempDir = Path.GetDirectoryName(tempLocation)!; + Directory.CreateDirectory(tempDir); + + try + { + var options = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + OutputPath = tempLocation + }; + + Connection.EnableProfiling(options); + + try + { + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 1;"; + using (var r = Command.ExecuteReader()) { } + + var jsonFiles = Array.Empty(); + for (int i = 0; i < 10; i++) + { + jsonFiles = Directory.EnumerateFiles(tempDir, "*.json", SearchOption.AllDirectories).ToArray(); + if (jsonFiles.Length > 0) break; + Thread.Sleep(200); + } + + jsonFiles.Length.Should().BeGreaterThan(0, "Expected at least one JSON file in the profiling output path after enabling with OutputPath"); + var beforeCount = jsonFiles.Length; + + var newOptions = new ProfilingOptions + { + Coverage = DuckDBProfilingCoverage.All, + EnabledMetrics = new DuckDBMetricTypeCollection { DuckDBMetricType.QueryName }, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard + }; + + Connection.EditProfilingOptions(newOptions); + Connection.ResetStatistics(); + + Command.CommandText = "SELECT 2;"; + using (var r = Command.ExecuteReader()) { } + + var jsonFilesAfter = Array.Empty(); + for (int i = 0; i < 10; i++) + { + jsonFilesAfter = Directory.EnumerateFiles(tempDir, "*.json", SearchOption.AllDirectories).ToArray(); + if (jsonFilesAfter.Length != beforeCount) break; + Thread.Sleep(200); + } + + jsonFilesAfter.Length.Should().Be(beforeCount, "No additional profiling JSON files should be created after clearing OutputPath"); + } + finally + { + Connection.DisableProfiling(); + } + } + finally + { + try { Directory.Delete(tempDir, true); } catch { } + } + } + [Fact] public void ProfilingSummary_ConversionsAreConsistent() { From 4332fbeeebc3423657e17cc3a609643534f2f589 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 09:36:27 +0200 Subject: [PATCH 38/43] Make QuerySummaryList nullable and update sample usage - Changed ProfilingSummary to allow nullable QuerySummaryList. - Updated NativeDebugResolver.Initialize to accept DLL path as parameter. - Enabled execution of multiple sample methods in Program.cs. --- .../Statistics/Summary/ProfilingSummary.cs | 2 +- DuckDB.NET.Samples/NativeDebugResolver.cs | 5 ++-- DuckDB.NET.Samples/Program.cs | 24 +++++++------------ 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs index 4c79bbed..5dfc8969 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs @@ -6,7 +6,7 @@ public readonly record struct ProfilingSummary( double ConnectionTimeMilliseconds, double ExecutionTimeMilliseconds, int QueryCount, - ProfilingQuerySummary[] QuerySummaryList) + ProfilingQuerySummary[]? QuerySummaryList) { public IDictionary ToDictionary() { diff --git a/DuckDB.NET.Samples/NativeDebugResolver.cs b/DuckDB.NET.Samples/NativeDebugResolver.cs index 55ec933f..0428e72e 100644 --- a/DuckDB.NET.Samples/NativeDebugResolver.cs +++ b/DuckDB.NET.Samples/NativeDebugResolver.cs @@ -10,13 +10,12 @@ internal static class NativeDebugResolver private static IntPtr _duckdbHandle; private static bool _initialized; - public static void Initialize() + public static void Initialize(string DuckDBDebugdllPath) { if (_initialized) return; _initialized = true; - //string repo = @"C:\Sources\Git\duckdb_scalarfs"; - string repo = @"C:\Users\Lucap\Downloads\duckdb 5"; + string repo = DuckDBDebugdllPath; string duckdbDir = Path.Combine(repo, "build", "debug", "src", "Debug"); string duckdbPath = Path.Combine(duckdbDir, "duckdb.dll"); diff --git a/DuckDB.NET.Samples/Program.cs b/DuckDB.NET.Samples/Program.cs index 8eb64411..f9555336 100644 --- a/DuckDB.NET.Samples/Program.cs +++ b/DuckDB.NET.Samples/Program.cs @@ -32,17 +32,17 @@ static void Main(string[] args) PrintVersion(); - //DapperSample(); + DapperSample(); - //AdoNetSamples(); + AdoNetSamples(); - //LowLevelBindingsSample(); + LowLevelBindingsSample(); - //BulkDataLoad(); + BulkDataLoad(); - //ParametersBinding(); + ParametersBinding(); - Profiling(); + //Profiling(); } private static void PrintVersion() @@ -275,9 +275,6 @@ private static void Profiling() con.Open(); - var before = con.RetrieveStatistics(); - var beforeCount = before.QuerySummaryList.Length; - var id = Guid.NewGuid().ToString("N"); using var dBCommand = con.CreateCommand(); @@ -289,8 +286,6 @@ private static void Profiling() using var cmd = con.CreateCommand(); cmd.CommandText = "SELECT a:1;"; using var reader = cmd.ExecuteReader(); - //while (reader.Read()) { }; - //while (reader.NextResult()) { }; var metrics = con.RetrieveStatistics(); @@ -299,18 +294,15 @@ private static void Profiling() metrics = con.RetrieveStatistics(); - Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); + Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList?.Length}"); cmd.CommandText = $"PRAGMA tpch(2); CREATE TABLE test (id int);"; cmd.ExecuteNonQuery(); metrics = con.RetrieveStatistics(); - Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList.Length}"); + Console.WriteLine($"QuerySummaryList: {metrics.QuerySummaryList?.Length}"); PrintMetrics(metrics); - - //PrintQueryResults(reader); - } private static void LoadTpch(DuckDBConnection connection, int scaleFactor = 1) From ed53afa3836a5e8753736a5856e67df432d7172f Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 09:44:27 +0200 Subject: [PATCH 39/43] Add Irion.DuckDB.NET.Benchmark project to solution Added the Irion.DuckDB.NET.Benchmark project to DuckDB.NET.sln, including its project reference and build configuration mappings for Debug and Release modes. --- DuckDB.NET.sln | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DuckDB.NET.sln b/DuckDB.NET.sln index 76619fcc..e7df9322 100644 --- a/DuckDB.NET.sln +++ b/DuckDB.NET.sln @@ -20,6 +20,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test", "DuckDB.NET.Test\Tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Irion.DuckDB.NET.Test", "Irion.DuckDB.NET.Test\Irion.DuckDB.NET.Test.csproj", "{6D707558-05DF-4DA6-9203-F79BA634D4D2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Irion.DuckDB.NET.Benchmark", "Irion.DuckDB.NET.Benchmark\Irion.DuckDB.NET.Benchmark.csproj", "{7A7A7A7A-7777-7777-7777-777777777777}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {6D707558-05DF-4DA6-9203-F79BA634D4D2}.Debug|Any CPU.Build.0 = Debug|Any CPU {6D707558-05DF-4DA6-9203-F79BA634D4D2}.Release|Any CPU.ActiveCfg = Release|Any CPU {6D707558-05DF-4DA6-9203-F79BA634D4D2}.Release|Any CPU.Build.0 = Release|Any CPU + {7A7A7A7A-7777-7777-7777-777777777777}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A7A7A7A-7777-7777-7777-777777777777}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A7A7A7A-7777-7777-7777-777777777777}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A7A7A7A-7777-7777-7777-777777777777}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From a66c251eaf598bcf2fc17667556e0a607b4c97ff Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 10:32:34 +0200 Subject: [PATCH 40/43] Enhance native library download process with configurable retries and delays; update profiling summary to handle null QuerySummaryList. --- DuckDB.NET.Bindings/Bindings.csproj | 2 +- .../DownloadNativeLibs.targets | 13 ++- .../Statistics/Summary/ProfilingSummary.cs | 2 +- README.md | 6 +- build/irion.version | 2 +- scripts/irion-package.ps1 | 79 +++++++++++++++---- 6 files changed, 80 insertions(+), 24 deletions(-) diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index eb7dc5fa..927d520c 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -41,7 +41,7 @@ <_NativeLib Include="linux-x64" LibUrl="$(DuckDbArtifactRoot)/duckdb-binaries-linux-amd64.zip" InnerZipName="libduckdb-linux-amd64.zip" Condition="$([MSBuild]::IsOSPlatform('Linux'))" /> <_NativeLib Include="osx" LibUrl="$(DuckDbArtifactRoot)/duckdb-binaries-osx.zip" InnerZipName="libduckdb-osx-universal.zip" Condition="$([MSBuild]::IsOSPlatform('OSX'))" /> - + diff --git a/DuckDB.NET.Bindings/DownloadNativeLibs.targets b/DuckDB.NET.Bindings/DownloadNativeLibs.targets index b344ff87..4ddbb86b 100644 --- a/DuckDB.NET.Bindings/DownloadNativeLibs.targets +++ b/DuckDB.NET.Bindings/DownloadNativeLibs.targets @@ -1,5 +1,10 @@  + + 5 + 1000 + + @@ -18,8 +23,8 @@ diff --git a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs index 5dfc8969..17b81721 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/Summary/ProfilingSummary.cs @@ -17,7 +17,7 @@ public IDictionary ToDictionary() ["ConnectionTime"] = ConnectionTimeMilliseconds, ["ExecutionTime"] = ExecutionTimeMilliseconds, ["QueryCount"] = QueryCount, - ["QuerySummaryList"] = QuerySummaryList + ["QuerySummaryList"] = QuerySummaryList ?? Array.Empty() }; } } diff --git a/README.md b/README.md index f170b620..a057ef30 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,13 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.1 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.2 ``` Bump version in `build/irion.version`: ```powershell -.\scripts\irion-package.ps1 -Command bump -Part prerelease # 1.4.4-alpha.1 -> 1.4.4-alpha.2 +.\scripts\irion-package.ps1 -Command bump -Part prerelease # 1.4.4.0-alpha.1 -> 1.4.4.0-alpha.2 # .\scripts\irion-package.ps1 -Command bump -Part revision # 1.4.4.1 -> 1.4.4.2 # .\scripts\irion-package.ps1 -Command bump -Part build # 1.4.4.1 -> 1.4.5.0 # .\scripts\irion-package.ps1 -Command bump -Part minor # 1.4.4.1 -> 1.5.0.0 @@ -128,7 +128,7 @@ You can override package IDs if needed: #### 3. Build (Full) ```powershell -.\scripts\irion-package.ps1 -Command build +.\scripts\irion-package.ps1 -Command build -p "NativeDownloadRetries=30","NativeDownloadRetryDelayMilliseconds=15000" ``` The build command runs a full `dotnet clean` for both Irion projects before building. diff --git a/build/irion.version b/build/irion.version index 4d42b2c6..03043cd9 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-rc.1 \ No newline at end of file +1.5.2.2-rc.2 \ No newline at end of file diff --git a/scripts/irion-package.ps1 b/scripts/irion-package.ps1 index a230af6a..7a42a430 100644 --- a/scripts/irion-package.ps1 +++ b/scripts/irion-package.ps1 @@ -8,6 +8,9 @@ param( [ValidateSet("major", "minor", "build", "revision", "prerelease")] [string]$Part = "revision", + [Alias('p')] + [string[]]$MsBuildProperty, + [string]$VersionFile = "build/irion.version", [string]$Configuration = "Release", [string]$NuGetSource = "Repository", @@ -307,83 +310,131 @@ function Set-PackageVersionEnvironment { Write-Host "DUCKDB_VERSION_BUILD=$env:DUCKDB_VERSION_BUILD" } +function Get-MsBuildPropertyArguments { + if ($null -eq $MsBuildProperty -or $MsBuildProperty.Count -eq 0) { + return @() + } + + $args = New-Object System.Collections.Generic.List[string] + foreach ($property in $MsBuildProperty) { + $trimmed = $property.Trim() + if ([string]::IsNullOrWhiteSpace($trimmed)) { + continue + } + + if ($trimmed.StartsWith("/p:") -or $trimmed.StartsWith("-p:")) { + [void]$args.Add($trimmed) + } + else { + [void]$args.Add("/p:$trimmed") + } + } + + return @($args) +} + function Invoke-Clean { param([Parameter(Mandatory = $true)][string]$PackageVersion) $projects = Get-ProjectPaths + $msBuildPropertyArgs = Get-MsBuildPropertyArguments Set-PackageVersionEnvironment -PackageVersion $PackageVersion - Invoke-DotNet -Arguments @( + $bindingsArgs = @( "clean", $projects.Bindings, "-c", $Configuration, "/p:BuildType=Full" ) + $bindingsArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $bindingsArgs - Invoke-DotNet -Arguments @( + $dataArgs = @( "clean", $projects.Data, "-c", $Configuration, "/p:BuildType=Full" ) + $dataArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $dataArgs } function Invoke-Build { param([Parameter(Mandatory = $true)][string]$PackageVersion) $projects = Get-ProjectPaths + $msBuildPropertyArgs = Get-MsBuildPropertyArguments + $versionParts = Split-SemVersion -VersionString $PackageVersion + $assemblyVersion = $versionParts.Numeric.ToString(4) $nuGetPackageVersion = Get-NuGetPackageVersion -VersionString $PackageVersion Set-PackageVersionEnvironment -PackageVersion $PackageVersion + Write-Host "AssemblyVersion=$assemblyVersion" Write-Host "NuGetPackageVersion=$nuGetPackageVersion" - Invoke-DotNet -Arguments @( + $bindingsArgs = @( "build", $projects.Bindings, "-c", $Configuration, "/p:BuildType=Full", - "/p:Version=$PackageVersion", - "/p:FileVersion=$PackageVersion", + "/p:Version=$assemblyVersion", + "/p:FileVersion=$assemblyVersion", + "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + $bindingsArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $bindingsArgs - Invoke-DotNet -Arguments @( + $dataArgs = @( "build", $projects.Data, "-c", $Configuration, "/p:BuildType=Full", - "/p:Version=$PackageVersion", - "/p:FileVersion=$PackageVersion", + "/p:Version=$assemblyVersion", + "/p:FileVersion=$assemblyVersion", + "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + $dataArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $dataArgs } function Invoke-Pack { param([Parameter(Mandatory = $true)][string]$PackageVersion) $projects = Get-ProjectPaths + $msBuildPropertyArgs = Get-MsBuildPropertyArguments + $versionParts = Split-SemVersion -VersionString $PackageVersion + $assemblyVersion = $versionParts.Numeric.ToString(4) $nuGetPackageVersion = Get-NuGetPackageVersion -VersionString $PackageVersion Set-PackageVersionEnvironment -PackageVersion $PackageVersion + Write-Host "AssemblyVersion=$assemblyVersion" Write-Host "NuGetPackageVersion=$nuGetPackageVersion" - Invoke-DotNet -Arguments @( + $bindingsArgs = @( "pack", $projects.Bindings, "-c", $Configuration, "/p:BuildType=Full", - "/p:Version=$PackageVersion", - "/p:FileVersion=$PackageVersion", + "/p:Version=$assemblyVersion", + "/p:FileVersion=$assemblyVersion", + "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + $bindingsArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $bindingsArgs - Invoke-DotNet -Arguments @( + $dataArgs = @( "pack", $projects.Data, "-c", $Configuration, "/p:BuildType=Full", - "/p:Version=$PackageVersion", - "/p:FileVersion=$PackageVersion", + "/p:Version=$assemblyVersion", + "/p:FileVersion=$assemblyVersion", + "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + $dataArgs += $msBuildPropertyArgs + Invoke-DotNet -Arguments $dataArgs } function Invoke-Push { From a51f1a7d08ae646a1d57abf226f2da7d0d217966 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 11:49:20 +0200 Subject: [PATCH 41/43] Fix target framework version in benchmark project and improve version bumping logic in packaging script --- .../Irion.DuckDB.NET.Benchmark.csproj | 2 +- scripts/irion-package.ps1 | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj index c7efbb18..a56352d7 100644 --- a/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj +++ b/Irion.DuckDB.NET.Benchmark/Irion.DuckDB.NET.Benchmark.csproj @@ -1,7 +1,7 @@  - net10.0;net8 + net10.0;net8.0 Exe Irion.DuckDB.NET.Benchmark Irion.DuckDB.NET.Benchmark diff --git a/scripts/irion-package.ps1 b/scripts/irion-package.ps1 index 7a42a430..ed7c2ca8 100644 --- a/scripts/irion-package.ps1 +++ b/scripts/irion-package.ps1 @@ -180,9 +180,9 @@ function Get-BumpedVersion { else { $label = $s.Substring(0, $lastDot) $tail = $s.Substring($lastDot + 1) - if ([int]::TryParse($tail, [ref]$null)) { - $num = [int]$tail - $newSuffix = "$label.$($num + 1)" + $tailNum = 0 + if ([int]::TryParse($tail, [ref]$tailNum)) { + $newSuffix = "$label.$($tailNum + 1)" } else { # tail is not numeric; append .1 From 255cb9507cb53c070dd4883c4121165c666d056b Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 14:39:55 +0200 Subject: [PATCH 42/43] Add support for including release notes in package metadata and enhance profiling features --- Directory.Build.props | 16 +++++ .../PreparedStatement/PreparedStatement.cs | 4 +- .../Profiling/DuckDBProfilingInfoWrapper.cs | 1 - .../Profiling/Statistics/QueryProfiler.cs | 4 +- README.md | 9 ++- RELEASE-NOTE.md | 22 +++++++ build/irion.version | 2 +- scripts/irion-package.ps1 | 65 ++++++++++++++++--- 8 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 RELEASE-NOTE.md diff --git a/Directory.Build.props b/Directory.Build.props index ed73ab1d..eb286756 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -65,6 +65,22 @@ + + + false + + + + RELEASE-NOTE.md + + + + + True + + + + false diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index 22539cf8..2733a9ae 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,6 +1,5 @@ using DuckDB.NET.Data.Connection; using DuckDB.NET.Data.Profiling.Statistics; -using Microsoft.VisualBasic; using System.Linq; namespace DuckDB.NET.Data.PreparedStatement; @@ -36,7 +35,8 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c throw new DuckDBException(error); } - try { + try + { for (int index = 0; index < statementCount; index++) { diff --git a/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs index d6e4d97a..cacc32b3 100644 --- a/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs +++ b/DuckDB.NET.Data/Profiling/DuckDBProfilingInfoWrapper.cs @@ -19,7 +19,6 @@ public DuckDBProfilingInfoWrapper(DuckDBProfilingInfo handle) var handle = NativeMethods.ProfilingInfo.DuckDBGetProfilingInfo(connection); if (handle.IsInvalid || handle.IsClosed) { - Console.WriteLine("No profiling info"); return null; } return new DuckDBProfilingInfoWrapper(handle); diff --git a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs index 76cd3800..89ee2507 100644 --- a/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs +++ b/DuckDB.NET.Data/Profiling/Statistics/QueryProfiler.cs @@ -4,7 +4,7 @@ namespace DuckDB.NET.Data.Profiling.Statistics; -internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, DuckDBNativeConnection connection, ProfilingOptions? profilingOptions) +internal sealed class QueryProfiler(int queryIdentifier, int statementCount, DuckDBNativeConnection connection, ProfilingOptions? profilingOptions) : ExecutionProfiler(connection, profilingOptions) { private readonly StatementProfiler[] statementProfilers = new StatementProfiler[statementCount]; @@ -12,7 +12,7 @@ internal sealed class QueryProfiler(IntPtr queryIdentifier, int statementCount, internal int StatementCount => statementCount; - internal IntPtr QueryIdentifier => queryIdentifier; + internal int QueryIdentifier => queryIdentifier; internal StatementProfiler CreateStatementProfiler(int index) { diff --git a/README.md b/README.md index a057ef30..36db77b7 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.2 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.3 ``` Bump version in `build/irion.version`: @@ -152,6 +152,13 @@ Generated packages: - `DuckDB.NET.Bindings/bin/Release/Irion.DuckDB.NET.Bindings.Full..nupkg` - `DuckDB.NET.Data/bin/Release/Irion.DuckDB.NET.Data.Full..nupkg` +if `-PackageReleaseNotes` or `-PackageReleaseNotesFile` are provided, the content is included in the generated `.nupkg` metadata and visible on NuGet.org. + +```powershell +.\scripts\irion-package.ps1 -Command pack -PackageReleaseNotes "Introduced support for DuckDB ProfilingInfo feature." -PackageReleaseNotesFile "RELEASE-NOTE.md" +``` + + #### 5. Push packages to feed ```powershell diff --git a/RELEASE-NOTE.md b/RELEASE-NOTE.md new file mode 100644 index 00000000..899ff240 --- /dev/null +++ b/RELEASE-NOTE.md @@ -0,0 +1,22 @@ +New features: + +- Added support for DuckDB's `ProfilingInfo` feature, allowing retrieval of detailed query execution profiles. + +Quick snippet — enable profiling, run a query, read summary, then disable/reset: + +```csharp +// Enable profiling (in-memory JSON summary) +connection.EnableProfiling(new ProfilingOptions { Coverage = DuckDBProfilingCoverage.All, Format = DuckDBProfilingFormat.Json }); + +// Run a query +using var cmd = connection.CreateCommand(); +cmd.CommandText = "SELECT 1;"; +using var r = cmd.ExecuteReader(); + +// Retrieve summary +var summary = connection.RetrieveStatistics(); +Console.WriteLine($"Queries collected: {summary.QueryCount}"); + +// Disable and reset +connection.DisableProfiling(true); +``` \ No newline at end of file diff --git a/build/irion.version b/build/irion.version index 03043cd9..43000e2d 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-rc.2 \ No newline at end of file +1.5.2.2-rc.3 \ No newline at end of file diff --git a/scripts/irion-package.ps1 b/scripts/irion-package.ps1 index ed7c2ca8..a96615e9 100644 --- a/scripts/irion-package.ps1 +++ b/scripts/irion-package.ps1 @@ -11,6 +11,9 @@ param( [Alias('p')] [string[]]$MsBuildProperty, + [string]$PackageReleaseNotes, + [string]$PackageReleaseNotesFile, + [string]$VersionFile = "build/irion.version", [string]$Configuration = "Release", [string]$NuGetSource = "Repository", @@ -315,7 +318,7 @@ function Get-MsBuildPropertyArguments { return @() } - $args = New-Object System.Collections.Generic.List[string] + $propertyArgs = New-Object System.Collections.Generic.List[string] foreach ($property in $MsBuildProperty) { $trimmed = $property.Trim() if ([string]::IsNullOrWhiteSpace($trimmed)) { @@ -323,14 +326,14 @@ function Get-MsBuildPropertyArguments { } if ($trimmed.StartsWith("/p:") -or $trimmed.StartsWith("-p:")) { - [void]$args.Add($trimmed) + [void]$propertyArgs.Add($trimmed) } else { - [void]$args.Add("/p:$trimmed") + [void]$propertyArgs.Add("/p:$trimmed") } } - return @($args) + return @($propertyArgs) } function Invoke-Clean { @@ -346,6 +349,17 @@ function Invoke-Clean { "-c", $Configuration, "/p:BuildType=Full" ) + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotes)) { + $bindingsArgs += "/p:PackageReleaseNotes=$PackageReleaseNotes" + } + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotesFile)) { + $resolved = Resolve-RepoPath -Path $PackageReleaseNotesFile + if (-not (Test-Path $resolved)) { throw "PackageReleaseNotesFile not found: $resolved" } + $leaf = Split-Path -Leaf $resolved + $bindingsArgs += "/p:IncludeReleaseNotesFromRepoRoot=true" + $bindingsArgs += "/p:PackageReadmeFile=$leaf" + } + $bindingsArgs += $msBuildPropertyArgs Invoke-DotNet -Arguments $bindingsArgs @@ -355,6 +369,17 @@ function Invoke-Clean { "-c", $Configuration, "/p:BuildType=Full" ) + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotes)) { + $dataArgs += "/p:PackageReleaseNotes=$PackageReleaseNotes" + } + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotesFile)) { + $resolved = Resolve-RepoPath -Path $PackageReleaseNotesFile + if (-not (Test-Path $resolved)) { throw "PackageReleaseNotesFile not found: $resolved" } + $leaf = Split-Path -Leaf $resolved + $dataArgs += "/p:IncludeReleaseNotesFromRepoRoot=true" + $dataArgs += "/p:PackageReadmeFile=$leaf" + } + $dataArgs += $msBuildPropertyArgs Invoke-DotNet -Arguments $dataArgs } @@ -420,6 +445,17 @@ function Invoke-Pack { "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotes)) { + $bindingsArgs += "/p:PackageReleaseNotes=$PackageReleaseNotes" + } + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotesFile)) { + $resolved = Resolve-RepoPath -Path $PackageReleaseNotesFile + if (-not (Test-Path $resolved)) { throw "PackageReleaseNotesFile not found: $resolved" } + $repoReleaseNote = Join-Path $repoRoot 'RELEASE-NOTE.md' + if ($resolved -ne $repoReleaseNote) { throw "PackageReleaseNotesFile must be RELEASE-NOTE.md at the repository root for props-based inclusion. Move it to repo root or omit this parameter." } + $bindingsArgs += "/p:IncludeReleaseNotesFromRepoRoot=true" + $bindingsArgs += "/p:PackageReadmeFile=RELEASE-NOTE.md" + } $bindingsArgs += $msBuildPropertyArgs Invoke-DotNet -Arguments $bindingsArgs @@ -433,6 +469,17 @@ function Invoke-Pack { "/p:InformationalVersion=$PackageVersion", "/p:PackageVersion=$nuGetPackageVersion" ) + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotes)) { + $dataArgs += "/p:PackageReleaseNotes=$PackageReleaseNotes" + } + if (-not [string]::IsNullOrWhiteSpace($PackageReleaseNotesFile)) { + $resolved = Resolve-RepoPath -Path $PackageReleaseNotesFile + if (-not (Test-Path $resolved)) { throw "PackageReleaseNotesFile not found: $resolved" } + $repoReleaseNote = Join-Path $repoRoot 'RELEASE-NOTE.md' + if ($resolved -ne $repoReleaseNote) { throw "PackageReleaseNotesFile must be RELEASE-NOTE.md at the repository root for props-based inclusion. Move it to repo root or omit this parameter." } + $dataArgs += "/p:IncludeReleaseNotesFromRepoRoot=true" + $dataArgs += "/p:PackageReadmeFile=RELEASE-NOTE.md" + } $dataArgs += $msBuildPropertyArgs Invoke-DotNet -Arguments $dataArgs } @@ -465,7 +512,7 @@ function Invoke-Push { function Get-RemoteVersions { param([Parameter(Mandatory = $true)][string]$PackageId) - $args = @( + $propertyArgs = @( "package", "search", $PackageId, @@ -476,18 +523,18 @@ function Get-RemoteVersions { ) if ($IncludePrerelease) { - $args += "--prerelease" + $propertyArgs += "--prerelease" } if ($Interactive) { - $args += "--interactive" + $propertyArgs += "--interactive" } if (-not [string]::IsNullOrWhiteSpace($ConfigFile)) { - $args += @("--configfile", (Resolve-RepoPath -Path $ConfigFile)) + $propertyArgs += @("--configfile", (Resolve-RepoPath -Path $ConfigFile)) } - $output = Invoke-DotNetCapture -Arguments $args + $output = Invoke-DotNetCapture -Arguments $propertyArgs $outputText = ($output -join [Environment]::NewLine).Trim() $jsonStart = $outputText.IndexOf("{") if ($jsonStart -lt 0) { From 2a5875a70125f276cf809e826810b37b9da5e503 Mon Sep 17 00:00:00 2001 From: Luca Prezzi Date: Mon, 8 Jun 2026 15:10:43 +0200 Subject: [PATCH 43/43] Update version to 1.5.2.2 and enhance release notes for ProfilingInfo feature --- README.md | 4 +-- RELEASE-NOTE.md | 80 +++++++++++++++++++++++++++++++++++++-------- build/irion.version | 2 +- 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 36db77b7..3f91aef2 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Current version: Set explicit version: ```powershell -.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2-rc.3 +.\scripts\irion-package.ps1 -Command set -Version 1.5.2.2 ``` Bump version in `build/irion.version`: @@ -155,7 +155,7 @@ Generated packages: if `-PackageReleaseNotes` or `-PackageReleaseNotesFile` are provided, the content is included in the generated `.nupkg` metadata and visible on NuGet.org. ```powershell -.\scripts\irion-package.ps1 -Command pack -PackageReleaseNotes "Introduced support for DuckDB ProfilingInfo feature." -PackageReleaseNotesFile "RELEASE-NOTE.md" +.\scripts\irion-package.ps1 -Command pack -PackageReleaseNotes "New feature short description" -PackageReleaseNotesFile "RELEASE-NOTE.md" ``` diff --git a/RELEASE-NOTE.md b/RELEASE-NOTE.md index 899ff240..5e9f8db6 100644 --- a/RELEASE-NOTE.md +++ b/RELEASE-NOTE.md @@ -1,22 +1,74 @@ -New features: +New functionality -- Added support for DuckDB's `ProfilingInfo` feature, allowing retrieval of detailed query execution profiles. +- ProfilingInfo: collect detailed per-query execution profiles and metrics (timing, operator breakdown, runtime statistics). -Quick snippet — enable profiling, run a query, read summary, then disable/reset: +How to enable + +1. Enable profiling on an open `DuckDBConnection` with `ProfilingOptions`: ```csharp -// Enable profiling (in-memory JSON summary) -connection.EnableProfiling(new ProfilingOptions { Coverage = DuckDBProfilingCoverage.All, Format = DuckDBProfilingFormat.Json }); +connection.EnableProfiling(new ProfilingOptions { + Coverage = DuckDBProfilingCoverage.All, + Format = DuckDBProfilingFormat.Json, + Mode = DuckDBProfilingMode.Standard, + //OutputPath = "profiling-output.json", // optional path for generated files + // EnabledMetrics = new DuckDBMetricTypeCollection(...), // optional: pick specific metrics + MetricsThreshold = 5 // optional: ignore measurements under X ms +}); +``` + +2. Run queries as usual using `CreateCommand()` / `ExecuteReader()` / `ExecuteNonQuery()`. + +Retrieve summaries -// Run a query -using var cmd = connection.CreateCommand(); -cmd.CommandText = "SELECT 1;"; -using var r = cmd.ExecuteReader(); +- Call `connection.RetrieveStatistics()` to obtain the profiling summary. The returned object includes: + - `QueryCount` and `QuerySummaryList` — overall counts and per-query summaries + - Per-query timings, operator breakdowns, and collected metrics -// Retrieve summary +Example (read summary): + +```csharp var summary = connection.RetrieveStatistics(); -Console.WriteLine($"Queries collected: {summary.QueryCount}"); +Console.WriteLine($"Collected {summary.QueryCount} queries"); +foreach (var q in summary.QuerySummaryList ?? Enumerable.Empty()) +{ + var firstStatement = q.StatementSummaries != null && q.StatementSummaries.Length > 0 + ? q.StatementSummaries[0].Metrics[MetricsType.QueryName] + : "(no statement)"; + + Console.WriteLine($"Query: {firstStatement} — Duration: {q.ExecutionTimeMilliseconds} ms"); +} +``` + +Disable and reset + +- To stop profiling and clear collected data: + +```csharp +connection.DisableProfiling(reset: true); +``` + +Notes + +- `MetricsThreshold` helps filter noise by ignoring very short measurements. +- `QuerySummaryList` may be null when no queries were captured; use `?? Enumerable.Empty()` when enumerating. + +ProfilingOptions fields + +- `Coverage` (enum: `DuckDBProfilingCoverage`): controls which queries/operators are profiled. Valid values: + - `Select` — profile SELECT queries (default for fine-grained sampling of reads). + - `All` — profile all query kinds and collect operator-level details. + +- `Format` (enum: `DuckDBProfilingFormat`): output format for summaries. Valid values: + - `QueryTree` — tree-style query plan output (human-readable; use this for a "Text" style summary). + - `Json` — structured JSON suitable for storage and programmatic analysis. + - `QueryTreeOptimizer` — query-tree output focused on optimizer phases. + - `NoOutput` — disable textual/report output while still collecting metrics. + +- `Mode` (enum: `DuckDBProfilingMode`): optional mode selector controlling standard vs detailed captures: + - `Standard`, `Detailed`, `All` — choose the level of internal detail captured. + -// Disable and reset -connection.DisableProfiling(true); -``` \ No newline at end of file +- `MetricsThreshold` (int): optional threshold in milliseconds; measurements shorter than this are ignored to reduce noise. +- `OutputPath` (string): optional file path for query plan output. +- `EnabledMetrics` (DuckDBMetricTypeCollection): optional collection of metric types to collect; omit to use the library defaults. diff --git a/build/irion.version b/build/irion.version index 43000e2d..45cbcf54 100644 --- a/build/irion.version +++ b/build/irion.version @@ -1 +1 @@ -1.5.2.2-rc.3 \ No newline at end of file +1.5.2.2 \ No newline at end of file