From 3c436eeba5f9ff5bb2b49672bdf11d03346e4219 Mon Sep 17 00:00:00 2001 From: Joel Christner Date: Wed, 29 Jul 2026 08:12:55 -0700 Subject: [PATCH 1/3] Fix transient accept reset handling --- CHANGELOG.md | 11 +++- README.md | 12 ++++ src/Test.Shared/WatsonTcpScenarios.cs | 74 +++++++++++++++++++++ src/WatsonTcp/WatsonTcp.csproj | 4 +- src/WatsonTcp/WatsonTcp.xml | 7 ++ src/WatsonTcp/WatsonTcpServer.cs | 93 ++++++++++++++++++--------- 6 files changed, 166 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f35d41..c05fa1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Current Version +v6.3.2 + +### Listener Reliability + +- Continued accepting new clients after transient accept-time `SocketError.ConnectionReset` or `SocketError.ConnectionAborted` exceptions +- Added regression coverage for accept-loop recovery after a reset before a client is fully established + +## Previous Version + v6.3.1 ### Performance @@ -29,7 +38,7 @@ v6.3.1 - This is a patch release focused on internal performance and benchmarking improvements - Public APIs remain compatible; behavior-sensitive liveness probing changes were limited to internal hot-path detection strategy -## Previous Version +## Earlier Version v6.3.0 diff --git a/README.md b/README.md index e40aa12..4b6ccd1 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ Special thanks to the following people for their support and contributions to th If you'd like to contribute, please jump right into the source code and create a pull request, or, file an issue with your enhancement request. +## New in v6.3.2 + +### Listener Reliability + +WatsonTcp now keeps the server listener running when the underlying TCP accept call hits a transient connection reset or abort before a client is fully established. + +Key improvements include: + +- recover from accept-time `SocketError.ConnectionReset` and `SocketError.ConnectionAborted` without stopping the accept loop +- preserve exception reporting through `ExceptionEncountered` and add warning-level listener logging +- add regression coverage for accept-loop recovery after a reset + ## New in v6.3.1 ### Performance diff --git a/src/Test.Shared/WatsonTcpScenarios.cs b/src/Test.Shared/WatsonTcpScenarios.cs index 3c5f441..872818b 100644 --- a/src/Test.Shared/WatsonTcpScenarios.cs +++ b/src/Test.Shared/WatsonTcpScenarios.cs @@ -125,6 +125,39 @@ private static bool LogContains(IEnumerable messages, string fragment) return messages.Any(msg => msg != null && msg.IndexOf(fragment, StringComparison.OrdinalIgnoreCase) >= 0); } + private sealed class TransientAcceptFailureServer : WatsonTcpServer + { + private readonly SocketException _exception; + private int _remainingFailures; + private int _injectedFailures; + + internal TransientAcceptFailureServer(string listenerIp, int listenerPort, SocketException exception, int failureCount = 1) + : base(listenerIp, listenerPort) + { + _exception = exception ?? throw new ArgumentNullException(nameof(exception)); + _remainingFailures = failureCount; + } + + internal int InjectedFailures + { + get + { + return _injectedFailures; + } + } + + protected override async Task AcceptTcpClientAsync(CancellationToken token) + { + if (Interlocked.Decrement(ref _remainingFailures) >= 0) + { + Interlocked.Increment(ref _injectedFailures); + throw _exception; + } + + return await base.AcceptTcpClientAsync(token).ConfigureAwait(false); + } + } + private static async Task WaitForConditionAsync(Func condition, int timeoutMs = DefaultConditionTimeoutMs, string failureMessage = null) { if (condition == null) throw new ArgumentNullException(nameof(condition)); @@ -219,6 +252,47 @@ public static async Task BasicClientConnection() SafeDispose(server); } } + public static async Task ServerContinuesAcceptingAfterConnectionResetDuringAccept() + { + int port = GetNextPort(); + List logs = CreateLogCapture(out Action logger); + var server = new TransientAcceptFailureServer(_hostname, port, new SocketException((int)SocketError.ConnectionReset)); + server.Settings.Logger = logger; + SetupDefaultServerHandlers(server); + + int exceptionCount = 0; + server.Events.ExceptionEncountered += (s, e) => + { + if (e.Exception is SocketException socketException + && socketException.SocketErrorCode == SocketError.ConnectionReset) + { + Interlocked.Increment(ref exceptionCount); + } + }; + + WatsonTcpClient client = null; + + try + { + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + TestAssert.True(client.Connected); + TestAssert.Equal(1, server.InjectedFailures, "Expected the accept loop test server to inject one transient failure."); + TestAssert.Equal(1, exceptionCount, "Expected the transient accept exception to be reported once."); + TestAssert.True(LogContains(logs, "transient listener exception while accepting connection"), "Expected a warning log for the transient accept failure."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + } + } public static async Task ClientServerConnection() { int port = GetNextPort(); diff --git a/src/WatsonTcp/WatsonTcp.csproj b/src/WatsonTcp/WatsonTcp.csproj index f1e9362..2f88ad9 100644 --- a/src/WatsonTcp/WatsonTcp.csproj +++ b/src/WatsonTcp/WatsonTcp.csproj @@ -7,7 +7,7 @@ true latest-recommended true - 6.3.1 + 6.3.2 Joel Christner Joel Christner A simple C# async TCP server and client with integrated framing for reliable transmission and receipt of data @@ -17,7 +17,7 @@ https://github.com/dotnet/WatsonTcp Github - Performance-focused release with lower-allocation framing and stream handling, buffered header parsing, reduced disconnect hot-path overhead, internal async connection setup, and an automated performance benchmark suite. + Fixes a listener reliability issue where transient accept-time connection reset or abort socket errors could stop the server accept loop. LICENSE.md watson.png diff --git a/src/WatsonTcp/WatsonTcp.xml b/src/WatsonTcp/WatsonTcp.xml index 03411e1..a8e07ba 100644 --- a/src/WatsonTcp/WatsonTcp.xml +++ b/src/WatsonTcp/WatsonTcp.xml @@ -1635,6 +1635,13 @@ Indicate if resources should be disposed. + + + Accept a TCP client from the listener. + + Cancellation token used to stop accepting clients. + The accepted TCP client. + Watson TCP server callbacks. diff --git a/src/WatsonTcp/WatsonTcpServer.cs b/src/WatsonTcp/WatsonTcpServer.cs index 76453ec..23aa8ef 100644 --- a/src/WatsonTcp/WatsonTcpServer.cs +++ b/src/WatsonTcp/WatsonTcpServer.cs @@ -680,10 +680,10 @@ protected virtual void Dispose(bool disposing) #region Connection - private void EnableKeepalives(TcpClient client) - { - // issues with definitions: https://github.com/dotnet/sdk/issues/14540 - + private void EnableKeepalives(TcpClient client) + { + // issues with definitions: https://github.com/dotnet/sdk/issues/14540 + try { #if NET6_0_OR_GREATER @@ -717,13 +717,28 @@ private void EnableKeepalives(TcpClient client) catch (Exception) { _Settings.Logger?.Invoke(Severity.Error, _Header + "keepalives not supported on this platform, disabled"); - _Keepalive.EnableTcpKeepAlives = false; - } - } - - private async Task AcceptConnections(CancellationToken token) - { - _IsListening = true; + _Keepalive.EnableTcpKeepAlives = false; + } + } + + /// + /// Accept a TCP client from the listener. + /// + /// Cancellation token used to stop accepting clients. + /// The accepted TCP client. + protected virtual async Task AcceptTcpClientAsync(CancellationToken token) + { +#if NET6_0_OR_GREATER + return await _Listener.AcceptTcpClientAsync(token).ConfigureAwait(false); +#else + _ = token; + return await _Listener.AcceptTcpClientAsync().ConfigureAwait(false); +#endif + } + + private async Task AcceptConnections(CancellationToken token) + { + _IsListening = true; while (true) { @@ -744,17 +759,24 @@ private async Task AcceptConnections(CancellationToken token) _IsListening = true; } - #endregion - - #region Accept-and-Validate - -#if NET6_0_OR_GREATER - TcpClient tcpClient = await _Listener.AcceptTcpClientAsync(token).ConfigureAwait(false); -#else - TcpClient tcpClient = await _Listener.AcceptTcpClientAsync().ConfigureAwait(false); -#endif - tcpClient.LingerState.Enabled = false; - tcpClient.NoDelay = _Settings.NoDelay; + #endregion + + #region Accept-and-Validate + + TcpClient tcpClient; + try + { + tcpClient = await AcceptTcpClientAsync(token).ConfigureAwait(false); + } + catch (SocketException e) when (_IsListening && !token.IsCancellationRequested && IsTransientAcceptSocketException(e)) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "transient listener exception while accepting connection, continuing: " + e.SocketErrorCode + " (" + e.Message + ")"); + _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + continue; + } + + tcpClient.LingerState.Enabled = false; + tcpClient.NoDelay = _Settings.NoDelay; // Enforce max connections - reject if at capacity if (_Connections >= _Settings.MaxConnections && _Settings.EnforceMaxConnections) @@ -836,19 +858,26 @@ private async Task AcceptConnections(CancellationToken token) { break; } - catch (ObjectDisposedException) - { - break; - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "listener exception: " + e.Message); + catch (ObjectDisposedException) + { + break; + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "listener exception: " + e.Message); _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); break; } - } - } - + } + } + + private static bool IsTransientAcceptSocketException(SocketException e) + { + return e != null + && (e.SocketErrorCode == SocketError.ConnectionReset + || e.SocketErrorCode == SocketError.ConnectionAborted); + } + private async Task InitializeAcceptedSslClientAsync(ClientMetadata client, CancellationToken token) { bool success = await StartTls(client, token).ConfigureAwait(false); From 4d167a84f04c66ba52082f080f50d5cfbf4c75e1 Mon Sep 17 00:00:00 2001 From: Joel Christner Date: Wed, 29 Jul 2026 08:15:20 -0700 Subject: [PATCH 2/3] Add BrvSqr to contributors --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b6ccd1..fc59302 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Special thanks to the following people for their support and contributions to th @DuAell @syntacs @zsolt777 @broms95 @Antwns @MartyIX @Jyck @Memphizzz @nirajgenius @cee-sharp @jeverz @cbarraco @DenisBalan @Markonius @Ahmed310 @markashleybell @thechosensausage @JVemon @eatyouroats @bendablegears @Laiteux @fisherman6v6 @wesoos -@YorVeX @tovich37 @sancheolz @lunedis @ShayanFiroozi +@YorVeX @tovich37 @sancheolz @lunedis @ShayanFiroozi @BrvSqr If you'd like to contribute, please jump right into the source code and create a pull request, or, file an issue with your enhancement request. From a46812ed53d1b79d4706f1cdff4cd3dce56ebe51 Mon Sep 17 00:00:00 2001 From: Joel Christner Date: Sat, 1 Aug 2026 21:15:31 -0700 Subject: [PATCH 3/3] Add vendor-neutral telemetry (metrics + tracing) for v6.4.0 Emit metrics and distributed-tracing spans through System.Diagnostics.Metrics (Meter "WatsonTcp") and System.Diagnostics.ActivitySource (ActivitySource "WatsonTcp"), consumable by Radiant, the OpenTelemetry SDK, Prometheus, and others with no dependency on any telemetry backend. - Add public WatsonTcpMetrics contract (source/metric/span/tag-key constants) - Add internal WatsonTcpInstrumentation (per-instance Meter/ActivitySource, 24 metrics, 6 spans, observable gauges), disposed with its owner - Wire recording into client/server lifecycle: send/receive, connect/disconnect by reason, handshake, auth, authorization, sync request/response/timeout, exceptions (funnelled), transient accept errors, stream drain, uptime - Keep metric tags low-cardinality; put GUIDs/endpoints on spans only - Add Settings.EnableMetrics/EnableTracing (default true) to both settings - Reference System.Diagnostics.DiagnosticSource on down-level TFMs only - Add 17 positive/negative telemetry tests (BCL MeterListener/ActivityListener) and a telemetry suite; 126/126 pass on net8.0 and net10.0 - Bump version to 6.4.0; update README, CHANGELOG, ARCHITECTURE, CLAUDE - Replace TELEMETRY.md with a consumer integration guide; archive the plan Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01SrLigwKj3JqRXPzZZq8fU2 --- ARCHITECTURE.md | 11 + CHANGELOG.md | 699 ++++----- CLAUDE.md | 8 +- README.md | 841 ++++++----- TELEMETRY.md | 273 ++++ archive/TELEMETRY_PLAN.md | 563 +++++++ src/Test.Automated/Program.cs | 12 +- src/Test.Shared/MetricMeasurement.cs | 21 + src/Test.Shared/TelemetryCollector.cs | 233 +++ src/Test.Shared/WatsonTcpScenarios.cs | 632 +++++++- src/Test.Shared/WatsonTcpSuites.cs | 58 +- src/WatsonTcp/ClientMetadata.cs | 2 + src/WatsonTcp/WatsonTcp.csproj | 13 +- src/WatsonTcp/WatsonTcp.xml | 398 +++++ src/WatsonTcp/WatsonTcpClient.cs | 1666 +++++++++++---------- src/WatsonTcp/WatsonTcpClientSettings.cs | 75 +- src/WatsonTcp/WatsonTcpInstrumentation.cs | 489 ++++++ src/WatsonTcp/WatsonTcpMetrics.cs | 358 +++++ src/WatsonTcp/WatsonTcpServer.cs | 1635 ++++++++++---------- src/WatsonTcp/WatsonTcpServerSettings.cs | 107 +- 20 files changed, 5701 insertions(+), 2393 deletions(-) create mode 100644 TELEMETRY.md create mode 100644 archive/TELEMETRY_PLAN.md create mode 100644 src/Test.Shared/MetricMeasurement.cs create mode 100644 src/Test.Shared/TelemetryCollector.cs create mode 100644 src/WatsonTcp/WatsonTcpInstrumentation.cs create mode 100644 src/WatsonTcp/WatsonTcpMetrics.cs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 250b7d2..56b3678 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -534,3 +534,14 @@ Each connection has its own `SemaphoreSlim(1,1)` for write serialization. On the ### WatsonStream as Bounded Stream Wrapper `WatsonStream` wraps the raw TCP/SSL stream but tracks `_Position` and `_BytesRemaining` based on the declared `ContentLength`. This prevents the consumer from reading beyond the current message's data into the next message's header. The stream is read-only, non-seekable, and non-writable, reflecting its role as a view over a specific segment of the underlying transport stream. + +## 11. Telemetry + +WatsonTcp emits metrics and distributed-tracing spans using only the .NET base class library, so any host that speaks the BCL diagnostics APIs (Radiant, the OpenTelemetry SDK, Prometheus) can observe it without WatsonTcp depending on that host. + +- **`WatsonTcpMetrics`** (public) declares the source names, metric names, UCUM units, and tag keys. The `Meter`/`ActivitySource` name `WatsonTcp` is the stable contract; hosts subscribe by that string. +- **`WatsonTcpInstrumentation`** (internal, `IDisposable`) is created once per `WatsonTcpClient`/`WatsonTcpServer` in `Start()`/`ConnectCoreAsync()` and disposed with its owner. It owns one `Meter` and one `ActivitySource`, all counters/histograms, and the observable gauges. Instance scope is deliberate: observable gauges (`connections.active`, `connections.pending`, `sync.pending`, `uptime`) read live instance state through callbacks captured in the constructor, and the deterministic lifecycle guarantees no leaked subscriptions. Multiple instances may create a same-named `Meter`; OpenTelemetry aggregates them, so the contract stays singular. + +Recording is threaded through the same proven points as the pre-existing `WatsonTcpStatistics` (send/receive accounting) plus the connection, handshake, authentication, authorization, and synchronous-request lifecycle. Every call site is null-guarded (`_Instrumentation?.…`) and fire-and-forget, so telemetry can never throw into the send, receive, or connection path. Exception counting is funnelled through a single private `HandleException` helper on each of the client and server that records `watsontcp.exceptions.total` and then raises the existing `ExceptionEncountered` event. + +Metric tags are restricted to low-cardinality dimensions (`role`, `protocol`, `outcome`, `reason`, `message.kind`, `kind`, `exception.type`, `socket.error`). High-cardinality identifiers — client GUID, remote endpoint, conversation GUID, message size — are attached to spans only. Telemetry is on by default and is a near-free no-op when unobserved; `Settings.EnableMetrics` and `Settings.EnableTracing` disable it per instance (when metrics are disabled, no `Meter` is created at all). The `System.Diagnostics.Metrics` and `System.Diagnostics.ActivitySource` APIs are in-box on net8.0/net10.0 and are supplied by the `System.Diagnostics.DiagnosticSource` package on the down-level target frameworks. See `TELEMETRY.md` for the full catalog and integration guide. diff --git a/CHANGELOG.md b/CHANGELOG.md index c05fa1d..2c3203c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,22 @@ -# Change Log - +# Change Log + ## Current Version +v6.4.0 + +### Telemetry and Observability + +- Added vendor-neutral telemetry emitted through `System.Diagnostics.Metrics` (a `Meter` named `WatsonTcp`) and `System.Diagnostics.ActivitySource` (an `ActivitySource` named `WatsonTcp`), consumable by Radiant, the OpenTelemetry SDK, Prometheus, and others with no dependency on any telemetry backend +- Added the public `WatsonTcpMetrics` class exposing stable source names, metric names, units, and tag keys as the consumer contract +- Added 24 metrics spanning messages, bytes, connection lifecycle, disconnections by reason, handshakes, authentication, authorization, synchronous request/response, exceptions, transient listener errors, and uptime +- Added six tracing spans (`watsontcp.connect`, `watsontcp.session`, `watsontcp.handshake`, `watsontcp.send`, `watsontcp.receive`, `watsontcp.sync`) that carry high-cardinality identifiers kept off metric tags +- Added `Settings.EnableMetrics` and `Settings.EnableTracing` (both default `true`) to `WatsonTcpClientSettings` and `WatsonTcpServerSettings` +- Added the `System.Diagnostics.DiagnosticSource` package reference for the down-level target frameworks only; it is in-box on net8.0 and net10.0 +- Added an exhaustive positive/negative telemetry test suite exercised through the Touchstone runners +- Public API changes are additive; this is a minor release with no breaking changes + +## Previous Version + v6.3.2 ### Listener Reliability @@ -113,344 +128,344 @@ v6.2.0 v6.1.0 ### Performance - -- **Header parsing rewrite** - The old ```BuildFromStream``` read one byte at a time, allocated a new array on every byte via ```AppendBytes```, and ran LINQ ```.Skip().Take().ToArray()``` on every iteration to check for the ```\r\n\r\n``` delimiter. For a header of length H, that produced H-24 array allocations plus H-24 LINQ allocations. The new version writes into a ```MemoryStream``` and tracks the last 4 bytes with simple variables. Same byte-by-byte read (necessary to avoid over-reading past the delimiter into message body data on non-seekable ```NetworkStream```s), but zero per-byte allocations. -- **Buffer pooling** - Both client and server ```SendDataStreamAsync``` methods were allocating ```new byte[bufferSize]``` on every loop iteration of every send. Now they rent from ```ArrayPool.Shared``` once per send and return when done. ```GetHeaderBytes``` also replaced ```AppendBytes``` with ```Buffer.BlockCopy```. - -### Thread Safety - -- **ClientMetadataManager consolidated** - Had 5 separate ```ReaderWriterLockSlim``` instances protecting 5 dictionaries that represent one logical unit of client state. Operations like ```ReplaceGuid``` and ```Remove``` acquired/released each lock individually, creating windows where a client existed in one dictionary but not another. Now uses a single lock; all mutations are atomic. Also fixed ```GetClient``` which did ```ContainsKey``` then ```[guid]``` across two separate lock acquisitions (TOCTOU race leading to ```KeyNotFoundException```). Now uses ```TryGetValue```. -- **Sync response matching** - Both client and server used ```AutoResetEvent``` + multicast event handler subscription (```_SyncResponseReceived += handler```) with a ```lock``` around invocation. Race conditions existed between handler registration and message send, and between concurrent sync requests. Replaced with ```ConcurrentDictionary>```: register before sending, ```DataReceiver``` does ```TryRemove``` + ```TrySetResult``` on response arrival. Cleaner, lock-free, no signal loss. - -### Bug Fixes - -- **WaitHandle leak** - ```WatsonTcpClient.Connect()``` called ```BeginConnect```, got a ```WaitHandle```, but never closed it (commented out with a link to an MSDN forum post). Now closed in ```finally```. -- **Busy-wait spin loops** - ```ClientMetadata.Dispose()``` had ```while (DataReceiver?.Status == Running) Task.Delay(30).Wait()``` which blocks a thread pool thread spinning. Same pattern in ```WatsonTcpClient.Disconnect()``` for both ```_DataReceiver``` and ```_IdleServerMonitor```. Replaced with ```Task.Wait(TimeSpan.FromSeconds(5))```. -- **Stale client records** - ```_ClientsKicked``` and ```_ClientsTimedout``` dictionaries accumulated entries forever. Added ```PurgeStaleRecords(TimeSpan)``` to ```ClientMetadataManager```, called every ~60 seconds from ```MonitorForIdleClients```, purging records older than 5 minutes that don't correspond to active clients. - -### New Features - -- **```Settings.MaxHeaderSize```** (client and server, default 262144/256KB) - The old header parser had no upper bound. A malformed or malicious peer could send megabytes without a ```\r\n\r\n``` delimiter and the server would allocate until OOM. Now throws ```IOException``` when exceeded. -- **```Settings.EnforceMaxConnections```** (server, default ```true```) - Previously, ```MaxConnections``` only paused the listener (stopped accepting) but the check happened after the connection was already accepted, so it was really just a soft warning. Now, when enforcement is on, connections are actively rejected with ```tcpClient.Close()``` before any client state is created. When off, the old behavior is preserved (accept anyway, log a warning, don't stop the listener). - -### Observability - -- Added ```Severity.Debug``` logging to all previously silent ```catch (TaskCanceledException) { }``` and ```catch (OperationCanceledException) { }``` blocks in ```IdleServerMonitor```, ```MonitorForIdleClients```, and ```SendInternalAsync``` on both client and server. - -### Testing - -- 10 new automated tests (46 total): MaxConnections enforcement (happy + sad), MaxHeaderSize validation, rapid connect/disconnect (10 cycles), concurrent sync requests (5 parallel ```SendAndWaitAsync``` with response verification), SSL connectivity + message exchange, server stop detection from client, duplicate client GUID handling, and send-with-byte-offset. - -### Breaking Changes - -- ```Settings.EnforceMaxConnections``` defaults to ```true```. If you were relying on the server accepting unlimited connections despite ```MaxConnections``` being set, connections will now be rejected at capacity. Set ```EnforceMaxConnections = false``` to restore the old behavior. -- All other changes are internal implementation details with identical public API signatures and wire protocol. - + +- **Header parsing rewrite** - The old ```BuildFromStream``` read one byte at a time, allocated a new array on every byte via ```AppendBytes```, and ran LINQ ```.Skip().Take().ToArray()``` on every iteration to check for the ```\r\n\r\n``` delimiter. For a header of length H, that produced H-24 array allocations plus H-24 LINQ allocations. The new version writes into a ```MemoryStream``` and tracks the last 4 bytes with simple variables. Same byte-by-byte read (necessary to avoid over-reading past the delimiter into message body data on non-seekable ```NetworkStream```s), but zero per-byte allocations. +- **Buffer pooling** - Both client and server ```SendDataStreamAsync``` methods were allocating ```new byte[bufferSize]``` on every loop iteration of every send. Now they rent from ```ArrayPool.Shared``` once per send and return when done. ```GetHeaderBytes``` also replaced ```AppendBytes``` with ```Buffer.BlockCopy```. + +### Thread Safety + +- **ClientMetadataManager consolidated** - Had 5 separate ```ReaderWriterLockSlim``` instances protecting 5 dictionaries that represent one logical unit of client state. Operations like ```ReplaceGuid``` and ```Remove``` acquired/released each lock individually, creating windows where a client existed in one dictionary but not another. Now uses a single lock; all mutations are atomic. Also fixed ```GetClient``` which did ```ContainsKey``` then ```[guid]``` across two separate lock acquisitions (TOCTOU race leading to ```KeyNotFoundException```). Now uses ```TryGetValue```. +- **Sync response matching** - Both client and server used ```AutoResetEvent``` + multicast event handler subscription (```_SyncResponseReceived += handler```) with a ```lock``` around invocation. Race conditions existed between handler registration and message send, and between concurrent sync requests. Replaced with ```ConcurrentDictionary>```: register before sending, ```DataReceiver``` does ```TryRemove``` + ```TrySetResult``` on response arrival. Cleaner, lock-free, no signal loss. + +### Bug Fixes + +- **WaitHandle leak** - ```WatsonTcpClient.Connect()``` called ```BeginConnect```, got a ```WaitHandle```, but never closed it (commented out with a link to an MSDN forum post). Now closed in ```finally```. +- **Busy-wait spin loops** - ```ClientMetadata.Dispose()``` had ```while (DataReceiver?.Status == Running) Task.Delay(30).Wait()``` which blocks a thread pool thread spinning. Same pattern in ```WatsonTcpClient.Disconnect()``` for both ```_DataReceiver``` and ```_IdleServerMonitor```. Replaced with ```Task.Wait(TimeSpan.FromSeconds(5))```. +- **Stale client records** - ```_ClientsKicked``` and ```_ClientsTimedout``` dictionaries accumulated entries forever. Added ```PurgeStaleRecords(TimeSpan)``` to ```ClientMetadataManager```, called every ~60 seconds from ```MonitorForIdleClients```, purging records older than 5 minutes that don't correspond to active clients. + +### New Features + +- **```Settings.MaxHeaderSize```** (client and server, default 262144/256KB) - The old header parser had no upper bound. A malformed or malicious peer could send megabytes without a ```\r\n\r\n``` delimiter and the server would allocate until OOM. Now throws ```IOException``` when exceeded. +- **```Settings.EnforceMaxConnections```** (server, default ```true```) - Previously, ```MaxConnections``` only paused the listener (stopped accepting) but the check happened after the connection was already accepted, so it was really just a soft warning. Now, when enforcement is on, connections are actively rejected with ```tcpClient.Close()``` before any client state is created. When off, the old behavior is preserved (accept anyway, log a warning, don't stop the listener). + +### Observability + +- Added ```Severity.Debug``` logging to all previously silent ```catch (TaskCanceledException) { }``` and ```catch (OperationCanceledException) { }``` blocks in ```IdleServerMonitor```, ```MonitorForIdleClients```, and ```SendInternalAsync``` on both client and server. + +### Testing + +- 10 new automated tests (46 total): MaxConnections enforcement (happy + sad), MaxHeaderSize validation, rapid connect/disconnect (10 cycles), concurrent sync requests (5 parallel ```SendAndWaitAsync``` with response verification), SSL connectivity + message exchange, server stop detection from client, duplicate client GUID handling, and send-with-byte-offset. + +### Breaking Changes + +- ```Settings.EnforceMaxConnections``` defaults to ```true```. If you were relying on the server accepting unlimited connections despite ```MaxConnections``` being set, connections will now be rejected at capacity. Set ```EnforceMaxConnections = false``` to restore the old behavior. +- All other changes are internal implementation details with identical public API signatures and wire protocol. + ## Previous Versions - -v6.0.x - -- Remove unsupported frameworks -- Async version of ```SyncMessageReceived``` callback -- Moving usings inside namespace -- Remove obsolete methods -- Mark non-async APIs obsolete -- Modified test projects to use async -- Ensured background tasks honored cancellation tokens - -v5.1.x - -- Strong name signing -- Better exception logging -- Set ```Settings.NoDelay``` to ```true``` by default (disabling Nagle's algorithm) - -v5.0.x - -- Breaking changes -- Migrate from using ```IpPort``` as a client key to using ```Guid``` -- Removal of ```Newtonsoft.Json``` as a dependency -- Separate ```WatsonMessageBuilder``` class to reduce code bloat -- ```ClientMetadata``` now includes ```Guid``` -- ```ListClients``` now returns list of ```ClientMetadata``` instead of list of ```IpPort``` -- Mark ```Send*``` methods that use ```ipPort``` as obsolete (pending removal in future release) -- Restrict message metadata dictionary to `````` -- Targeting for .NET 7.0 - -v4.8.11 - -- TLS extensions, thank you @cee-sharp - -v4.8.10 - -- Bugfix, authentication failure now disconnects clients and propagates the correct reason (thank you @Jyck) - -v4.8.9 - -- Added optional parameter ```offset``` to ```Send``` and ```SendAsync``` methods that use ```byte[]``` data (thank you @pha3z) - -v4.8.8 - -- Move listener start into ```Start()``` method (thank you @avoitenko) - -v4.8.7 - -- Bugfix, timeout values for .NET Framework now properly handled as milliseconds (thank you @zsolt777) - -v4.8.6 - -- Specify the client port by setting ```Settings.LocalPort``` (0, 1024-65535 are valid, where 0 is auto-assigned) - -v4.8.0 - -- Breaking change; log messages now include a ```Severity``` parameter -- TCP keepalives moved to the socket instead of the listener - -v4.7.1 - -- Breaking change; TCP keepalives now disabled by default due to incompatibility and problems on some platforms - -v4.7.0 - -- Breaking changes -- Consolidated connection/disconnection event arguments -- Consolidated message/stream received event arguments -- Aligned disconnection reason with message status - -v4.6.0.0 - -- More changes based on suggestions from @syntacs and @MartyIX -- Consolidated ```Send``` constructors with optional params to reduce complexity -- Optional ```CancellationToken``` parameters for async ```Send``` methods -- Use of ```ConfigureAwait``` for better reliability - -v4.5.0.1 - -- Excellent changes and recommendations led by @syntacs for reliability -- Better coordination between Dispose and server Stop and client Disconnect -- Exception handling in server and client event handlers as well as callbacks - -v4.4.0 - -- Breaking changes; header name fields have been reduced -- Performance improvements -- Elimination of sending unnecessary headers -- Thank you @broms95! - -v4.3.0 - -- Breaking changes -- Retarget to include .NET Core 3.1 (previously .NET Framework 4.6.1 and .NET Standard 2.1 only) -- Added support for TCP keepalives for .NET Framework and .NET Core (.NET Standard does not have such facilities) -- Consolidated settings into separate classes - -v4.2.0 - -- Breaking changes -- Introduced ```WatsonStream``` class to prevent stream consumers from reading into the next message's header -- ```MaxProxiedStreamSize``` property to dictate whether data is sent to ```StreamReceived``` in a new ```MemoryStream``` or the underlying data stream is sent -- Minor refactor and removal of compression - -v4.1.12 - -- Fix for ClientMetadata.Dispose - -v4.1.11 - -- Fix to order of ServerConnected and starting DataReceiver in WatsonTcpClient (thank you @ozrecsec) - -v4.1.10 - -- Minor fixes to synchronous message expiration (thank you @karstennilsen) - -v4.1.9 - -- Fix for being unable to disconnect a client from ClientConnected (thank you @motridox) - -v4.1.8 - -- Fix for message expiration (thank you @karstennilsen) - -v4.1.7 - -- AuthenticationRequested, AuthenticationSucceeded, and AuthenticationFailed events in WatsonTcpServer - -v4.1.6 - -- Added SenderTimestamp to sync messages and derived expiration based on difference in sender vs receiver perception of time (thank you @karstennilsen) - -v4.1.5 - -- Fix for synchronous request timeout leaving message data in the underlying stream (thank you @ozrecsec!) - -v4.1.4 - -- Minor internal refactor - -v4.1.3 - -- Fix for issue: compression with SSL enabled causes deserialization exceptions; not recommended for use -- Minor refactor - -v4.1.2 - -- New constructor for SSL, taking certificate as parameter (thank you @NormenSchwettmann) -- **Known issue**: compression with SSL enabled causes deserialization exceptions; not recommended for use - -v4.1.1 - -- Bugfix for disconnect scenarios causing the next message headers to be read as part of the prior message -- **Known issue**: compression with SSL enabled causes deserialization exceptions; not recommended for use - -v4.1.0 - -- Compression of message data using either GZip or Deflate (thanks @developervariety!) -- Message data is now a property that fully reads the underlying stream -- Internal code refactoring to better follow DRY principles (SendHeaders, SendDataStream, etc) -- Reduce log verbosity on disconnect - -v4.0.2 - -- Bugfix (thank you @ozrecsec!) for ClientDisconnected firing too early - -v4.0.1 - -- Bugfixes (thank you @ozrecsec!) for DateTime serialization - -v4.0.0 - -- Overhaul to internal framing, refer to ```FRAMING.md``` -- Fixes to ```Test.Throughput``` projects (incorrectly reporting statistics) - -v3.1.4 - -- Better handling for cases where no message/stream event handler is set - -v3.1.3 - -- Fix synchronous messaging expiration bug - -v3.1.2 - -- Fix DateTime string format - -v3.1.1 - -- APIs to support sending async or sync (send-and-wait) messages with a metadata dictionary and no data -- Better handling of null input when sending data - -v3.1.0 - -- Added support for synchronous messaging, i.e. send and wait for a response (see ```SendAndWait``` methods) with timeouts. See the updated examples below or refer to the ```Test.Client``` and ```Test.Server``` project for examples -- Consolidated Logger for client, server, and messages -- ```Debug``` is now ```DebugMessages``` -- Minor internal refactor - -v3.0.3 - -- Now supports serialized metadata sizes (i.e. calculated after serialization of your dictionary) of up to 99,999,999 bytes - -v3.0.2 - -- ```.Data``` property in both ```StreamReceivedFromClientEventArgs``` and ```StreamReceivedFromServerEventArgs```. - -v3.0.1 - -- Bugfix in pre-shared key authentication - -v3.0.0 - -- Breaking changes; move from Func-based callbacks to Event -- Added MaxConnections and Connection values in WatsonTcpServer - -v2.2.2 - -- Added Statistics object. - -v2.2.1 - -- Added Logger method to both WatsonTcpServer and WatsonTcpClient (thanks @crushedice) - -v2.2.0 - -- Add support for sending and receiving messages with metadata ```Dictionary``` -- New callbacks for receiving messages with metadata: MessageReceivedWithMetadata and StreamReceivedWithMetadata - -- New callbacks for sending messages with metadata (overloads on existing methods added) -- Now dependent upon Newtonsoft.Json as metadata must be serialized; only serializable types are supported in metadata - -v2.1.7 - -- Add support for Send(string) and SendAsync(string) - -v2.1.6 - -- ListClients now returns IEnumerable (thanks @pha3z!) - -v2.1.5 - -- Fix for larger message cases (thanks @mikkleini!) - -v2.1.4 - -- Minor breaking change; ClientDisconnect now includes DisconnectReason to differentiate between normal, kicked, or timeout disconnections - -v2.1.3 - -- Fix for ClientMetadata dispose (too many extranneous Dispose calls) -- TestThroughput project - -v2.1.2 - -- Client timeout now only reset upon receiving a message from a client, and no longer reset when sending a message to a client - -v2.1.1 - -- Automatically disconnect idle clients by setting ```WatsonTcpServer.IdleClientTimeoutSeconds``` to a positive integer (excellent suggestion, @pha3z!) - -v2.1.0 - -- Breaking changes -- Better documentation on StreamReceived vs MessageReceived in the XML documentation and in the README -- Modified getters and setters on StreamReceived and MessageReceived to make them mutually exclusive -- Removal of (now unnecessary) ReadDataStream parameter -- ReadStreamBufferSize is now renamed to StreamBufferSize - -v2.0.8 - -- StartAsync() method for client and server - -v2.0.x - -- Changed .NET Framework minimum requirement to 4.6.1 to support use of ```TcpClient.Dispose``` -- Better disconnect handling and support (thank you to @mikkleini) -- Async Task-based callbacks -- Configurable connect timeout in WatsonTcpClient -- Clients can now connect via SSL without a certificate -- Big thanks to @MrMikeJJ for his extensive commits and pull requests -- Bugfix for graceful disconnect through dispose (thank you @mikkleini!) - -v1.3.x -- Numerous fixes to authentication using preshared keys -- Authentication callbacks in the client to handle authentication events - - ```AuthenticationRequested``` - authentication requested by the server, return the preshared key string (16 bytes) - - ```AuthenticationSucceeded``` - authentication has succeeded, return true - - ```AuthenticationFailure``` - authentication has failed, return true -- Support for sending and receiving larger messages by using streams instead of byte arrays -- Refer to ```TestServerStream``` and ```TestClientStream``` for a reference implementation. You must set ```client.ReadDataStream = false``` and ```server.ReadDataStream = false``` and use the ```StreamReceived``` callback instead of ```MessageReceived``` - -v1.2.x -- Breaking changes for assigning callbacks, various server/client class variables, and starting them -- Consolidated SSL and non-SSL clients and servers into single classes for each -- Retargeted test projects to both .NET Core and .NET Framework -- Added more extensible framing support to later carry more metadata as needed -- Added authentication via pre-shared key (set Server.PresharedKey class variable, and use Client.Authenticate() method) - -v1.1.x -- Re-targeted to both .NET Core 2.0 and .NET Framework 4.5.2 -- Various bugfixes - -v1.0.x -- Initial release -- Async support and IDisposable support -- IP filtering/permitted IP addresses support -- Improved disconnect detection -- SSL support + +v6.0.x + +- Remove unsupported frameworks +- Async version of ```SyncMessageReceived``` callback +- Moving usings inside namespace +- Remove obsolete methods +- Mark non-async APIs obsolete +- Modified test projects to use async +- Ensured background tasks honored cancellation tokens + +v5.1.x + +- Strong name signing +- Better exception logging +- Set ```Settings.NoDelay``` to ```true``` by default (disabling Nagle's algorithm) + +v5.0.x + +- Breaking changes +- Migrate from using ```IpPort``` as a client key to using ```Guid``` +- Removal of ```Newtonsoft.Json``` as a dependency +- Separate ```WatsonMessageBuilder``` class to reduce code bloat +- ```ClientMetadata``` now includes ```Guid``` +- ```ListClients``` now returns list of ```ClientMetadata``` instead of list of ```IpPort``` +- Mark ```Send*``` methods that use ```ipPort``` as obsolete (pending removal in future release) +- Restrict message metadata dictionary to `````` +- Targeting for .NET 7.0 + +v4.8.11 + +- TLS extensions, thank you @cee-sharp + +v4.8.10 + +- Bugfix, authentication failure now disconnects clients and propagates the correct reason (thank you @Jyck) + +v4.8.9 + +- Added optional parameter ```offset``` to ```Send``` and ```SendAsync``` methods that use ```byte[]``` data (thank you @pha3z) + +v4.8.8 + +- Move listener start into ```Start()``` method (thank you @avoitenko) + +v4.8.7 + +- Bugfix, timeout values for .NET Framework now properly handled as milliseconds (thank you @zsolt777) + +v4.8.6 + +- Specify the client port by setting ```Settings.LocalPort``` (0, 1024-65535 are valid, where 0 is auto-assigned) + +v4.8.0 + +- Breaking change; log messages now include a ```Severity``` parameter +- TCP keepalives moved to the socket instead of the listener + +v4.7.1 + +- Breaking change; TCP keepalives now disabled by default due to incompatibility and problems on some platforms + +v4.7.0 + +- Breaking changes +- Consolidated connection/disconnection event arguments +- Consolidated message/stream received event arguments +- Aligned disconnection reason with message status + +v4.6.0.0 + +- More changes based on suggestions from @syntacs and @MartyIX +- Consolidated ```Send``` constructors with optional params to reduce complexity +- Optional ```CancellationToken``` parameters for async ```Send``` methods +- Use of ```ConfigureAwait``` for better reliability + +v4.5.0.1 + +- Excellent changes and recommendations led by @syntacs for reliability +- Better coordination between Dispose and server Stop and client Disconnect +- Exception handling in server and client event handlers as well as callbacks + +v4.4.0 + +- Breaking changes; header name fields have been reduced +- Performance improvements +- Elimination of sending unnecessary headers +- Thank you @broms95! + +v4.3.0 + +- Breaking changes +- Retarget to include .NET Core 3.1 (previously .NET Framework 4.6.1 and .NET Standard 2.1 only) +- Added support for TCP keepalives for .NET Framework and .NET Core (.NET Standard does not have such facilities) +- Consolidated settings into separate classes + +v4.2.0 + +- Breaking changes +- Introduced ```WatsonStream``` class to prevent stream consumers from reading into the next message's header +- ```MaxProxiedStreamSize``` property to dictate whether data is sent to ```StreamReceived``` in a new ```MemoryStream``` or the underlying data stream is sent +- Minor refactor and removal of compression + +v4.1.12 + +- Fix for ClientMetadata.Dispose + +v4.1.11 + +- Fix to order of ServerConnected and starting DataReceiver in WatsonTcpClient (thank you @ozrecsec) + +v4.1.10 + +- Minor fixes to synchronous message expiration (thank you @karstennilsen) + +v4.1.9 + +- Fix for being unable to disconnect a client from ClientConnected (thank you @motridox) + +v4.1.8 + +- Fix for message expiration (thank you @karstennilsen) + +v4.1.7 + +- AuthenticationRequested, AuthenticationSucceeded, and AuthenticationFailed events in WatsonTcpServer + +v4.1.6 + +- Added SenderTimestamp to sync messages and derived expiration based on difference in sender vs receiver perception of time (thank you @karstennilsen) + +v4.1.5 + +- Fix for synchronous request timeout leaving message data in the underlying stream (thank you @ozrecsec!) + +v4.1.4 + +- Minor internal refactor + +v4.1.3 + +- Fix for issue: compression with SSL enabled causes deserialization exceptions; not recommended for use +- Minor refactor + +v4.1.2 + +- New constructor for SSL, taking certificate as parameter (thank you @NormenSchwettmann) +- **Known issue**: compression with SSL enabled causes deserialization exceptions; not recommended for use + +v4.1.1 + +- Bugfix for disconnect scenarios causing the next message headers to be read as part of the prior message +- **Known issue**: compression with SSL enabled causes deserialization exceptions; not recommended for use + +v4.1.0 + +- Compression of message data using either GZip or Deflate (thanks @developervariety!) +- Message data is now a property that fully reads the underlying stream +- Internal code refactoring to better follow DRY principles (SendHeaders, SendDataStream, etc) +- Reduce log verbosity on disconnect + +v4.0.2 + +- Bugfix (thank you @ozrecsec!) for ClientDisconnected firing too early + +v4.0.1 + +- Bugfixes (thank you @ozrecsec!) for DateTime serialization + +v4.0.0 + +- Overhaul to internal framing, refer to ```FRAMING.md``` +- Fixes to ```Test.Throughput``` projects (incorrectly reporting statistics) + +v3.1.4 + +- Better handling for cases where no message/stream event handler is set + +v3.1.3 + +- Fix synchronous messaging expiration bug + +v3.1.2 + +- Fix DateTime string format + +v3.1.1 + +- APIs to support sending async or sync (send-and-wait) messages with a metadata dictionary and no data +- Better handling of null input when sending data + +v3.1.0 + +- Added support for synchronous messaging, i.e. send and wait for a response (see ```SendAndWait``` methods) with timeouts. See the updated examples below or refer to the ```Test.Client``` and ```Test.Server``` project for examples +- Consolidated Logger for client, server, and messages +- ```Debug``` is now ```DebugMessages``` +- Minor internal refactor + +v3.0.3 + +- Now supports serialized metadata sizes (i.e. calculated after serialization of your dictionary) of up to 99,999,999 bytes + +v3.0.2 + +- ```.Data``` property in both ```StreamReceivedFromClientEventArgs``` and ```StreamReceivedFromServerEventArgs```. + +v3.0.1 + +- Bugfix in pre-shared key authentication + +v3.0.0 + +- Breaking changes; move from Func-based callbacks to Event +- Added MaxConnections and Connection values in WatsonTcpServer + +v2.2.2 + +- Added Statistics object. + +v2.2.1 + +- Added Logger method to both WatsonTcpServer and WatsonTcpClient (thanks @crushedice) + +v2.2.0 + +- Add support for sending and receiving messages with metadata ```Dictionary``` +- New callbacks for receiving messages with metadata: MessageReceivedWithMetadata and StreamReceivedWithMetadata - +- New callbacks for sending messages with metadata (overloads on existing methods added) +- Now dependent upon Newtonsoft.Json as metadata must be serialized; only serializable types are supported in metadata + +v2.1.7 + +- Add support for Send(string) and SendAsync(string) + +v2.1.6 + +- ListClients now returns IEnumerable (thanks @pha3z!) + +v2.1.5 + +- Fix for larger message cases (thanks @mikkleini!) + +v2.1.4 + +- Minor breaking change; ClientDisconnect now includes DisconnectReason to differentiate between normal, kicked, or timeout disconnections + +v2.1.3 + +- Fix for ClientMetadata dispose (too many extranneous Dispose calls) +- TestThroughput project + +v2.1.2 + +- Client timeout now only reset upon receiving a message from a client, and no longer reset when sending a message to a client + +v2.1.1 + +- Automatically disconnect idle clients by setting ```WatsonTcpServer.IdleClientTimeoutSeconds``` to a positive integer (excellent suggestion, @pha3z!) + +v2.1.0 + +- Breaking changes +- Better documentation on StreamReceived vs MessageReceived in the XML documentation and in the README +- Modified getters and setters on StreamReceived and MessageReceived to make them mutually exclusive +- Removal of (now unnecessary) ReadDataStream parameter +- ReadStreamBufferSize is now renamed to StreamBufferSize + +v2.0.8 + +- StartAsync() method for client and server + +v2.0.x + +- Changed .NET Framework minimum requirement to 4.6.1 to support use of ```TcpClient.Dispose``` +- Better disconnect handling and support (thank you to @mikkleini) +- Async Task-based callbacks +- Configurable connect timeout in WatsonTcpClient +- Clients can now connect via SSL without a certificate +- Big thanks to @MrMikeJJ for his extensive commits and pull requests +- Bugfix for graceful disconnect through dispose (thank you @mikkleini!) + +v1.3.x +- Numerous fixes to authentication using preshared keys +- Authentication callbacks in the client to handle authentication events + - ```AuthenticationRequested``` - authentication requested by the server, return the preshared key string (16 bytes) + - ```AuthenticationSucceeded``` - authentication has succeeded, return true + - ```AuthenticationFailure``` - authentication has failed, return true +- Support for sending and receiving larger messages by using streams instead of byte arrays +- Refer to ```TestServerStream``` and ```TestClientStream``` for a reference implementation. You must set ```client.ReadDataStream = false``` and ```server.ReadDataStream = false``` and use the ```StreamReceived``` callback instead of ```MessageReceived``` + +v1.2.x +- Breaking changes for assigning callbacks, various server/client class variables, and starting them +- Consolidated SSL and non-SSL clients and servers into single classes for each +- Retargeted test projects to both .NET Core and .NET Framework +- Added more extensible framing support to later carry more metadata as needed +- Added authentication via pre-shared key (set Server.PresharedKey class variable, and use Client.Authenticate() method) + +v1.1.x +- Re-targeted to both .NET Core 2.0 and .NET Framework 4.5.2 +- Various bugfixes + +v1.0.x +- Initial release +- Async support and IDisposable support +- IP filtering/permitted IP addresses support +- Improved disconnect detection +- SSL support diff --git a/CLAUDE.md b/CLAUDE.md index 5d2591c..ae5b6ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,6 +153,8 @@ src/WatsonTcp/ # Main library source WatsonMessageBuilder.cs # Message construction WatsonStream.cs # Stream reading/writing ClientMetadata*.cs # Client tracking for server + WatsonTcpMetrics.cs # Public telemetry names/units/tag keys (Meter + ActivitySource "WatsonTcp") + WatsonTcpInstrumentation.cs # Internal per-instance metrics/spans recorder *Events.cs # Event definitions *Callbacks.cs # Callback definitions *Settings.cs # Configuration classes @@ -162,4 +164,8 @@ src/Test.*/ # Manual test projects (not unit tests) ## NuGet Package -Version is defined in `src/WatsonTcp/WatsonTcp.csproj` (currently 6.0.11). Package builds automatically with `GeneratePackageOnBuild`. +Version is defined in `src/WatsonTcp/WatsonTcp.csproj` (currently 6.4.0). Package builds automatically with `GeneratePackageOnBuild`. + +## Telemetry + +WatsonTcp emits vendor-neutral telemetry via `System.Diagnostics.Metrics` (a `Meter` named `WatsonTcp`) and `System.Diagnostics.ActivitySource` (an `ActivitySource` named `WatsonTcp`). Hosts (Radiant, OpenTelemetry, Prometheus) subscribe by name; WatsonTcp takes no telemetry-backend dependency. Public string contract lives in `WatsonTcpMetrics`. See `TELEMETRY.md`. Telemetry is on by default and can be disabled per instance via `Settings.EnableMetrics` / `Settings.EnableTracing`. diff --git a/README.md b/README.md index fc59302..403a658 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,60 @@ -![alt tag](https://github.com/jchristn/watsontcp/blob/main/assets/watson.ico) - -# WatsonTcp - -[![NuGet Version](https://img.shields.io/nuget/v/WatsonTcp.svg?style=flat)](https://www.nuget.org/packages/WatsonTcp/) [![NuGet](https://img.shields.io/nuget/dt/WatsonTcp.svg)](https://www.nuget.org/packages/WatsonTcp) - -WatsonTcp is the fastest, easiest, most efficient way to build TCP-based clients and servers in C# with integrated framing, reliable transmission, and fast disconnect detection. - +![alt tag](https://github.com/jchristn/watsontcp/blob/main/assets/watson.ico) + +# WatsonTcp + +[![NuGet Version](https://img.shields.io/nuget/v/WatsonTcp.svg?style=flat)](https://www.nuget.org/packages/WatsonTcp/) [![NuGet](https://img.shields.io/nuget/dt/WatsonTcp.svg)](https://www.nuget.org/packages/WatsonTcp) + +WatsonTcp is the fastest, easiest, most efficient way to build TCP-based clients and servers in C# with integrated framing, reliable transmission, and fast disconnect detection. + **IMPORTANT** WatsonTcp provides framing to ensure message-level delivery which also dictates that you must either 1) use WatsonTcp for both the server and the client, or, 2) ensure that your client/server exchange messages with the WatsonTcp node using WatsonTcp's framing. Refer to `FRAMING.md` for a reference on WatsonTcp message structure. - -- If you want a library that doesn't use framing, but has a similar implementation, use [SuperSimpleTcp](https://github.com/jchristn/supersimpletcp) -- If you want a library that doesn't use framing and provides explicit control over how much data to read, use [CavemanTcp](https://github.com/jchristn/cavemantcp) - -## .NET Foundation - -This project is part of the [.NET Foundation](http://www.dotnetfoundation.org/projects) along with other projects like [the .NET Runtime](https://github.com/dotnet/runtime/). - + +- If you want a library that doesn't use framing, but has a similar implementation, use [SuperSimpleTcp](https://github.com/jchristn/supersimpletcp) +- If you want a library that doesn't use framing and provides explicit control over how much data to read, use [CavemanTcp](https://github.com/jchristn/cavemantcp) + +## .NET Foundation + +This project is part of the [.NET Foundation](http://www.dotnetfoundation.org/projects) along with other projects like [the .NET Runtime](https://github.com/dotnet/runtime/). + ## Contributions - -Special thanks to the following people for their support and contributions to this project! - -@brudo @MrMikeJJ @mikkleini @pha3z @crushedice @marek-petak @ozrecsec @developervariety -@NormenSchwettmann @karstennilsen @motridox @AdamFrisby @Job79 @Dijkstra-ru @playingoDEERUX -@DuAell @syntacs @zsolt777 @broms95 @Antwns @MartyIX @Jyck @Memphizzz @nirajgenius -@cee-sharp @jeverz @cbarraco @DenisBalan @Markonius @Ahmed310 @markashleybell -@thechosensausage @JVemon @eatyouroats @bendablegears @Laiteux @fisherman6v6 @wesoos + +Special thanks to the following people for their support and contributions to this project! + +@brudo @MrMikeJJ @mikkleini @pha3z @crushedice @marek-petak @ozrecsec @developervariety +@NormenSchwettmann @karstennilsen @motridox @AdamFrisby @Job79 @Dijkstra-ru @playingoDEERUX +@DuAell @syntacs @zsolt777 @broms95 @Antwns @MartyIX @Jyck @Memphizzz @nirajgenius +@cee-sharp @jeverz @cbarraco @DenisBalan @Markonius @Ahmed310 @markashleybell +@thechosensausage @JVemon @eatyouroats @bendablegears @Laiteux @fisherman6v6 @wesoos @YorVeX @tovich37 @sancheolz @lunedis @ShayanFiroozi @BrvSqr - + If you'd like to contribute, please jump right into the source code and create a pull request, or, file an issue with your enhancement request. +## New in v6.4.0 + +### Telemetry and Observability + +WatsonTcp now emits standardized, vendor-neutral telemetry using only the .NET base class library, so it can be observed by [Radiant](https://github.com/jchristn), the OpenTelemetry SDK, Prometheus, or any compatible host with **no dependency on any telemetry backend**. + +- metrics are recorded into a `System.Diagnostics.Metrics.Meter` named `WatsonTcp` +- distributed-tracing spans are started from a `System.Diagnostics.ActivitySource` named `WatsonTcp` +- both names are exposed as public constants on the new `WatsonTcp.WatsonTcpMetrics` class and are stable across releases +- 24 metrics cover messages, bytes, connections, disconnections (by reason), handshakes, authentication, authorization, synchronous request/response, exceptions, transient listener errors, and uptime +- metric tags are low-cardinality only (`role`, `protocol`, `outcome`, `reason`); high-cardinality identifiers (client GUID, remote endpoint, conversation GUID) are placed on spans, never on metric tags +- telemetry is on by default and is a near-free no-op when nobody subscribes; it can be turned off per instance via `Settings.EnableMetrics` and `Settings.EnableTracing` + +A telemetry host subscribes by name and nothing else changes in your WatsonTcp code: + +```csharp +// Radiant host +settings.Sources.AddMeter(WatsonTcp.WatsonTcpMetrics.MeterName); // "WatsonTcp" +settings.Sources.AddActivitySource(WatsonTcp.WatsonTcpMetrics.MeterName); // "WatsonTcp" + +// or a raw OpenTelemetry host +Sdk.CreateMeterProviderBuilder().AddMeter("WatsonTcp").AddPrometheusHttpListener().Build(); +Sdk.CreateTracerProviderBuilder().AddSource("WatsonTcp").AddOtlpExporter().Build(); +``` + +See `TELEMETRY.md` for the full metric catalog, tag dictionaries, Prometheus series names, and integration walkthrough. + ## New in v6.3.2 ### Listener Reliability @@ -176,62 +203,62 @@ dotnet test src/Test.NUnit/Test.NUnit.csproj --framework net8.0 ``` ## New in v6.1.0 - -### Performance -- Rewrote message header parsing to eliminate O(n^2) array allocations and per-byte LINQ overhead; now uses a ```MemoryStream``` accumulator with direct byte comparison -- Send operations now use ```ArrayPool``` pooling instead of allocating new buffers on every iteration - -### Thread Safety -- Consolidated ```ClientMetadataManager``` from 5 independent ```ReaderWriterLockSlim``` instances to a single lock, eliminating race conditions during multi-dictionary operations (```ReplaceGuid```, ```Remove```) -- Fixed TOCTOU race in ```GetClient()``` (```ContainsKey``` then indexer across separate lock acquisitions); now uses ```TryGetValue``` -- Replaced ```AutoResetEvent``` + event-based sync response matching with ```ConcurrentDictionary>``` in both client and server, eliminating handler registration race conditions and signal loss - -### Bug Fixes -- Fixed ```WaitHandle``` resource leak in ```WatsonTcpClient.Connect()``` (was commented out, now properly closed) -- Replaced busy-wait spin loops in ```ClientMetadata.Dispose()``` and ```WatsonTcpClient.Disconnect()``` with ```Task.Wait(timeout)``` -- Stale kicked/timed-out client records now automatically purged every 60 seconds (previously accumulated forever) - -### New Features -- ```Settings.MaxHeaderSize``` (client and server, default 256KB) guards against memory exhaustion from oversized or malicious headers -- ```Settings.EnforceMaxConnections``` (server, default ```true```) actively rejects connections at capacity; set to ```false``` for legacy behavior - -### Observability -- Added debug-level logging to all previously silent ```TaskCanceledException``` and ```OperationCanceledException``` catch blocks - -### Testing -- 10 new automated tests (46 total) covering MaxConnections enforcement, MaxHeaderSize validation, rapid connect/disconnect, concurrent sync requests, SSL, server stop detection, duplicate GUIDs, and send-with-offset - -### Breaking Changes -- ```Settings.EnforceMaxConnections``` defaults to ```true```. If you relied on accepting connections beyond ```MaxConnections```, set ```EnforceMaxConnections = false```. -- All other changes are internal with identical public API and wire protocol. - -## Previous in v6.0.x - -- Remove unsupported frameworks -- Async version of ```SyncMessageReceived``` callback -- Moving usings inside namespace -- Remove obsolete methods -- Mark non-async APIs obsolete -- Modified test projects to use async -- Ensured background tasks honored cancellation tokens -- Ability to specify a client's GUID before attempting to connect - -## Architecture - -Refer to [ARCHITECTURE.md](ARCHITECTURE.md) for a detailed overview of the internal design, message flow, threading model, and key design decisions. - -For the wire protocol specification (header format, delimiter, payload layout), see [FRAMING.md](FRAMING.md). - + +### Performance +- Rewrote message header parsing to eliminate O(n^2) array allocations and per-byte LINQ overhead; now uses a ```MemoryStream``` accumulator with direct byte comparison +- Send operations now use ```ArrayPool``` pooling instead of allocating new buffers on every iteration + +### Thread Safety +- Consolidated ```ClientMetadataManager``` from 5 independent ```ReaderWriterLockSlim``` instances to a single lock, eliminating race conditions during multi-dictionary operations (```ReplaceGuid```, ```Remove```) +- Fixed TOCTOU race in ```GetClient()``` (```ContainsKey``` then indexer across separate lock acquisitions); now uses ```TryGetValue``` +- Replaced ```AutoResetEvent``` + event-based sync response matching with ```ConcurrentDictionary>``` in both client and server, eliminating handler registration race conditions and signal loss + +### Bug Fixes +- Fixed ```WaitHandle``` resource leak in ```WatsonTcpClient.Connect()``` (was commented out, now properly closed) +- Replaced busy-wait spin loops in ```ClientMetadata.Dispose()``` and ```WatsonTcpClient.Disconnect()``` with ```Task.Wait(timeout)``` +- Stale kicked/timed-out client records now automatically purged every 60 seconds (previously accumulated forever) + +### New Features +- ```Settings.MaxHeaderSize``` (client and server, default 256KB) guards against memory exhaustion from oversized or malicious headers +- ```Settings.EnforceMaxConnections``` (server, default ```true```) actively rejects connections at capacity; set to ```false``` for legacy behavior + +### Observability +- Added debug-level logging to all previously silent ```TaskCanceledException``` and ```OperationCanceledException``` catch blocks + +### Testing +- 10 new automated tests (46 total) covering MaxConnections enforcement, MaxHeaderSize validation, rapid connect/disconnect, concurrent sync requests, SSL, server stop detection, duplicate GUIDs, and send-with-offset + +### Breaking Changes +- ```Settings.EnforceMaxConnections``` defaults to ```true```. If you relied on accepting connections beyond ```MaxConnections```, set ```EnforceMaxConnections = false```. +- All other changes are internal with identical public API and wire protocol. + +## Previous in v6.0.x + +- Remove unsupported frameworks +- Async version of ```SyncMessageReceived``` callback +- Moving usings inside namespace +- Remove obsolete methods +- Mark non-async APIs obsolete +- Modified test projects to use async +- Ensured background tasks honored cancellation tokens +- Ability to specify a client's GUID before attempting to connect + +## Architecture + +Refer to [ARCHITECTURE.md](ARCHITECTURE.md) for a detailed overview of the internal design, message flow, threading model, and key design decisions. + +For the wire protocol specification (header format, delimiter, payload layout), see [FRAMING.md](FRAMING.md). + ## Test Applications Test projects for both client and server are included which will help you understand and exercise the class library. Shared automated coverage lives in `Test.Shared`, while `Test.Automated`, `Test.XUnit`, and `Test.NUnit` are the supported unattended test hosts. - -## SSL - -WatsonTcp supports data exchange with or without SSL. The server and client classes include constructors that allow you to include fields for the PFX certificate file and password. An example certificate can be found in the test projects, which has a password of 'password'. - -## To Stream or Not To Stream... - + +## SSL + +WatsonTcp supports data exchange with or without SSL. The server and client classes include constructors that allow you to include fields for the PFX certificate file and password. An example certificate can be found in the test projects, which has a password of 'password'. + +## To Stream or Not To Stream... + WatsonTcp allows you to receive messages using either byte arrays or streams. - Set `Events.MessageReceived` if you want a buffered `byte[]` @@ -263,334 +290,334 @@ server.Callbacks.StreamReceivedAsync = async (args, token) => Console.WriteLine("Received " + ms.Length + " bytes"); }; ``` - -## Including Metadata with a Message - + +## Including Metadata with a Message + Should you with to include metadata with any message, use the `Send` or `SendAsync` method that allows you to pass in metadata (`Dictionary`). Refer to the `TestClient`, `TestServer`, `TestClientStream`, and `TestServerStream` projects for a full example. Keys must be of type `string`. - -Note: if you use a class instance as either the value, you'll need to deserialize on the receiving end from JSON. -``` -object myVal = args.Metadata["myKey"]; -MyClass instance = myVal.ToObject(); -``` - -This is not necessary if you are using simple types (int, string, etc). Simply cast to the simple type. - -**IMPORTANT** - + +Note: if you use a class instance as either the value, you'll need to deserialize on the receiving end from JSON. +``` +object myVal = args.Metadata["myKey"]; +MyClass instance = myVal.ToObject(); +``` + +This is not necessary if you are using simple types (int, string, etc). Simply cast to the simple type. + +**IMPORTANT** + Metadata is serialized into the message header as JSON, increasing header size. While v6.1.0 significantly improved header parsing performance (eliminating O(n^2) allocations), it is still recommended to keep metadata small (less than 1KB) as large metadata increases JSON serialization overhead and network transfer time. Use `Settings.MaxHeaderSize` to control the maximum allowed header size (default 256KB). - -### Local vs External Connections - -**IMPORTANT** -* If you specify ```127.0.0.1``` as the listener IP address in WatsonTcpServer, it will only be able to accept connections from within the local host. -* To accept connections from other machines: - * Use a specific interface IP address, or - * Use ```null```, ```*```, ```+```, or ```0.0.0.0``` for the listener IP address (requires admin privileges to listen on any IP address) -* Make sure you create a permit rule on your firewall to allow inbound connections on that port -* If you use a port number under 1024, admin privileges will be required - -## Running under Mono - -.NET Core should always be the preferred option for multi-platform deployments. However, WatsonTcp works well in Mono environments with the .NET Framework to the extent that we have tested it. It is recommended that when running under Mono, you execute the containing EXE using --server and after using the Mono Ahead-of-Time Compiler (AOT). Note that TLS 1.2 is hard-coded, which may need to be downgraded to TLS in Mono environments. - -NOTE: Windows accepts '0.0.0.0' as an IP address representing any interface. On Mac and Linux you must be specified ('127.0.0.1' is also acceptable, but '0.0.0.0' is NOT). -``` -mono --aot=nrgctx-trampolines=8096,nimt-trampolines=8096,ntrampolines=4048 --server myapp.exe -mono --server myapp.exe -``` - -## Examples - -The following examples show a simple client and server example using WatsonTcp without SSL and consuming messages using byte arrays instead of streams. For full examples, please refer to the ```Test.*``` projects. - -### Server -```csharp -using WatsonTcp; - -static void Main(string[] args) -{ - WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000); - server.Events.ClientConnected += ClientConnected; - server.Events.ClientDisconnected += ClientDisconnected; - server.Events.MessageReceived += MessageReceived; - server.Callbacks.SyncRequestReceivedAsync = SyncRequestReceived; - server.Start(); - - // list clients - IEnumerable clients = server.ListClients(); - - // send a message - await server.SendAsync([guid], "Hello, client!"); - - // send a message with metadata - Dictionary md = new Dictionary(); - md.Add("foo", "bar"); - await server.SendAsync([guid], "Hello, client! Here's some metadata!", md); - - // send and wait for a response - try - { - SyncResponse resp = await server.SendAndWaitAsync( - [guid], - 5000, - "Hey, say hello back within 5 seconds!"); - - Console.WriteLine("My friend says: " + Encoding.UTF8.GetString(resp.Data)); - } - catch (TimeoutException) - { - Console.WriteLine("Too slow..."); - } -} - -static void ClientConnected(object sender, ConnectionEventArgs args) -{ - Console.WriteLine("Client connected: " + args.Client.ToString()); -} - -static void ClientDisconnected(object sender, DisconnectionEventArgs args) -{ - Console.WriteLine( - "Client disconnected: " - + args.Client.ToString() - + ": " - + args.Reason.ToString()); -} - -static void MessageReceived(object sender, MessageReceivedEventArgs args) -{ - Console.WriteLine( - "Message from " - + args.Client.ToString() - + ": " - + Encoding.UTF8.GetString(args.Data)); -} - -static async Task SyncRequestReceived(SyncRequest req) -{ - return new SyncResponse(req, "Hello back at you!"); -} -``` - -### Client -```csharp -using WatsonTcp; - -static void Main(string[] args) -{ - WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); - client.Events.ServerConnected += ServerConnected; - client.Events.ServerDisconnected += ServerDisconnected; - client.Events.MessageReceived += MessageReceived; - client.Callbacks.SyncRequestReceivedAsync = SyncRequestReceived; - client.Connect(); - - // check connectivity - Console.WriteLine("Am I connected? " + client.Connected); - - // send a message - client.Send("Hello!"); - - // send a message with metadata - Dictionary md = new Dictionary(); - md.Add("foo", "bar"); - await client.SendAsync("Hello, client! Here's some metadata!", md); - - // send and wait for a response - try - { - SyncResponse resp = await client.SendAndWaitAsync( - 5000, - "Hey, say hello back within 5 seconds!"); - - Console.WriteLine("My friend says: " + Encoding.UTF8.GetString(resp.Data)); - } - catch (TimeoutException) - { - Console.WriteLine("Too slow..."); - } -} - -static void MessageReceived(object sender, MessageReceivedEventArgs args) -{ - Console.WriteLine("Message from server: " + Encoding.UTF8.GetString(args.Data)); -} - -static void ServerConnected(object sender, ConnectionEventArgs args) -{ - Console.WriteLine("Server connected"); -} - -static void ServerDisconnected(object sender, DisconnectionEventArgs args) -{ - Console.WriteLine("Server disconnected"); -} - -static async Task SyncRequestReceived(SyncRequest req) -{ - return new SyncResponse(req, "Hello back at you!"); -} -``` - -## Example with SSL - -The examples above can be modified to use SSL as follows. No other changes are needed. Ensure that the certificate is exported as a PFX file and is resident in the directory of execution. -```csharp -// server -WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000, "test.pfx", "password"); -server.Settings.AcceptInvalidCertificates = true; -server.Settings.MutuallyAuthenticate = true; -server.Start(); - -// client -WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000, "test.pfx", "password"); -client.Settings.AcceptInvalidCertificates = true; -client.Settings.MutuallyAuthenticate = true; -client.Connect(); -``` - -## Example with Streams - -Refer to the ```Test.ClientStream``` and ```Test.ServerStream``` projects for a full example. -```csharp -// server -WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000); -server.Events.ClientConnected += ClientConnected; -server.Events.ClientDisconnected += ClientDisconnected; -server.Events.StreamReceived += StreamReceived; -server.Start(); - -static void StreamReceived(object sender, StreamReceivedEventArgs args) -{ - long bytesRemaining = args.ContentLength; - int bytesRead = 0; - byte[] buffer = new byte[65536]; - - using (MemoryStream ms = new MemoryStream()) - { - while (bytesRemaining > 0) - { - bytesRead = args.DataStream.Read(buffer, 0, buffer.Length); - if (bytesRead > 0) - { - ms.Write(buffer, 0, bytesRead); - bytesRemaining -= bytesRead; - } - } - } - - Console.WriteLine( - "Stream received from " - + args.Client.ToString() - + ": " - + Encoding.UTF8.GetString(ms.ToArray())); -} - -// client -WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); -client.Events.ServerConnected += ServerConnected; -client.Events.ServerDisconnected += ServerDisconnected; -client.Events.StreamReceived += StreamReceived; -client.Connect(); - -static void StreamReceived(object sender, StreamReceivedEventArgs args) -{ - long bytesRemaining = args.ContentLength; - int bytesRead = 0; - byte[] buffer = new byte[65536]; - - using (MemoryStream ms = new MemoryStream()) - { - while (bytesRemaining > 0) - { - bytesRead = args.DataStream.Read(buffer, 0, buffer.Length); - if (bytesRead > 0) - { - ms.Write(buffer, 0, bytesRead); - bytesRemaining -= bytesRead; - } - } - } - - Console.WriteLine("Stream received from server: " + Encoding.UTF8.GetString(ms.ToArray())); -} -``` - -## Specifying a Client GUID - -If you wish to specify a client's GUID, you can modify ```WatsonTcpClient.Settings.Guid``` prior to calling ```WatsonTcpClient.Connect()```. - -```csharp -WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); -client.Events.ServerConnected += ServerConnected; -client.Events.ServerDisconnected += ServerDisconnected; -client.Events.StreamReceived += StreamReceived; -client.Settings.Guid = Guid.Parse("12345678-1234-1234-123456781234"); -client.Connect(); -``` - -## Troubleshooting - -The first step in troubleshooting is to implement a logging method and attach it to ```Settings.Logger```, and as a general best practice while debugging, set ```Settings.DebugMessages``` to ```true```. - -```csharp -client.Settings.DebugMessages = true; -client.Settings.Logger = MyLoggerMethod; - -private void MyLoggerMethod(Severity sev, string msg) -{ - Console.WriteLine(sev.ToString() + ": " + msg); -} -``` - -Additionally it is recommended that you implement the ```Events.ExceptionEncountered``` event. - -```csharp -client.Events.ExceptionEncountered += MyExceptionEvent; - -private void MyExceptionEvent(object sender, ExceptionEventArgs args) -{ - Console.WriteLine(args.Json); -} -``` - -## Disconnection Handling - -The project TcpTest (https://github.com/jchristn/TcpTest) was built specifically to provide a reference for WatsonTcp to handle a variety of disconnection scenarios. The disconnection tests for which WatsonTcp is evaluated include: - -| Test case | Description | Pass/Fail | -|---|---|---| -| Server-side dispose | Graceful termination of all client connections | PASS | -| Server-side client removal | Graceful termination of a single client | PASS | -| Server-side termination | Abrupt termination due to process abort or CTRL-C | PASS | -| Client-side dispose | Graceful termination of a client connection | PASS | -| Client-side termination | Abrupt termination due to a process abort or CTRL-C | PASS | -| Network interface down | Network interface disabled or cable removed | Partial (see below) | - -Additionally, as of v4.3.0, support for TCP keepalives has been added to WatsonTcp, primarily to address the issue of a network interface being shut down, the cable unplugged, or the media otherwise becoming unavailable. It is important to note that keepalives are supported in .NET Core and .NET Framework, but NOT .NET Standard. As of this release, .NET Standard provides no facilities for TCP keepalives. - -TCP keepalives are NOT enabled by default. To enable and configure: -```csharp -server.Keepalive.EnableTcpKeepAlives = true; -server.Keepalive.TcpKeepAliveInterval = 5; // seconds to wait before sending subsequent keepalive -server.Keepalive.TcpKeepAliveTime = 5; // seconds to wait before sending a keepalive -server.Keepalive.TcpKeepAliveRetryCount = 5; // number of failed keepalive probes before terminating connection -``` - -Some important notes about TCP keepalives: - -- Keepalives only work in .NET Core and .NET Framework -- ```Keepalive.TcpKeepAliveRetryCount``` is only applicable to .NET Core; for .NET Framework, this value is forced to 10 - -## Disconnecting Idle Clients - -If you wish to have WatsonTcpServer automatically disconnect clients that have been idle for a period of time, set ```WatsonTcpServer.IdleClientTimeoutSeconds``` to a positive integer. Receiving a message from a client automatically resets their timeout. Client timeouts are evaluated every 5 seconds by Watson, so the disconnection may not be precise (for instance, if you use 7 seconds as your disconnect interval). - -## Donations - -If you would like to financially support my efforts, first of all, thank you! Please refer to DONATIONS.md. - -## Version History - -Please refer to CHANGELOG.md for details. + +### Local vs External Connections + +**IMPORTANT** +* If you specify ```127.0.0.1``` as the listener IP address in WatsonTcpServer, it will only be able to accept connections from within the local host. +* To accept connections from other machines: + * Use a specific interface IP address, or + * Use ```null```, ```*```, ```+```, or ```0.0.0.0``` for the listener IP address (requires admin privileges to listen on any IP address) +* Make sure you create a permit rule on your firewall to allow inbound connections on that port +* If you use a port number under 1024, admin privileges will be required + +## Running under Mono + +.NET Core should always be the preferred option for multi-platform deployments. However, WatsonTcp works well in Mono environments with the .NET Framework to the extent that we have tested it. It is recommended that when running under Mono, you execute the containing EXE using --server and after using the Mono Ahead-of-Time Compiler (AOT). Note that TLS 1.2 is hard-coded, which may need to be downgraded to TLS in Mono environments. + +NOTE: Windows accepts '0.0.0.0' as an IP address representing any interface. On Mac and Linux you must be specified ('127.0.0.1' is also acceptable, but '0.0.0.0' is NOT). +``` +mono --aot=nrgctx-trampolines=8096,nimt-trampolines=8096,ntrampolines=4048 --server myapp.exe +mono --server myapp.exe +``` + +## Examples + +The following examples show a simple client and server example using WatsonTcp without SSL and consuming messages using byte arrays instead of streams. For full examples, please refer to the ```Test.*``` projects. + +### Server +```csharp +using WatsonTcp; + +static void Main(string[] args) +{ + WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000); + server.Events.ClientConnected += ClientConnected; + server.Events.ClientDisconnected += ClientDisconnected; + server.Events.MessageReceived += MessageReceived; + server.Callbacks.SyncRequestReceivedAsync = SyncRequestReceived; + server.Start(); + + // list clients + IEnumerable clients = server.ListClients(); + + // send a message + await server.SendAsync([guid], "Hello, client!"); + + // send a message with metadata + Dictionary md = new Dictionary(); + md.Add("foo", "bar"); + await server.SendAsync([guid], "Hello, client! Here's some metadata!", md); + + // send and wait for a response + try + { + SyncResponse resp = await server.SendAndWaitAsync( + [guid], + 5000, + "Hey, say hello back within 5 seconds!"); + + Console.WriteLine("My friend says: " + Encoding.UTF8.GetString(resp.Data)); + } + catch (TimeoutException) + { + Console.WriteLine("Too slow..."); + } +} + +static void ClientConnected(object sender, ConnectionEventArgs args) +{ + Console.WriteLine("Client connected: " + args.Client.ToString()); +} + +static void ClientDisconnected(object sender, DisconnectionEventArgs args) +{ + Console.WriteLine( + "Client disconnected: " + + args.Client.ToString() + + ": " + + args.Reason.ToString()); +} + +static void MessageReceived(object sender, MessageReceivedEventArgs args) +{ + Console.WriteLine( + "Message from " + + args.Client.ToString() + + ": " + + Encoding.UTF8.GetString(args.Data)); +} + +static async Task SyncRequestReceived(SyncRequest req) +{ + return new SyncResponse(req, "Hello back at you!"); +} +``` + +### Client +```csharp +using WatsonTcp; + +static void Main(string[] args) +{ + WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); + client.Events.ServerConnected += ServerConnected; + client.Events.ServerDisconnected += ServerDisconnected; + client.Events.MessageReceived += MessageReceived; + client.Callbacks.SyncRequestReceivedAsync = SyncRequestReceived; + client.Connect(); + + // check connectivity + Console.WriteLine("Am I connected? " + client.Connected); + + // send a message + client.Send("Hello!"); + + // send a message with metadata + Dictionary md = new Dictionary(); + md.Add("foo", "bar"); + await client.SendAsync("Hello, client! Here's some metadata!", md); + + // send and wait for a response + try + { + SyncResponse resp = await client.SendAndWaitAsync( + 5000, + "Hey, say hello back within 5 seconds!"); + + Console.WriteLine("My friend says: " + Encoding.UTF8.GetString(resp.Data)); + } + catch (TimeoutException) + { + Console.WriteLine("Too slow..."); + } +} + +static void MessageReceived(object sender, MessageReceivedEventArgs args) +{ + Console.WriteLine("Message from server: " + Encoding.UTF8.GetString(args.Data)); +} + +static void ServerConnected(object sender, ConnectionEventArgs args) +{ + Console.WriteLine("Server connected"); +} + +static void ServerDisconnected(object sender, DisconnectionEventArgs args) +{ + Console.WriteLine("Server disconnected"); +} + +static async Task SyncRequestReceived(SyncRequest req) +{ + return new SyncResponse(req, "Hello back at you!"); +} +``` + +## Example with SSL + +The examples above can be modified to use SSL as follows. No other changes are needed. Ensure that the certificate is exported as a PFX file and is resident in the directory of execution. +```csharp +// server +WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000, "test.pfx", "password"); +server.Settings.AcceptInvalidCertificates = true; +server.Settings.MutuallyAuthenticate = true; +server.Start(); + +// client +WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000, "test.pfx", "password"); +client.Settings.AcceptInvalidCertificates = true; +client.Settings.MutuallyAuthenticate = true; +client.Connect(); +``` + +## Example with Streams + +Refer to the ```Test.ClientStream``` and ```Test.ServerStream``` projects for a full example. +```csharp +// server +WatsonTcpServer server = new WatsonTcpServer("127.0.0.1", 9000); +server.Events.ClientConnected += ClientConnected; +server.Events.ClientDisconnected += ClientDisconnected; +server.Events.StreamReceived += StreamReceived; +server.Start(); + +static void StreamReceived(object sender, StreamReceivedEventArgs args) +{ + long bytesRemaining = args.ContentLength; + int bytesRead = 0; + byte[] buffer = new byte[65536]; + + using (MemoryStream ms = new MemoryStream()) + { + while (bytesRemaining > 0) + { + bytesRead = args.DataStream.Read(buffer, 0, buffer.Length); + if (bytesRead > 0) + { + ms.Write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + } + + Console.WriteLine( + "Stream received from " + + args.Client.ToString() + + ": " + + Encoding.UTF8.GetString(ms.ToArray())); +} + +// client +WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); +client.Events.ServerConnected += ServerConnected; +client.Events.ServerDisconnected += ServerDisconnected; +client.Events.StreamReceived += StreamReceived; +client.Connect(); + +static void StreamReceived(object sender, StreamReceivedEventArgs args) +{ + long bytesRemaining = args.ContentLength; + int bytesRead = 0; + byte[] buffer = new byte[65536]; + + using (MemoryStream ms = new MemoryStream()) + { + while (bytesRemaining > 0) + { + bytesRead = args.DataStream.Read(buffer, 0, buffer.Length); + if (bytesRead > 0) + { + ms.Write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + } + + Console.WriteLine("Stream received from server: " + Encoding.UTF8.GetString(ms.ToArray())); +} +``` + +## Specifying a Client GUID + +If you wish to specify a client's GUID, you can modify ```WatsonTcpClient.Settings.Guid``` prior to calling ```WatsonTcpClient.Connect()```. + +```csharp +WatsonTcpClient client = new WatsonTcpClient("127.0.0.1", 9000); +client.Events.ServerConnected += ServerConnected; +client.Events.ServerDisconnected += ServerDisconnected; +client.Events.StreamReceived += StreamReceived; +client.Settings.Guid = Guid.Parse("12345678-1234-1234-123456781234"); +client.Connect(); +``` + +## Troubleshooting + +The first step in troubleshooting is to implement a logging method and attach it to ```Settings.Logger```, and as a general best practice while debugging, set ```Settings.DebugMessages``` to ```true```. + +```csharp +client.Settings.DebugMessages = true; +client.Settings.Logger = MyLoggerMethod; + +private void MyLoggerMethod(Severity sev, string msg) +{ + Console.WriteLine(sev.ToString() + ": " + msg); +} +``` + +Additionally it is recommended that you implement the ```Events.ExceptionEncountered``` event. + +```csharp +client.Events.ExceptionEncountered += MyExceptionEvent; + +private void MyExceptionEvent(object sender, ExceptionEventArgs args) +{ + Console.WriteLine(args.Json); +} +``` + +## Disconnection Handling + +The project TcpTest (https://github.com/jchristn/TcpTest) was built specifically to provide a reference for WatsonTcp to handle a variety of disconnection scenarios. The disconnection tests for which WatsonTcp is evaluated include: + +| Test case | Description | Pass/Fail | +|---|---|---| +| Server-side dispose | Graceful termination of all client connections | PASS | +| Server-side client removal | Graceful termination of a single client | PASS | +| Server-side termination | Abrupt termination due to process abort or CTRL-C | PASS | +| Client-side dispose | Graceful termination of a client connection | PASS | +| Client-side termination | Abrupt termination due to a process abort or CTRL-C | PASS | +| Network interface down | Network interface disabled or cable removed | Partial (see below) | + +Additionally, as of v4.3.0, support for TCP keepalives has been added to WatsonTcp, primarily to address the issue of a network interface being shut down, the cable unplugged, or the media otherwise becoming unavailable. It is important to note that keepalives are supported in .NET Core and .NET Framework, but NOT .NET Standard. As of this release, .NET Standard provides no facilities for TCP keepalives. + +TCP keepalives are NOT enabled by default. To enable and configure: +```csharp +server.Keepalive.EnableTcpKeepAlives = true; +server.Keepalive.TcpKeepAliveInterval = 5; // seconds to wait before sending subsequent keepalive +server.Keepalive.TcpKeepAliveTime = 5; // seconds to wait before sending a keepalive +server.Keepalive.TcpKeepAliveRetryCount = 5; // number of failed keepalive probes before terminating connection +``` + +Some important notes about TCP keepalives: + +- Keepalives only work in .NET Core and .NET Framework +- ```Keepalive.TcpKeepAliveRetryCount``` is only applicable to .NET Core; for .NET Framework, this value is forced to 10 + +## Disconnecting Idle Clients + +If you wish to have WatsonTcpServer automatically disconnect clients that have been idle for a period of time, set ```WatsonTcpServer.IdleClientTimeoutSeconds``` to a positive integer. Receiving a message from a client automatically resets their timeout. Client timeouts are evaluated every 5 seconds by Watson, so the disconnection may not be precise (for instance, if you use 7 seconds as your disconnect interval). + +## Donations + +If you would like to financially support my efforts, first of all, thank you! Please refer to DONATIONS.md. + +## Version History + +Please refer to CHANGELOG.md for details. diff --git a/TELEMETRY.md b/TELEMETRY.md new file mode 100644 index 0000000..b20e92a --- /dev/null +++ b/TELEMETRY.md @@ -0,0 +1,273 @@ +# WatsonTcp Telemetry + +WatsonTcp emits **metrics** and **distributed-tracing spans** using only the .NET base +class library (`System.Diagnostics.Metrics` and `System.Diagnostics.ActivitySource`). Any +host that understands these APIs — [Radiant](https://github.com/jchristn), the OpenTelemetry +SDK, Prometheus, Grafana, Jaeger, or your own listener — can observe a WatsonTcp client or +server. **WatsonTcp takes no dependency on any telemetry backend.** Your code emits; the +host subscribes by name. + +- **Meter name:** `WatsonTcp` +- **ActivitySource name:** `WatsonTcp` + +Both names are stable across releases and are exposed as public constants on +`WatsonTcp.WatsonTcpMetrics` (`WatsonTcpMetrics.MeterName`, `WatsonTcpMetrics.ActivitySourceName`). + +Telemetry is **on by default** and is a near-free no-op (roughly 1–5 ns, zero allocation) +when nobody is listening, so there is nothing to turn on to start emitting — you only wire up +a consumer. + +--- + +## 1. Quick start + +### Consume with Radiant + +At your application's composition root, subscribe Radiant to the two source names: + +```csharp +using Radiant; +using WatsonTcp; + +RadiantSettings settings = new RadiantSettings("my-service"); +settings.Otlp.Endpoint = "http://localhost:4317"; +settings.Prometheus.Enable = true; +settings.Prometheus.Port = 9464; + +settings.Sources.AddMeter(WatsonTcpMetrics.MeterName); // "WatsonTcp" +settings.Sources.AddActivitySource(WatsonTcpMetrics.MeterName); // "WatsonTcp" + +using (RadiantHost host = RadiantHost.Start(settings)) +{ + // ... run your WatsonTcpServer / WatsonTcpClient as usual ... +} +``` + +That is the entire integration. WatsonTcp itself needs no changes. + +### Consume with the OpenTelemetry SDK + +```csharp +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +using MeterProvider metrics = Sdk.CreateMeterProviderBuilder() + .AddMeter("WatsonTcp") + .AddPrometheusHttpListener() // or .AddOtlpExporter() + .Build(); + +using TracerProvider traces = Sdk.CreateTracerProviderBuilder() + .AddSource("WatsonTcp") + .AddOtlpExporter() + .Build(); +``` + +Wildcard subscription (`AddMeter("WatsonTcp*")`) also works, should future versions add +sub-meters. + +### Consume with only the base class library + +No SDK is required to read the numbers. A `MeterListener` (metrics) or `ActivityListener` +(spans) subscribes directly: + +```csharp +using System.Diagnostics.Metrics; + +MeterListener listener = new MeterListener(); +listener.InstrumentPublished = (instrument, l) => +{ + if (instrument.Meter.Name == "WatsonTcp") l.EnableMeasurementEvents(instrument); +}; +listener.SetMeasurementEventCallback((inst, value, tags, state) => + Console.WriteLine($"{inst.Name} += {value}")); +listener.Start(); +``` + +--- + +## 2. Turning telemetry off + +Because unobserved recording is essentially free, telemetry is enabled by default. You can +disable it per instance: + +```csharp +server.Settings.EnableMetrics = false; // no Meter is created; zero overhead +server.Settings.EnableTracing = false; // no spans are created + +client.Settings.EnableMetrics = false; +client.Settings.EnableTracing = false; +``` + +When `EnableMetrics` is `false`, WatsonTcp does not create a `Meter` at all, so no +instruments are ever published. `EnableTracing` gates span creation independently. + +--- + +## 3. Metric catalog + +All metrics are recorded on the `Meter` named `WatsonTcp`. Names are dotted and lowercase; +units are UCUM. The right-most column shows the approximate series name the OpenTelemetry +Prometheus exporter produces (it lowercases, replaces `.` with `_`, and appends unit and +`_total` suffixes; `{…}` annotation units add no suffix). + +| Metric name | Instrument | Unit | Tags | Meaning | Prometheus series | +|-------------|-----------|------|------|---------|-------------------| +| `watsontcp.messages.sent` | Counter | `{message}` | role, protocol, message.kind | Messages written to the wire | `watsontcp_messages_sent_total` | +| `watsontcp.messages.received` | Counter | `{message}` | role, protocol, message.kind | Messages read from the wire | `watsontcp_messages_received_total` | +| `watsontcp.bytes.sent` | Counter | `By` | role, protocol | Payload bytes sent | `watsontcp_bytes_sent_bytes_total` | +| `watsontcp.bytes.received` | Counter | `By` | role, protocol | Payload bytes received | `watsontcp_bytes_received_bytes_total` | +| `watsontcp.message.sent.size` | Histogram | `By` | role, protocol | Distribution of sent message sizes | `watsontcp_message_sent_size_bytes` | +| `watsontcp.message.received.size` | Histogram | `By` | role, protocol | Distribution of received message sizes | `watsontcp_message_received_size_bytes` | +| `watsontcp.message.send.duration` | Histogram | `s` | role, protocol | Time to write a message to the stream | `watsontcp_message_send_duration_seconds` | +| `watsontcp.connections.active` | ObservableGauge | `{connection}` | role, protocol | Current live connections | `watsontcp_connections_active` | +| `watsontcp.connections.pending` | ObservableGauge | `{connection}` | role, protocol | Accepted-but-not-yet-admitted clients (server) | `watsontcp_connections_pending` | +| `watsontcp.connections.total` | Counter | `{connection}` | role, protocol, outcome | Connection admission outcomes | `watsontcp_connections_total` | +| `watsontcp.disconnections.total` | Counter | `{connection}` | role, protocol, reason | Disconnections by reason | `watsontcp_disconnections_total` | +| `watsontcp.handshakes.total` | Counter | `{handshake}` | role, protocol, outcome | Custom-handshake completions | `watsontcp_handshakes_total` | +| `watsontcp.handshake.duration` | Histogram | `s` | role, protocol, outcome | Handshake duration | `watsontcp_handshake_duration_seconds` | +| `watsontcp.authentications.total` | Counter | `{authentication}` | role, protocol, outcome | Preshared-key auth results | `watsontcp_authentications_total` | +| `watsontcp.authorizations.total` | Counter | `{authorization}` | role, protocol, outcome | Connection-authorization results | `watsontcp_authorizations_total` | +| `watsontcp.sync.requests.sent` | Counter | `{request}` | role, protocol | `SendAndWaitAsync` requests issued | `watsontcp_sync_requests_sent_total` | +| `watsontcp.sync.responses.received` | Counter | `{response}` | role, protocol | Sync responses matched to a request | `watsontcp_sync_responses_received_total` | +| `watsontcp.sync.duration` | Histogram | `s` | role, protocol, outcome | Sync round-trip time | `watsontcp_sync_duration_seconds` | +| `watsontcp.sync.pending` | ObservableGauge | `{request}` | role, protocol | In-flight synchronous conversations | `watsontcp_sync_pending` | +| `watsontcp.sync.expired` | Counter | `{message}` | role, protocol, kind | Expired sync requests/responses discarded | `watsontcp_sync_expired_total` | +| `watsontcp.exceptions.total` | Counter | `{exception}` | role, protocol, exception.type | Exceptions surfaced via `ExceptionEncountered` | `watsontcp_exceptions_total` | +| `watsontcp.listener.transient_errors` | Counter | `{error}` | role, protocol, socket.error | Recovered transient accept-loop socket errors | `watsontcp_listener_transient_errors_total` | +| `watsontcp.stream.drained_bytes` | Counter | `By` | role, protocol | Unread stream bytes drained after a handler | `watsontcp_stream_drained_bytes_bytes_total` | +| `watsontcp.uptime` | ObservableGauge | `s` | role, protocol | Seconds since the client/server started | `watsontcp_uptime_seconds` | + +### Tag values + +Metric tags are deliberately **low cardinality** so every series stays enumerable. + +| Tag key | Values | +|---------|--------| +| `role` | `server`, `client` | +| `protocol` | `tcp`, `ssl` | +| `outcome` (connections) | `accepted`, `connected`, `rejected_maxconnections`, `rejected_notpermitted`, `rejected_blocked`, `rejected_authorization`, `failed` | +| `outcome` (handshake / auth / authorization) | `success`, `failure`, `timeout`, `canceled` | +| `outcome` (sync) | `completed`, `timeout` | +| `reason` | `Normal`, `Removed`, `Timeout`, `Shutdown`, `AuthFailure`, `ConnectionRejected`, `HandshakeFailure` | +| `message.kind` | `data`, `control`, `sync_request`, `sync_response` | +| `kind` (sync.expired) | `request`, `response` | +| `exception.type` | short exception type name (e.g. `IOException`, `SocketException`) | +| `socket.error` | `SocketError` value name (e.g. `ConnectionReset`) | + +All of these keys are available as public constants on `WatsonTcpMetrics` (`TagRole`, +`TagProtocol`, `TagOutcome`, `TagReason`, `TagMessageKind`, `TagKind`, `TagExceptionType`, +`TagSocketError`). + +--- + +## 4. Tracing spans + +Spans are started from the `ActivitySource` named `WatsonTcp`. They carry the +high-cardinality identifiers that metrics deliberately omit, so you can correlate an +individual connection or request without exploding your metric series. + +| Span name | Kind | Started when | Notable tags | +|-----------|------|--------------|--------------| +| `watsontcp.connect` | Client | A client connects (through registration/handshake, ends at disconnect) | `server.address`, `server.port`, `protocol`, `outcome`, `reason` | +| `watsontcp.session` | Server | A client is admitted (ends at disconnect) | `client.address`, `client.guid`, `protocol`, `reason` | +| `watsontcp.handshake` | Internal | A custom handshake runs | `role` | +| `watsontcp.send` | Producer | A message is sent | `message.bytes`, `message.sync`, `client.guid` (server) | +| `watsontcp.receive` | Consumer | A data message is processed | `message.bytes`, `client.guid` (server) | +| `watsontcp.sync` | Client | A `SendAndWaitAsync` round trip runs | `conversation.guid`, `message.bytes`, `outcome` | + +> **Note:** span tag values that are not strings (for example `message.bytes`, a 64-bit +> integer, or `message.sync`, a boolean) are stored on `Activity.TagObjects`, not on the +> string-only `Activity.Tags` collection. Enumerate `TagObjects` to see them all. + +--- + +## 5. Cardinality: what goes where + +WatsonTcp follows the OpenTelemetry guidance that metric labels must be low cardinality. + +- **Metric tags** are limited to values you could list on a whiteboard: `role`, `protocol`, + `outcome`, `reason`, `message.kind`, `kind`, `exception.type`, `socket.error`. +- **Client GUIDs, remote `ip:port`, and conversation GUIDs never appear on metrics.** They + are placed on spans (and are therefore available in your tracing backend and logs for + per-connection or per-request correlation). + +This keeps your time-series database from growing one series per client forever, while still +letting you drill into a specific connection through traces. + +--- + +## 6. Example queries and panels + +Once a Prometheus exporter is scraping (`watsontcp_*` series), useful PromQL includes: + +```promql +# Inbound message rate, server side +sum(rate(watsontcp_messages_received_total{role="server"}[1m])) + +# Live server connections +watsontcp_connections_active{role="server"} + +# 95th-percentile synchronous round-trip latency +histogram_quantile(0.95, sum(rate(watsontcp_sync_duration_seconds_bucket[5m])) by (le)) + +# Disconnections broken out by reason +sum(rate(watsontcp_disconnections_total[5m])) by (reason) + +# Connection rejections +sum(rate(watsontcp_connections_total{outcome=~"rejected_.*"}[5m])) by (outcome) + +# Handshake failure ratio +sum(rate(watsontcp_handshakes_total{outcome="failure"}[5m])) + / sum(rate(watsontcp_handshakes_total[5m])) +``` + +--- + +## 7. Target frameworks + +The diagnostics APIs WatsonTcp uses are in-box on **net8.0** and **net10.0**. For the +down-level target frameworks (**netstandard2.0**, **netstandard2.1**, **net462**, +**net48**), WatsonTcp references the `System.Diagnostics.DiagnosticSource` package (pinned to +`8.0.1`), which supplies the same `Meter`, `Counter`, `Histogram`, `ObservableGauge`, +`TagList`, and `ActivitySource` types. No action is required on your part; the reference is +part of the WatsonTcp package. + +If your host uses Radiant's `Radiant.SemConv` package for shared naming constants, note that +it too pins `System.Diagnostics.DiagnosticSource 8.0.1`, so the two unify cleanly. + +--- + +## 8. Public constants reference + +Everything a consumer needs is available symbolically on `WatsonTcp.WatsonTcpMetrics` so you +never have to hard-code a string: + +```csharp +WatsonTcpMetrics.MeterName // "WatsonTcp" +WatsonTcpMetrics.ActivitySourceName // "WatsonTcp" + +// Metric names, e.g.: +WatsonTcpMetrics.MessagesSent // "watsontcp.messages.sent" +WatsonTcpMetrics.ConnectionsActive // "watsontcp.connections.active" +WatsonTcpMetrics.SyncDuration // "watsontcp.sync.duration" + +// Span names, e.g.: +WatsonTcpMetrics.SpanSend // "watsontcp.send" + +// Tag keys, e.g.: +WatsonTcpMetrics.TagRole // "role" +WatsonTcpMetrics.TagOutcome // "outcome" +``` + +--- + +## 9. How it works internally (for the curious) + +Each `WatsonTcpClient` / `WatsonTcpServer` owns one instrumentation object created in +`ConnectAsync`/`Start`, holding a single `Meter` and `ActivitySource` and disposed with its +owner. Recording is threaded through the same points as the existing `WatsonTcpStatistics` +plus the connection, handshake, authentication, authorization, and synchronous-request +lifecycle. Every recording call is null-guarded and fire-and-forget, so telemetry can never +throw into the send, receive, or connection path. See `ARCHITECTURE.md` (§11) for details. +The original design/implementation plan is preserved at `archive/TELEMETRY_PLAN.md`. diff --git a/archive/TELEMETRY_PLAN.md b/archive/TELEMETRY_PLAN.md new file mode 100644 index 0000000..c62c974 --- /dev/null +++ b/archive/TELEMETRY_PLAN.md @@ -0,0 +1,563 @@ +# WatsonTcp Telemetry & Instrumentation Plan + +> **Status:** Proposed — target release **v6.4.0** (minor). +> **Author/Owner:** _unassigned_ +> **Tracking:** Check the boxes as work lands. Every table row and checklist item is independently actionable. + +This document is the implementation plan for adding standardized, vendor-neutral +telemetry (metrics + traces) to WatsonTcp so that [Radiant](file:///c:/code/radiant), +Prometheus, OpenTelemetry Collector, or any other consumer can observe a WatsonTcp +client or server with **zero coupling to any specific telemetry backend**. + +It is written to conform to Radiant's `c:\code\radiant\INTEGRATION.md` and to the +coding standards under `c:\code\agents\requirements`. + +--- + +## 1. Design principles (non-negotiable) + +These are lifted directly from `INTEGRATION.md` ("your code emits, the application hosts") +and are the contract this plan is built on. + +1. **BCL only — no Radiant dependency.** WatsonTcp emits into a `System.Diagnostics.Metrics.Meter` + and a `System.Diagnostics.ActivitySource`. Both ship in the platform. Radiant (or any + OpenTelemetry host) *subscribes by name*. WatsonTcp never references the `Radiant` + host SDK, `OpenTelemetry.*`, or `prometheus-net`. +2. **The name is the public API.** The `Meter` name and `ActivitySource` name — both + `"WatsonTcp"` — are a **stable, namespaced contract**. Treat them like public API: do + not rename across releases. They are exposed as public constants (see §4) so consumers + can reference them symbolically. +3. **Free when unobserved.** An unsubscribed `Counter.Add` / `Histogram.Record` is ~1–5 ns + and allocation-free when tags are passed through a stack-allocated `TagList`. Telemetry + is therefore always-on by default; a subscriber pays, nobody else does. +4. **Instrumentation must never break the primary path.** Every recording site is + fire-and-forget. Recording is a synchronous, non-throwing BCL call; where any + computation is required to build a value it is guarded so a telemetry failure can never + propagate into send/receive/connect logic. +5. **Low-cardinality metric tags only.** Metric dimensions are values you "could list on a + whiteboard": `role` (`server`/`client`), `protocol` (`tcp`/`ssl`), `outcome`, `reason`. + High-cardinality identifiers — client GUID, remote `ip:port`, conversation GUID — go on + **span tags** and **logs**, never on metric tags. +6. **OpenTelemetry semantic-convention style names & UCUM units.** Dotted, lowercase names + (`watsontcp.messages.sent`) and UCUM units (`s`, `By`, `{message}`, `{connection}`) so + the exporter's automatic Prometheus suffixing produces conventional series. + +--- + +## 2. Consumer quick-start (what a telemetry consumer needs to know) + +A Radiant host observes WatsonTcp with exactly two subscriptions at the composition root: + +```csharp +RadiantSettings settings = new RadiantSettings("my-service"); +settings.Otlp.Endpoint = "http://localhost:4317"; +settings.Prometheus.Enable = true; + +settings.Sources.AddMeter("WatsonTcp"); // metrics +settings.Sources.AddActivitySource("WatsonTcp"); // traces + +using (RadiantHost host = RadiantHost.Start(settings)) +{ + // ... run WatsonTcp client/server as usual ... +} +``` + +A raw OpenTelemetry host is equivalent: + +```csharp +Sdk.CreateMeterProviderBuilder().AddMeter("WatsonTcp").AddOtlpExporter().Build(); +Sdk.CreateTracerProviderBuilder().AddSource("WatsonTcp").AddOtlpExporter().Build(); +``` + +Wildcard subscription (`"WatsonTcp*"`) also works if we ever split into +`WatsonTcp.Server` / `WatsonTcp.Client` sub-meters (not planned for v6.4.0 — see §11). + +**Strings a consumer needs** (all published as public constants in code, see §4): + +| Concept | Value | +|---|---| +| Meter name | `WatsonTcp` | +| ActivitySource name | `WatsonTcp` | +| Metric name prefix | `watsontcp.` | +| Span name prefix | `watsontcp.` | +| Tag key: role | `role` (`server` \| `client`) | +| Tag key: protocol | `protocol` (`tcp` \| `ssl`) | +| Tag key: outcome | `outcome` | +| Tag key: disconnect reason | `reason` | + +--- + +## 3. Instrumentation surface — where telemetry is emitted in the stack + +The existing hand-rolled `WatsonTcpStatistics` (bytes/messages counters on both +`WatsonTcpClient` and `WatsonTcpServer`) stays as-is for backward compatibility. The new +`Meter`-based instrumentation is added **alongside** it, at the same proven insertion +points plus the connection/handshake/auth/sync lifecycle. + +``` + ┌───────────────────────────── WatsonTcp process ─────────────────────────────┐ + TCP accept ───▶ AcceptConnections ──▶ [connections.total{outcome}] [listener.transient_errors] + │ + ▼ + StartTls / Authorize / Handshake ──▶ [authorizations.total] [handshakes.total] + │ [handshake.duration] [span watsontcp.handshake] + ▼ + ActivateClient ─────────────────────▶ [connections.total{outcome=accepted}] + │ [connections.active gauge] [span watsontcp.session] + ▼ + read ◀── DataReceiver ──────────────────────────▶ [messages.received] [bytes.received] + │ [message.received.size] [span watsontcp.receive] + │ [sync.responses.received] [sync.expired] + ▼ + (handler) [stream.drained_bytes] [exceptions.total] + │ + write ◀── SendInternal / SendAndWaitInternal ─────▶ [messages.sent] [bytes.sent] + │ [message.sent.size] [message.send.duration] + │ [sync.requests.sent] [sync.duration{outcome}] + ▼ [span watsontcp.send] [span watsontcp.sync] + DataReceiver exit ──────────────────▶ [disconnections.total{reason}] + [connections.active gauge −1] +``` + +--- + +## 4. Metric catalog (the contract) + +All instruments live on `Meter("WatsonTcp")`. Names are dotted/lowercase; units are UCUM. +The right-most column is the approximate series name the OpenTelemetry Prometheus exporter +produces (it lowercases, replaces `.`→`_`, appends unit and `_total` suffixes; `{…}` +annotation units contribute no suffix). + +| # | Metric name | Kind | Unit | Base tags | Extra tags | Description | Prometheus series (approx.) | Done | +|---|-------------|------|------|-----------|-----------|-------------|------------------------------|:----:| +| M01 | `watsontcp.messages.sent` | Counter\ | `{message}` | role, protocol | message.kind | Messages written to the wire | `watsontcp_messages_sent_total` | ☐ | +| M02 | `watsontcp.messages.received` | Counter\ | `{message}` | role, protocol | message.kind | Messages read off the wire | `watsontcp_messages_received_total` | ☐ | +| M03 | `watsontcp.bytes.sent` | Counter\ | `By` | role, protocol | | Payload bytes sent (content length) | `watsontcp_bytes_sent_bytes_total` | ☐ | +| M04 | `watsontcp.bytes.received` | Counter\ | `By` | role, protocol | | Payload bytes received | `watsontcp_bytes_received_bytes_total` | ☐ | +| M05 | `watsontcp.message.sent.size` | Histogram\ | `By` | role, protocol | | Distribution of sent message sizes | `watsontcp_message_sent_size_bytes` | ☐ | +| M06 | `watsontcp.message.received.size` | Histogram\ | `By` | role, protocol | | Distribution of received message sizes | `watsontcp_message_received_size_bytes` | ☐ | +| M07 | `watsontcp.message.send.duration` | Histogram\ | `s` | role, protocol | | Time to write header+payload to the stream (buckets: `LatencyBuckets.Network`) | `watsontcp_message_send_duration_seconds` | ☐ | +| M08 | `watsontcp.connections.active` | ObservableGauge\ | `{connection}` | role, protocol | | Current live connections (server: `Connections`; client: `0/1`) | `watsontcp_connections_active` | ☐ | +| M09 | `watsontcp.connections.pending` | ObservableGauge\ | `{connection}` | role, protocol | | Accepted-but-not-yet-admitted clients (server only) | `watsontcp_connections_pending` | ☐ | +| M10 | `watsontcp.connections.total` | Counter\ | `{connection}` | role, protocol | outcome | Connection admission outcomes | `watsontcp_connections_total` | ☐ | +| M11 | `watsontcp.disconnections.total` | Counter\ | `{connection}` | role, protocol | reason | Disconnections by `DisconnectReason` | `watsontcp_disconnections_total` | ☐ | +| M12 | `watsontcp.handshakes.total` | Counter\ | `{handshake}` | role, protocol | outcome | Custom-handshake completions | `watsontcp_handshakes_total` | ☐ | +| M13 | `watsontcp.handshake.duration` | Histogram\ | `s` | role, protocol | outcome | Handshake begin→resolve time (buckets: `Network`) | `watsontcp_handshake_duration_seconds` | ☐ | +| M14 | `watsontcp.authentications.total` | Counter\ | `{authentication}` | role, protocol | outcome | Preshared-key auth results | `watsontcp_authentications_total` | ☐ | +| M15 | `watsontcp.authorizations.total` | Counter\ | `{authorization}` | role, protocol | outcome | Server `AuthorizeConnection` results | `watsontcp_authorizations_total` | ☐ | +| M16 | `watsontcp.sync.requests.sent` | Counter\ | `{request}` | role, protocol | | `SendAndWaitAsync` requests issued | `watsontcp_sync_requests_sent_total` | ☐ | +| M17 | `watsontcp.sync.responses.received` | Counter\ | `{response}` | role, protocol | | Sync responses matched to a request | `watsontcp_sync_responses_received_total` | ☐ | +| M18 | `watsontcp.sync.duration` | Histogram\ | `s` | role, protocol | outcome | Sync round-trip time, `outcome=completed\|timeout` (buckets: `Network`) | `watsontcp_sync_duration_seconds` | ☐ | +| M19 | `watsontcp.sync.pending` | ObservableGauge\ | `{request}` | role, protocol | | In-flight sync conversations (`_SyncRequests.Count`) | `watsontcp_sync_pending` | ☐ | +| M20 | `watsontcp.sync.expired` | Counter\ | `{message}` | role, protocol | kind | Expired sync request/response discarded, `kind=request\|response` | `watsontcp_sync_expired_total` | ☐ | +| M21 | `watsontcp.exceptions.total` | Counter\ | `{exception}` | role, protocol | exception.type | Exceptions surfaced through `ExceptionEncountered` | `watsontcp_exceptions_total` | ☐ | +| M22 | `watsontcp.listener.transient_errors` | Counter\ | `{error}` | role, protocol | socket.error | Recovered transient accept-loop socket errors | `watsontcp_listener_transient_errors_total` | ☐ | +| M23 | `watsontcp.stream.drained_bytes` | Counter\ | `By` | role, protocol | | Unread stream-payload bytes drained after a handler | `watsontcp_stream_drained_bytes_bytes_total` | ☐ | +| M24 | `watsontcp.uptime` | ObservableGauge\ | `s` | role, protocol | | Seconds since the client/server started | `watsontcp_uptime_seconds` | ☐ | + +### Tag value dictionaries (low-cardinality — the whiteboard test) + +| Tag key | Allowed values | Notes | +|---|---|---| +| `role` | `server`, `client` | Which side emitted. | +| `protocol` | `tcp`, `ssl` | Derived from `Mode`. Value `ssl` used for the TLS transport. | +| `outcome` (connections) | `accepted`, `connected`, `rejected_maxconnections`, `rejected_notpermitted`, `rejected_blocked`, `rejected_authorization`, `failed` | `connected` used on client side. | +| `outcome` (handshake/auth/authorization) | `success`, `failure`, `timeout`, `canceled` | | +| `outcome` (sync) | `completed`, `timeout` | | +| `reason` | `Normal`, `Removed`, `Timeout`, `Shutdown`, `AuthFailure`, `ConnectionRejected`, `HandshakeFailure` | Mirrors the public `DisconnectReason` enum member values. | +| `message.kind` | `data`, `control`, `sync_request`, `sync_response` | Optional dimension on M01/M02; `control` = register/status/auth/handshake frames. | +| `kind` (sync.expired) | `request`, `response` | | +| `exception.type` | short type name (`IOException`, `SocketException`, `TaskCanceledException`, `ObjectDisposedException`, `TimeoutException`, …) | Bounded set — the framework only ever raises a handful. | +| `socket.error` | `SocketError` enum name (`ConnectionReset`, `ConnectionAborted`, …) | Bounded. | + +> **Public API note:** These constants are published so a consumer can build dashboards +> without hard-coding strings. They also *are* the deliverable "namespaces/strings a +> consumer needs." + +--- + +## 5. Distributed tracing catalog (ActivitySource "WatsonTcp") + +Spans carry the high-cardinality identifiers that metrics deliberately omit. All spans are +created via `ActivitySource("WatsonTcp")`; when no tracer subscribes, `StartActivity` +returns `null` and the `using` wrapper is a no-op. + +| # | Span name | Kind | Started at | High-cardinality tags | Done | +|---|-----------|------|-----------|------------------------|:----:| +| T01 | `watsontcp.connect` | Client | `WatsonTcpClient.ConnectCoreAsync` (whole connect+register+handshake) | `server.address`, `server.port`, `protocol`, `outcome` | ☐ | +| T02 | `watsontcp.session` | Server | `WatsonTcpServer.ActivateClientAsync`→ends in `DataReceiver` exit | `client.address`, `client.port`, `client.guid`, `protocol`, `reason` | ☐ | +| T03 | `watsontcp.handshake` | Internal | `StartHandshakePhaseAsync` / `StartClientHandshake` | `role`, `outcome` | ☐ | +| T04 | `watsontcp.send` | Producer | `SendInternalAsync` | `message.bytes`, `message.sync`, `client.guid` (server) | ☐ | +| T05 | `watsontcp.receive` | Consumer | `DataReceiver` per data message | `message.bytes`, `client.guid` (server) | ☐ | +| T06 | `watsontcp.sync` | Client | `SendAndWaitInternalAsync` | `conversation.guid`, `message.bytes`, `outcome` | ☐ | + +Span tag keys follow OTel conventions (`server.address`, `server.port`, `client.address`). +`client.guid` / `conversation.guid` are exactly the values banned from metric tags. + +> **Scope decision for v6.4.0:** Metrics (M01–M24) are the committed deliverable. Tracing +> (T01–T06) is included in this plan and *may* be delivered in the same release, but if +> schedule pressure appears, tracing may be split to v6.5.0 without changing the metric +> contract. Mark the decision here: ☐ tracing in 6.4.0 ☐ tracing deferred. + +--- + +## 6. Code design + +### 6.1 New files (one type per file, per requirements) + +| File | Type | Visibility | Purpose | Done | +|---|---|---|---|:----:| +| `src/WatsonTcp/WatsonTcpMetrics.cs` | `static class WatsonTcpMetrics` | **public** | Public constants: `MeterName`, `ActivitySourceName`, every metric name, every tag key. The consumer-facing string contract. | ☐ | +| `src/WatsonTcp/WatsonTcpInstrumentation.cs` | `sealed class WatsonTcpInstrumentation : IDisposable` | internal | Owns one `Meter` + one `ActivitySource` and every instrument; exposes typed recording helpers and gauge callbacks; disposed with its owner. | ☐ | + +`WatsonTcpInstrumentation` is **instance-scoped** (one per `WatsonTcpClient` / +`WatsonTcpServer`), not static. Rationale: + +- ObservableGauges (M08/M09/M19/M24) read live instance state via callbacks captured in the + constructor — the correct BCL pattern for gauges and impossible to get right from a shared + static Meter across multiple instances. +- Deterministic lifecycle: the `Meter`/`ActivitySource` are disposed in the owner's + `Dispose(bool)`, so no leaked subscriptions. +- Multiple instances may create a `Meter` with the same name `"WatsonTcp"`; OpenTelemetry + aggregates them, so the *contract* (the name) is still singular and stable. + +Sketch (illustrative — not final code; must be fleshed out to full compliance): + +```csharp +namespace WatsonTcp +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.Metrics; + + internal sealed class WatsonTcpInstrumentation : IDisposable + { + #region Private-Members + + private readonly Meter _Meter; + private readonly ActivitySource _ActivitySource; + private readonly string _Role; // "server" | "client" + private readonly string _Protocol; // "tcp" | "ssl" + + private readonly Counter _MessagesSent; + private readonly Counter _BytesSent; + // ... remaining instruments ... + + #endregion + + #region Constructors-and-Factories + + internal WatsonTcpInstrumentation( + string role, + string protocol, + Func activeConnections, + Func pendingConnections, + Func pendingSyncRequests, + Func uptimeSeconds) + { + _Role = role ?? throw new ArgumentNullException(nameof(role)); + _Protocol = protocol ?? throw new ArgumentNullException(nameof(protocol)); + + _Meter = new Meter(WatsonTcpMetrics.MeterName); + _ActivitySource = new ActivitySource(WatsonTcpMetrics.ActivitySourceName); + + _MessagesSent = _Meter.CreateCounter( + WatsonTcpMetrics.MessagesSent, WatsonTcpMetrics.UnitMessage, "Messages written to the wire."); + // ... create the rest ... + + _Meter.CreateObservableGauge( + WatsonTcpMetrics.ConnectionsActive, + () => new Measurement(activeConnections(), BaseTags()), + WatsonTcpMetrics.UnitConnection); + // ... pending / sync.pending / uptime gauges ... + } + + #endregion + + #region Internal-Methods + + internal void MessageSent(long bytes) + { + TagList tags = BaseTags(); + _MessagesSent.Add(1, tags); + _BytesSent.Add(bytes, tags); + _MessageSentSize.Record(bytes, tags); + } + + // MessageReceived, ConnectionOutcome(string), Disconnection(DisconnectReason), + // Handshake(string outcome, double seconds), Authentication(string outcome), + // Authorization(string outcome), SyncRequestSent(), SyncCompleted(double, string outcome), + // SyncExpired(string kind), Exception(Exception), TransientAcceptError(SocketError), + // StreamDrained(long) ... + + internal Activity StartSend(long bytes) => + _ActivitySource.StartActivity(WatsonTcpMetrics.SpanSend, ActivityKind.Producer); + + #endregion + + #region Private-Methods + + private TagList BaseTags() + { + TagList tags = new TagList(); + tags.Add(WatsonTcpMetrics.TagRole, _Role); + tags.Add(WatsonTcpMetrics.TagProtocol, _Protocol); + return tags; + } + + #endregion + + #region IDisposable + + public void Dispose() + { + _ActivitySource.Dispose(); + _Meter.Dispose(); + } + + #endregion + } +} +``` + +### 6.2 Wiring into `WatsonTcpServer` + +| Insertion point (existing code) | Call to add | Metric/Span | Done | +|---|---|---|:----:| +| `Start()` after `_Statistics = new WatsonTcpStatistics();` | construct `_Instrumentation` (role `server`, protocol from `_Mode`, gauge callbacks → `Connections`, `PendingConnections`, `_SyncRequests.Count`, `Statistics.UpTime`) | M08/M09/M19/M24 | ☐ | +| `AcceptConnections` maxconn reject (`~L784`) | `ConnectionOutcome("rejected_maxconnections")` | M10 | ☐ | +| `AcceptConnections` not-permitted (`~L794`) | `ConnectionOutcome("rejected_notpermitted")` | M10 | ☐ | +| `AcceptConnections` blocked (`~L801`) | `ConnectionOutcome("rejected_blocked")` | M10 | ☐ | +| `AcceptConnections` transient catch (`~L773`) | `TransientAcceptError(e.SocketErrorCode)` | M22 | ☐ | +| `AuthorizePendingClientAsync` allow/reject/timeout | `Authorization(outcome)`; reject → also `ConnectionOutcome("rejected_authorization")` | M15/M10 | ☐ | +| `RunHandshakePhaseAsync` success (`~L1060`) / fail (`~L1068`) | `Handshake("success"/"failure"/"timeout", seconds)` | M12/M13, T03 | ☐ | +| `ActivateClientAsync` (`~L1088`) | `ConnectionOutcome("accepted")`; start `watsontcp.session` span stored on `ClientMetadata` | M10, T02 | ☐ | +| `DataReceiver` auth accepted (`~L1278`) / declined (`~L1287`,`~L1299`) | `Authentication("success"/"failure")` | M14 | ☐ | +| `DataReceiver` sync response matched (`~L1428`) | `SyncResponseReceived()` | M17 | ☐ | +| `DataReceiver` expired sync req (`~L1412`) / resp (`~L1437`) | `SyncExpired("request"/"response")` | M20 | ☐ | +| `DataReceiver` received accounting (`~L1463-1464`) | `MessageReceived(msg.ContentLength, kind)`; wrap handler in `watsontcp.receive` span | M02/M04/M06, T05 | ☐ | +| `DataReceiver` exit (`~L1513-1517`) | `Disconnection(reason)`; end session span | M11, T02 | ☐ | +| `SendInternalAsync` success (`~L1558-1559`) | `MessageSent(contentLength)`; time write for M07; `watsontcp.send` span | M01/M03/M05/M07, T04 | ☐ | +| `SendAndWaitInternalAsync` send (`~L1609-1610`) | `MessageSent(...)` + `SyncRequestSent()`; time round-trip → `SyncCompleted(seconds, outcome)`; `watsontcp.sync` span | M01/M03/M16/M18, T06 | ☐ | +| `HandleStreamPayloadAsync` drain finally | `StreamDrained(watsonStream.RemainingBytes)` | M23 | ☐ | +| every `_Events.HandleExceptionEncountered(...)` site | route through a new `private void HandleException(Exception e)` that calls `_Instrumentation?.Exception(e)` then raises the event | M21 | ☐ | +| `Dispose(bool disposing)` | `_Instrumentation?.Dispose(); _Instrumentation = null;` | lifecycle | ☐ | + +### 6.3 Wiring into `WatsonTcpClient` + +| Insertion point (existing code) | Call to add | Metric/Span | Done | +|---|---|---|:----:| +| `ConnectCoreAsync` after `_Statistics = new WatsonTcpStatistics();` (`~L552`) | construct `_Instrumentation` (role `client`, protocol from `_Mode`, gauges → `Connected?1:0`, `0`, `_SyncRequests.Count`, `Statistics.UpTime`); start `watsontcp.connect` span | M08/M19/M24, T01 | ☐ | +| `MarkConnected` (`~L739`) | `ConnectionOutcome("connected")` | M10 | ☐ | +| `RunClientHandshakeAsync` / `HandshakeSuccess` (`~L1196`) / failure | `Handshake(outcome, seconds)` | M12/M13, T03 | ☐ | +| `DataReceiver` `AuthSuccess` (`~L1110`) / `AuthFailure` (`~L1125`) | `Authentication("success"/"failure")` | M14 | ☐ | +| `DataReceiver` sync response matched (`~L1283`) | `SyncResponseReceived()` | M17 | ☐ | +| `DataReceiver` expired sync req (`~L1267`) / resp (`~L1292`) | `SyncExpired("request"/"response")` | M20 | ☐ | +| `DataReceiver` received accounting (`~L1320-1321`) | `MessageReceived(msg.ContentLength, kind)`; `watsontcp.receive` span | M02/M04/M06, T05 | ☐ | +| `DataReceiver` exit `HandleServerDisconnected` (`~L1392`) | `Disconnection(reason)`; end connect span | M11, T01 | ☐ | +| `SendInternalAsync` success (`~L1435-1436`) | `MessageSent(contentLength)`; time write; `watsontcp.send` span | M01/M03/M05/M07, T04 | ☐ | +| `SendAndWaitInternalAsync` send (`~L1501-1502`) | `MessageSent(...)` + `SyncRequestSent()`; time round-trip → `SyncCompleted(...)`; `watsontcp.sync` span | M01/M03/M16/M18, T06 | ☐ | +| `HandleStreamPayloadAsync` drain finally | `StreamDrained(...)` | M23 | ☐ | +| every `_Events.HandleExceptionEncountered(...)` site | route through `private void HandleException(Exception e)` | M21 | ☐ | +| `Dispose(bool disposing)` | `_Instrumentation?.Dispose(); _Instrumentation = null;` | lifecycle | ☐ | + +### 6.4 Histogram buckets + +Match Radiant presets (all in seconds, to line up with unit `s`): + +| Metric | Preset | Boundaries | +|---|---|---| +| M07 `message.send.duration` | `Network` | 0.01 … 120.0 | +| M13 `handshake.duration` | `Network` | 0.01 … 120.0 | +| M18 `sync.duration` | `Network` | 0.01 … 120.0 | + +WatsonTcp does not reference Radiant, so bucket boundaries are declared as an internal +`static readonly double[]` in `WatsonTcpInstrumentation` (documented as mirroring +`LatencyBuckets.Network`). The host chooses whether to apply them via an OTel `View`; the +producer only records raw values. Size histograms (M05/M06, unit `By`) use the default +OTel bucketing unless the host overrides. + +--- + +## 7. Configuration & opt-out + +Per requirements ("avoid constants a developer may want to change; use a public member with +a backing field and a sensible default"), expose an explicit switch on each settings class. +Because unobserved recording is already near-free, the default is **on**. + +| Setting | Type | Default | File | Done | +|---|---|---|---|:----:| +| `WatsonTcpClientSettings.EnableMetrics` | `bool` | `true` | `WatsonTcpClientSettings.cs` | ☐ | +| `WatsonTcpClientSettings.EnableTracing` | `bool` | `true` | `WatsonTcpClientSettings.cs` | ☐ | +| `WatsonTcpServerSettings.EnableMetrics` | `bool` | `true` | `WatsonTcpServerSettings.cs` | ☐ | +| `WatsonTcpServerSettings.EnableTracing` | `bool` | `true` | `WatsonTcpServerSettings.cs` | ☐ | + +When `EnableMetrics == false`, the owner passes a `null` instrumentation object (all call +sites use `_Instrumentation?.…`), so there is literally no `Meter` and zero cost. `EnableTracing` +gates only span creation. Both carry full XML docs stating default/meaning. + +--- + +## 8. Project / dependency changes + +`System.Diagnostics.Metrics` (`Meter`, `Counter`, `Histogram`, `UpDownCounter`, +`ObservableGauge`, `TagList`) and `System.Diagnostics.ActivitySource` are **in-box** on +`net8.0`/`net10.0` but ship via the `System.Diagnostics.DiagnosticSource` package for +`netstandard2.0`, `netstandard2.1`, `net462`, and `net48`. + +`src/WatsonTcp/WatsonTcp.csproj` changes: + +```xml + + + + +``` + +- [ ] Add the conditional `PackageReference` above (version pinned to `8.0.1`, matching + Radiant.SemConv's pin, so a Radiant host unifies cleanly). +- [ ] Confirm `TagList` and `Meter.CreateObservableGauge(...)` compile on **all six** TFMs + (they are available in DiagnosticSource ≥ 6.0; 8.0.1 covers net462). +- [ ] **Do NOT** add `OpenTelemetry`, `OpenTelemetry.Exporter.*`, or `Radiant`. +- [ ] *(Optional, defer)* Evaluate referencing `Radiant.SemConv` (`System.Diagnostics.DiagnosticSource`-only, + netstandard2.0-safe) for shared `SemConv.Attributes.Protocol` / `Convention` constants. + Default recommendation: **do not** take even this dependency for v6.4.0 — keep WatsonTcp + self-contained and publish our own constants in `WatsonTcpMetrics`. + +> **Compliance divergence (explicit & justified, per requirements):** The requirements' +> `BACKEND_TEST_ARCHITECTURE.md` prescribes `net8.0;net10.0` + `enable`. +> WatsonTcp is a pre-existing, widely-consumed multi-target library +> (`netstandard2.0;netstandard2.1;net462;net48;net8.0;net10.0`) without project-wide +> nullable enablement. Telemetry code follows every applicable **style** rule from +> `CODE_STYLE.md` / `BACKEND_ARCHITECTURE.md` (usings inside namespace, `_PascalCase` +> privates, XML docs on public members, `using(){}` blocks, one type per file, no `var`, +> no tuples, `Math.Clamp` on numeric settings, guard clauses, `Interlocked`/thread-safety +> notes) but **retains the library's existing TFM matrix and nullable posture**. This is a +> deliberate divergence recorded here rather than a regression of the shipping package's +> compatibility surface. + +--- + +## 9. Code-compliance checklist (`c:\code\agents\requirements`) + +Apply to `WatsonTcpMetrics.cs`, `WatsonTcpInstrumentation.cs`, and all edits. + +- [ ] `namespace WatsonTcp` first; `using` directives **inside** the namespace block. +- [ ] System/Microsoft usings alphabetized first, then others alphabetized. +- [ ] All public types/members/consts have XML `///` docs; **no** docs on private members. +- [ ] Public metric-name/tag constants documented with their meaning and stable-contract note. +- [ ] Private fields `_PascalCase` (`_Meter`, `_MessagesSent`, `_Instrumentation`). +- [ ] No `var`; no tuples. +- [ ] One class/enum per file. +- [ ] `IDisposable` implemented on `WatsonTcpInstrumentation`; owners call `.Dispose()` inside + their existing `protected virtual void Dispose(bool disposing)`. +- [ ] `using (…) { }` block form for any local disposables (e.g. `Activity`). +- [ ] `Interlocked` / thread-safety documented; counters are already thread-safe via `Meter`. +- [ ] `Math.Clamp` + documented default/min/max for any numeric setting introduced. +- [ ] Guard clauses (`ArgumentNullException`) on constructor/method inputs; nullable + call-site guards (`_Instrumentation?.`) everywhere. +- [ ] No `Console.*` anywhere in library code. +- [ ] Recording sites cannot throw into send/receive/connect paths (fire-and-forget). +- [ ] Builds warning-free on all six TFMs (`EnableNETAnalyzers` + `latest-recommended` already on). + +--- + +## 10. Testing plan + +Follow the Touchstone descriptor pattern already in `src/Test.Shared` (net8.0;net10.0), +executed via the console runner, xUnit, and NUnit projects. No console output in shared +test code; assert by throwing. + +- [ ] **In-memory metrics test** — attach an OTel `MeterProvider` with + `.AddMeter("WatsonTcp").AddInMemoryExporter(items)` (test-only dependency, not in the + library), run a client⇄server exchange, assert M01–M04 increment with the expected + `role`/`protocol` tags and that byte sums match payload sizes. +- [ ] **Connection lifecycle** — assert `connections.total{outcome=accepted}`, + `connections.active` gauge rises then returns to 0, and `disconnections.total{reason=…}` + records the correct `DisconnectReason` for normal, removed, timeout, and shutdown. +- [ ] **Sync round-trip** — assert `sync.requests.sent`, `sync.responses.received`, + `sync.duration{outcome=completed}`, and a forced timeout yields `outcome=timeout`. +- [ ] **Handshake / auth / authorization** — success and failure paths hit M12–M15 with + correct `outcome`. +- [ ] **Rejections** — permitted/blocked/max-connections/authorization-reject each hit + `connections.total` with the right `outcome`. +- [ ] **Cardinality guard** — a test asserting no metric carries a GUID/endpoint tag key + (only `role`, `protocol`, `outcome`, `reason`, `message.kind`, `kind`, + `exception.type`, `socket.error`). +- [ ] **Opt-out** — with `EnableMetrics = false`, no `WatsonTcp` meter is created (in-memory + exporter sees nothing). +- [ ] **Tracing** *(if delivered in 6.4.0)* — `.AddSource("WatsonTcp").AddInMemoryExporter(...)`, + assert T04/T05 spans exist with `message.bytes` and, server-side, `client.guid`. +- [ ] All test projects compile and pass on `net8.0` and `net10.0` (exit code 0). + +--- + +## 11. Documentation deliverables + +- [ ] **README.md** — new `## New in v6.4.0` section (above the current + `## New in v6.3.2`) describing telemetry, the `Meter`/`ActivitySource` name + `WatsonTcp`, and the two-line Radiant/OTel subscription snippet from §2. +- [ ] **README.md** — extend the existing `## Version History` / observability area with a + short "Metrics & Tracing" subsection linking to this file and the metric-name table. +- [ ] **TELEMETRY.md** (this file) — keep the metric catalog (§4) authoritative; consumers + cite it for dashboard building. +- [ ] **ARCHITECTURE.md** — add a "Telemetry" subsection describing `WatsonTcpInstrumentation` + and its insertion points; verify no stale version/feature references. +- [ ] **CLAUDE.md** — update the NuGet-version line (currently reads "currently 6.0.11", + which is already stale) to `6.4.0`, and note the new telemetry files under + "Key Components". +- [ ] **FRAMING.md** — no change expected (framing bytes are unaffected); confirm. + +--- + +## 12. Version bump — v6.3.2 → v6.4.0 (minor) + +A minor bump is correct: purely additive public surface (`WatsonTcpMetrics`, four new +settings flags), no breaking changes, new dependency only on down-level TFMs. + +Run `grep -rn "6\.3\.2"` before tagging to catch anything this list misses. + +| Artifact | Change | Done | +|---|---|:----:| +| `src/WatsonTcp/WatsonTcp.csproj` → `` | `6.3.2` → `6.4.0` | ☐ | +| `src/WatsonTcp/WatsonTcp.csproj` → `` | Replace with telemetry summary (Meter/ActivitySource `WatsonTcp`, OTel/Prometheus/Radiant-ready) | ☐ | +| `src/WatsonTcp/WatsonTcp.csproj` → `` | Confirm year (`(c)2025`) still correct at release | ☐ | +| `CHANGELOG.md` | Move `v6.3.2` block to "Previous Version"; add `## Current Version` `v6.4.0` with a "Telemetry & Observability" section | ☐ | +| `README.md` | Add `## New in v6.4.0` (see §11); NuGet badges auto-update | ☐ | +| `ARCHITECTURE.md` | Telemetry subsection + version/feature sync | ☐ | +| `CLAUDE.md` | Correct the stale `6.0.11` version line → `6.4.0`; list new files | ☐ | +| `benchmarks/README.md` | Verify no hard-coded version drift | ☐ | +| Any `Test.*` referencing WatsonTcp by version | None expected (ProjectReference); confirm | ☐ | + +--- + +## 13. Delivery sequence (suggested order) + +1. ☐ Add `WatsonTcpMetrics.cs` (public constants) — unblocks everything and is the consumer contract. +2. ☐ Add `WatsonTcpInstrumentation.cs` (Meter/instruments/gauges/helpers, `IDisposable`). +3. ☐ csproj: conditional `System.Diagnostics.DiagnosticSource` reference; verify 6-TFM build. +4. ☐ Add the four `Enable*` settings flags. +5. ☐ Wire `WatsonTcpServer` (§6.2), including the `HandleException` funnel refactor. +6. ☐ Wire `WatsonTcpClient` (§6.3). +7. ☐ Add tracing spans (T01–T06) *(or defer per §5)*. +8. ☐ Add Touchstone tests (§10). +9. ☐ Documentation (§11) + version bump (§12). +10. ☐ Full multi-TFM build, warning-free; run all test runners (exit 0). + +--- + +## 14. Sign-off + +| Gate | Owner | Date | Done | +|---|---|---|:----:| +| Metric contract (§4) reviewed & frozen | | | ☐ | +| Code compliance (§9) verified | | | ☐ | +| Tests green on net8.0 + net10.0 (§10) | | | ☐ | +| Docs + version artifacts updated (§11–§12) | | | ☐ | +| Validated end-to-end against a Radiant host (`AddMeter("WatsonTcp")`) | | | ☐ | +``` diff --git a/src/Test.Automated/Program.cs b/src/Test.Automated/Program.cs index 4ec078d..c97e7d6 100644 --- a/src/Test.Automated/Program.cs +++ b/src/Test.Automated/Program.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using Test.Shared; +using Touchstone.Core; using Touchstone.Cli; namespace Test.Automated @@ -11,6 +13,7 @@ internal static class Program private static async Task Main(string[] args) { string resultsPath = null; + string suiteId = null; for (int i = 0; i < args.Length; i++) { @@ -25,9 +28,16 @@ private static async Task Main(string[] args) i++; } + else if (String.Equals(args[i], "--suite", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length) + { + suiteId = args[i + 1]; + i++; + } } - return await ConsoleRunner.RunAsync(WatsonTcpSuites.All, resultsPath: resultsPath).ConfigureAwait(false); + IReadOnlyList suites = WatsonTcpSuites.WithId(suiteId); + + return await ConsoleRunner.RunAsync(suites, resultsPath: resultsPath).ConfigureAwait(false); } } } diff --git a/src/Test.Shared/MetricMeasurement.cs b/src/Test.Shared/MetricMeasurement.cs new file mode 100644 index 0000000..4867ab7 --- /dev/null +++ b/src/Test.Shared/MetricMeasurement.cs @@ -0,0 +1,21 @@ +namespace Test.Shared +{ + using System.Collections.Generic; + + /// + /// A single recorded metric measurement: instrument name, numeric value, and tag map. + /// + internal sealed class MetricMeasurement + { + internal string Name { get; } + internal double Value { get; } + internal Dictionary Tags { get; } + + internal MetricMeasurement(string name, double value, Dictionary tags) + { + Name = name; + Value = value; + Tags = tags; + } + } +} diff --git a/src/Test.Shared/TelemetryCollector.cs b/src/Test.Shared/TelemetryCollector.cs new file mode 100644 index 0000000..c0dd662 --- /dev/null +++ b/src/Test.Shared/TelemetryCollector.cs @@ -0,0 +1,233 @@ +namespace Test.Shared +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.Metrics; + using System.Linq; + + /// + /// Collects WatsonTcp metrics and spans using only the base class library + /// ( and ), so the shared test project + /// takes no dependency on the OpenTelemetry SDK. Construct one before creating the WatsonTcp + /// client/server under test so that instrument publication is captured, and dispose it when done. + /// + internal sealed class TelemetryCollector : IDisposable + { + #region Private-Members + + private readonly object _Lock = new object(); + private readonly List _Measurements = new List(); + private readonly List _Activities = new List(); + private readonly MeterListener _MeterListener; + private readonly ActivityListener _ActivityListener; + + #endregion + + #region Constructors-and-Factories + + internal TelemetryCollector(string meterName = "WatsonTcp", string activitySourceName = "WatsonTcp") + { + if (String.IsNullOrEmpty(meterName)) throw new ArgumentNullException(nameof(meterName)); + if (String.IsNullOrEmpty(activitySourceName)) throw new ArgumentNullException(nameof(activitySourceName)); + + _MeterListener = new MeterListener(); + _MeterListener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == meterName) listener.EnableMeasurementEvents(instrument); + }; + _MeterListener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + Record(instrument.Name, measurement, tags)); + _MeterListener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + Record(instrument.Name, measurement, tags)); + _MeterListener.Start(); + + _ActivityListener = new ActivityListener + { + ShouldListenTo = source => source.Name == activitySourceName, + Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData, + SampleUsingParentId = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData, + ActivityStopped = activity => + { + lock (_Lock) + { + _Activities.Add(activity); + } + } + }; + ActivitySource.AddActivityListener(_ActivityListener); + } + + #endregion + + #region Internal-Methods + + /// + /// Force collection of all observable (gauge) instruments so their latest values are recorded. + /// + internal void CollectObservable() + { + _MeterListener.RecordObservableInstruments(); + } + + /// + /// Sum the values of all recorded measurements for the named instrument that match the supplied tags. + /// + internal double Sum(string name, params string[] tagKeyValuePairs) + { + Dictionary filter = BuildFilter(tagKeyValuePairs); + lock (_Lock) + { + return _Measurements + .Where(m => m.Name == name && Matches(m, filter)) + .Sum(m => m.Value); + } + } + + /// + /// Count the recorded measurements for the named instrument that match the supplied tags. + /// + internal int Count(string name, params string[] tagKeyValuePairs) + { + Dictionary filter = BuildFilter(tagKeyValuePairs); + lock (_Lock) + { + return _Measurements.Count(m => m.Name == name && Matches(m, filter)); + } + } + + /// + /// Return the latest recorded value for the named instrument matching the supplied tags, or zero. + /// + internal double Latest(string name, params string[] tagKeyValuePairs) + { + Dictionary filter = BuildFilter(tagKeyValuePairs); + lock (_Lock) + { + MetricMeasurement match = _Measurements.LastOrDefault(m => m.Name == name && Matches(m, filter)); + return match == null ? 0.0 : match.Value; + } + } + + /// + /// Total number of metric measurements recorded across all instruments. + /// + internal int TotalMeasurements() + { + lock (_Lock) + { + return _Measurements.Count; + } + } + + /// + /// Distinct tag keys observed across all recorded measurements. + /// + internal IReadOnlyCollection AllTagKeys() + { + lock (_Lock) + { + HashSet keys = new HashSet(StringComparer.Ordinal); + foreach (MetricMeasurement measurement in _Measurements) + { + foreach (string key in measurement.Tags.Keys) keys.Add(key); + } + + return keys; + } + } + + /// + /// Number of stopped spans recorded with the given name. + /// + internal int SpanCount(string spanName) + { + lock (_Lock) + { + return _Activities.Count(a => a.OperationName == spanName); + } + } + + /// + /// Total number of stopped spans recorded. + /// + internal int TotalSpans() + { + lock (_Lock) + { + return _Activities.Count; + } + } + + /// + /// Return true if any recorded span with the given name carries a tag with the supplied key. + /// + internal bool AnySpanHasTag(string spanName, string tagKey) + { + lock (_Lock) + { + // Activity.Tags only surfaces string-valued tags; TagObjects includes non-string values too. + return _Activities.Any(a => a.OperationName == spanName + && a.TagObjects.Any(t => t.Key == tagKey)); + } + } + + #endregion + + #region IDisposable + + public void Dispose() + { + _MeterListener.Dispose(); + _ActivityListener.Dispose(); + } + + #endregion + + #region Private-Methods + + private void Record(string name, double value, ReadOnlySpan> tags) + { + Dictionary tagMap = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair tag in tags) + { + tagMap[tag.Key] = tag.Value == null ? String.Empty : tag.Value.ToString(); + } + + lock (_Lock) + { + _Measurements.Add(new MetricMeasurement(name, value, tagMap)); + } + } + + private static Dictionary BuildFilter(string[] tagKeyValuePairs) + { + Dictionary filter = new Dictionary(StringComparer.Ordinal); + if (tagKeyValuePairs == null) return filter; + if (tagKeyValuePairs.Length % 2 != 0) + { + throw new ArgumentException("Tag key/value pairs must be supplied in pairs.", nameof(tagKeyValuePairs)); + } + + for (int i = 0; i < tagKeyValuePairs.Length; i += 2) + { + filter[tagKeyValuePairs[i]] = tagKeyValuePairs[i + 1]; + } + + return filter; + } + + private static bool Matches(MetricMeasurement measurement, Dictionary filter) + { + foreach (KeyValuePair expected in filter) + { + if (!measurement.Tags.TryGetValue(expected.Key, out string actual)) return false; + if (!String.Equals(actual, expected.Value, StringComparison.Ordinal)) return false; + } + + return true; + } + + #endregion + } +} diff --git a/src/Test.Shared/WatsonTcpScenarios.cs b/src/Test.Shared/WatsonTcpScenarios.cs index 872818b..f880262 100644 --- a/src/Test.Shared/WatsonTcpScenarios.cs +++ b/src/Test.Shared/WatsonTcpScenarios.cs @@ -3380,6 +3380,636 @@ await session.SendAsync(new HandshakeMessage } #endregion + + #region Telemetry-Tests + + public static async Task TelemetryMessageAndByteCountersRecorded() + { + int port = GetNextPort(); + byte[] payload = CreatePatternedPayload(100); + ManualResetEvent serverReceived = new ManualResetEvent(false); + ManualResetEvent clientReceived = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => serverReceived.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + client.Events.MessageReceived += (s, e) => clientReceived.Set(); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + Guid clientGuid = server.ListClients().First().Guid; + await client.SendAsync(payload).ConfigureAwait(false); + WaitForSignal(serverReceived); + await server.SendAsync(clientGuid, payload).ConfigureAwait(false); + WaitForSignal(clientReceived); + + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.MessagesSent, WatsonTcpMetrics.TagRole, "client") >= 1).ConfigureAwait(false); + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.MessagesReceived, WatsonTcpMetrics.TagRole, "server") >= 1).ConfigureAwait(false); + + TestAssert.True(collector.Sum(WatsonTcpMetrics.BytesSent, WatsonTcpMetrics.TagRole, "client") >= 100, "Client should record at least the payload bytes sent."); + TestAssert.True(collector.Sum(WatsonTcpMetrics.BytesReceived, WatsonTcpMetrics.TagRole, "server") >= 100, "Server should record at least the payload bytes received."); + TestAssert.True(collector.Count(WatsonTcpMetrics.MessageReceivedSize, WatsonTcpMetrics.TagRole, "server") >= 1, "Server received-size histogram should record."); + TestAssert.True(collector.Sum(WatsonTcpMetrics.MessagesReceived, WatsonTcpMetrics.TagRole, "client") >= 1, "Client should record a received message."); + TestAssert.True(collector.Sum(WatsonTcpMetrics.MessagesSent, WatsonTcpMetrics.TagProtocol, "tcp") >= 1, "Sent-message protocol tag should be tcp."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryConnectionGaugesAndOutcomesRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "accepted") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.TagRole, "client", WatsonTcpMetrics.TagOutcome, "connected") >= 1, "Client should record a connected outcome."); + + collector.CollectObservable(); + TestAssert.Equal(1.0, collector.Latest(WatsonTcpMetrics.ConnectionsActive, WatsonTcpMetrics.TagRole, "server"), "Server active-connection gauge should read 1."); + TestAssert.Equal(1.0, collector.Latest(WatsonTcpMetrics.ConnectionsActive, WatsonTcpMetrics.TagRole, "client"), "Client active-connection gauge should read 1."); + + client.Disconnect(); + await WaitForServerClientCountAsync(server, 0).ConfigureAwait(false); + + collector.CollectObservable(); + TestAssert.Equal(0.0, collector.Latest(WatsonTcpMetrics.ConnectionsActive, WatsonTcpMetrics.TagRole, "server"), "Server active-connection gauge should return to 0."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryDisconnectionReasonRemovedRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + Guid clientGuid = server.ListClients().First().Guid; + await server.DisconnectClientAsync(clientGuid).ConfigureAwait(false); + await WaitForServerClientCountAsync(server, 0).ConfigureAwait(false); + + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.DisconnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagReason, "Removed") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.DisconnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagReason, "Removed") >= 1, "Server should record a Removed disconnection."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetrySyncRoundTripCompletedRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => { }; + server.Callbacks.SyncRequestReceivedAsync = async (req) => { await Task.Delay(10).ConfigureAwait(false); return new SyncResponse(req, "pong"); }; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + SyncResponse response = await client.SendAndWaitAsync(5000, "ping").ConfigureAwait(false); + TestAssert.NotNull(response); + + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.SyncRequestsSent, WatsonTcpMetrics.TagRole, "client") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.SyncResponsesReceived, WatsonTcpMetrics.TagRole, "client") >= 1, "Client should record a matched sync response."); + TestAssert.True(collector.Count(WatsonTcpMetrics.SyncDuration, WatsonTcpMetrics.TagRole, "client", WatsonTcpMetrics.TagOutcome, "completed") >= 1, "Client should record a completed sync duration."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetrySyncTimeoutRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => { }; + server.Callbacks.SyncRequestReceivedAsync = async (req) => { await Task.Delay(3000).ConfigureAwait(false); return new SyncResponse(req, "too late"); }; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await TestAssert.ThrowsAsync(() => client.SendAndWaitAsync(1000, "ping")).ConfigureAwait(false); + + await WaitForConditionAsync(() => collector.Count(WatsonTcpMetrics.SyncDuration, WatsonTcpMetrics.TagRole, "client", WatsonTcpMetrics.TagOutcome, "timeout") >= 1).ConfigureAwait(false); + TestAssert.Equal(0.0, collector.Sum(WatsonTcpMetrics.SyncResponsesReceived, WatsonTcpMetrics.TagRole, "client"), "No sync response should be recorded on timeout."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryHandshakeSuccessRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Callbacks.HandshakeAsync = async (session, token) => + { + await session.ReceiveAsync(token).ConfigureAwait(false); + return HandshakeResult.Succeed(); + }; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Callbacks.HandshakeAsync = async (session, token) => + { + await session.SendAsync(new HandshakeMessage { Type = "test", Data = Encoding.UTF8.GetBytes("hello") }, token).ConfigureAwait(false); + return HandshakeResult.Succeed(); + }; + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1, timeoutMs: 5000).ConfigureAwait(false); + + try + { + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.HandshakesTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "success") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.HandshakesTotal, WatsonTcpMetrics.TagRole, "client", WatsonTcpMetrics.TagOutcome, "success") >= 1, "Client should record a successful handshake."); + TestAssert.True(collector.Count(WatsonTcpMetrics.HandshakeDuration, WatsonTcpMetrics.TagRole, "server") >= 1, "Server should record a handshake duration."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryHandshakeFailureRecorded() + { + int port = GetNextPort(); + ManualResetEvent serverHandshakeFailed = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Callbacks.HandshakeAsync = async (session, token) => + { + await session.ReceiveAsync(token).ConfigureAwait(false); + return HandshakeResult.Fail("Rejected by test."); + }; + server.Events.HandshakeFailed += (s, e) => serverHandshakeFailed.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Callbacks.HandshakeAsync = async (session, token) => + { + await session.SendAsync(new HandshakeMessage { Type = "test", Data = Encoding.UTF8.GetBytes("hello") }, token).ConfigureAwait(false); + return HandshakeResult.Succeed(); + }; + + try + { + try + { + client.Connect(); + } + catch + { + } + + WaitForSignal(serverHandshakeFailed); + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.HandshakesTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "failure") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.HandshakesTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "failure") >= 1, "Server should record a failed handshake."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryAuthenticationSuccessRecorded() + { + int port = GetNextPort(); + string presharedKey = "0000000000000000"; + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => { }; + server.Settings.PresharedKey = presharedKey; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + client.Events.MessageReceived += (s, e) => { }; + client.Settings.PresharedKey = presharedKey; + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.AuthenticationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "success") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.AuthenticationsTotal, WatsonTcpMetrics.TagRole, "client", WatsonTcpMetrics.TagOutcome, "success") >= 1, "Client should record a successful authentication."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryAuthenticationFailureRecorded() + { + int port = GetNextPort(); + ManualResetEvent authFailed = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => { }; + server.Settings.PresharedKey = "correctkey123456"; + server.Events.AuthenticationFailed += (s, e) => authFailed.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + client.Events.MessageReceived += (s, e) => { }; + client.Settings.PresharedKey = "wrongkey12345678"; + + try + { + try + { + client.Connect(); + } + catch (ConnectionRejectedException) + { + } + + WaitForSignal(authFailed); + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.AuthenticationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "failure") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.AuthenticationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "failure") >= 1, "Server should record a failed authentication."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryAuthorizationAllowRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Callbacks.AuthorizeConnectionAsync = (ctx, token) => Task.FromResult(ConnectionAuthorizationResult.Allow()); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.AuthorizationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "success") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.AuthorizationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "success") >= 1, "Server should record an allowed authorization."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryAuthorizationRejectRecorded() + { + int port = GetNextPort(); + ManualResetEvent rejected = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Callbacks.AuthorizeConnectionAsync = (ctx, token) => Task.FromResult(ConnectionAuthorizationResult.Reject("no")); + server.Events.ConnectionRejected += (s, e) => rejected.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + + try + { + try + { + client.Connect(); + } + catch + { + } + + WaitForSignal(rejected); + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.AuthorizationsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "failure") >= 1).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "rejected_authorization") >= 1, "Server should record a rejected_authorization outcome."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryBlockedIpRejectionRecorded() + { + int port = GetNextPort(); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + SetupDefaultServerHandlers(server); + server.Settings.BlockedIPs = new List { "127.0.0.1" }; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + + try + { + try + { + client.Connect(); + } + catch + { + } + + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "rejected_blocked") >= 1, timeoutMs: 5000).ConfigureAwait(false); + TestAssert.True(collector.Sum(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagOutcome, "rejected_blocked") >= 1, "Server should record a rejected_blocked outcome."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryTransientAcceptErrorRecorded() + { + int port = GetNextPort(); + SocketException injected = new SocketException((int)SocketError.ConnectionReset); + + TelemetryCollector collector = new TelemetryCollector(); + TransientAcceptFailureServer server = new TransientAcceptFailureServer(_hostname, port, injected, failureCount: 1); + SetupDefaultServerHandlers(server); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + try + { + await WaitForConditionAsync(() => collector.Sum(WatsonTcpMetrics.ListenerTransientErrors, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagSocketError, "ConnectionReset") >= 1, timeoutMs: 5000).ConfigureAwait(false); + TestAssert.True(server.InjectedFailures >= 1, "Server should have injected a transient accept failure."); + TestAssert.True(collector.Sum(WatsonTcpMetrics.ListenerTransientErrors, WatsonTcpMetrics.TagRole, "server", WatsonTcpMetrics.TagSocketError, "ConnectionReset") >= 1, "Server should record a transient accept error."); + } + finally + { + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryMetricsDisabledProducesNoMeasurements() + { + int port = GetNextPort(); + byte[] payload = CreatePatternedPayload(64); + ManualResetEvent serverReceived = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Settings.EnableMetrics = false; + server.Events.MessageReceived += (s, e) => serverReceived.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + client.Settings.EnableMetrics = false; + client.Events.MessageReceived += (s, e) => { }; + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + // Allow any straggling measurements from prior tests to settle, then measure a delta so this + // assertion isolates this instance's behaviour from unrelated meters in the process. + await Task.Delay(200).ConfigureAwait(false); + int baseline = collector.TotalMeasurements(); + + await client.SendAsync(payload).ConfigureAwait(false); + WaitForSignal(serverReceived); + await Task.Delay(100).ConfigureAwait(false); + + TestAssert.Equal(baseline, collector.TotalMeasurements(), "No new metric measurements should be recorded when metrics are disabled."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryTracingDisabledProducesNoSpans() + { + int port = GetNextPort(); + byte[] payload = CreatePatternedPayload(64); + ManualResetEvent serverReceived = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Settings.EnableTracing = false; + server.Events.MessageReceived += (s, e) => serverReceived.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + client.Settings.EnableTracing = false; + client.Events.MessageReceived += (s, e) => { }; + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await client.SendAsync(payload).ConfigureAwait(false); + WaitForSignal(serverReceived); + await Task.Delay(100).ConfigureAwait(false); + + TestAssert.Equal(0, collector.TotalSpans(), "No spans should be recorded when tracing is disabled."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetrySendAndReceiveSpansRecorded() + { + int port = GetNextPort(); + byte[] payload = CreatePatternedPayload(64); + ManualResetEvent serverReceived = new ManualResetEvent(false); + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => serverReceived.Set(); + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await client.SendAsync(payload).ConfigureAwait(false); + WaitForSignal(serverReceived); + + await WaitForConditionAsync(() => collector.SpanCount(WatsonTcpMetrics.SpanSend) >= 1).ConfigureAwait(false); + await WaitForConditionAsync(() => collector.SpanCount(WatsonTcpMetrics.SpanReceive) >= 1).ConfigureAwait(false); + TestAssert.True(collector.AnySpanHasTag(WatsonTcpMetrics.SpanSend, WatsonTcpMetrics.TagMessageBytes), "Send span should carry a message.bytes tag."); + + client.Disconnect(); + await WaitForServerClientCountAsync(server, 0).ConfigureAwait(false); + await WaitForConditionAsync(() => collector.SpanCount(WatsonTcpMetrics.SpanSession) >= 1).ConfigureAwait(false); + TestAssert.True(collector.AnySpanHasTag(WatsonTcpMetrics.SpanSession, WatsonTcpMetrics.TagClientGuid), "Session span should carry a client.guid tag."); + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + public static async Task TelemetryMetricsHaveNoHighCardinalityTags() + { + int port = GetNextPort(); + byte[] payload = CreatePatternedPayload(48); + ManualResetEvent serverReceived = new ManualResetEvent(false); + + HashSet allowedKeys = new HashSet(StringComparer.Ordinal) + { + WatsonTcpMetrics.TagRole, + WatsonTcpMetrics.TagProtocol, + WatsonTcpMetrics.TagOutcome, + WatsonTcpMetrics.TagReason, + WatsonTcpMetrics.TagMessageKind, + WatsonTcpMetrics.TagKind, + WatsonTcpMetrics.TagExceptionType, + WatsonTcpMetrics.TagSocketError + }; + + TelemetryCollector collector = new TelemetryCollector(); + WatsonTcpServer server = new WatsonTcpServer(_hostname, port); + server.Events.MessageReceived += (s, e) => serverReceived.Set(); + server.Callbacks.SyncRequestReceivedAsync = async (req) => { await Task.Delay(5).ConfigureAwait(false); return new SyncResponse(req, "pong"); }; + server.Start(); + await WaitForServerListeningAsync(server).ConfigureAwait(false); + + WatsonTcpClient client = new WatsonTcpClient(_hostname, port); + SetupDefaultClientHandlers(client); + client.Connect(); + await WaitForClientConnectedAsync(client, server, 1).ConfigureAwait(false); + + try + { + await client.SendAsync(payload).ConfigureAwait(false); + WaitForSignal(serverReceived); + await client.SendAndWaitAsync(5000, "ping").ConfigureAwait(false); + + client.Disconnect(); + await WaitForServerClientCountAsync(server, 0).ConfigureAwait(false); + collector.CollectObservable(); + + foreach (string key in collector.AllTagKeys()) + { + TestAssert.True(allowedKeys.Contains(key), "Unexpected high-cardinality metric tag key: " + key); + } + } + finally + { + SafeDispose(client); + SafeDispose(server); + SafeDispose(collector); + } + } + + #endregion } } - + diff --git a/src/Test.Shared/WatsonTcpSuites.cs b/src/Test.Shared/WatsonTcpSuites.cs index 9e44e95..b91623f 100644 --- a/src/Test.Shared/WatsonTcpSuites.cs +++ b/src/Test.Shared/WatsonTcpSuites.cs @@ -53,13 +53,53 @@ public static class WatsonTcpSuites nameof(WatsonTcpScenarios.StreamSendReceive) }; + private static readonly string[] _TelemetryScenarioNames = + { + nameof(WatsonTcpScenarios.TelemetryMessageAndByteCountersRecorded), + nameof(WatsonTcpScenarios.TelemetryConnectionGaugesAndOutcomesRecorded), + nameof(WatsonTcpScenarios.TelemetryDisconnectionReasonRemovedRecorded), + nameof(WatsonTcpScenarios.TelemetrySyncRoundTripCompletedRecorded), + nameof(WatsonTcpScenarios.TelemetrySyncTimeoutRecorded), + nameof(WatsonTcpScenarios.TelemetryHandshakeSuccessRecorded), + nameof(WatsonTcpScenarios.TelemetryHandshakeFailureRecorded), + nameof(WatsonTcpScenarios.TelemetryAuthenticationSuccessRecorded), + nameof(WatsonTcpScenarios.TelemetryAuthenticationFailureRecorded), + nameof(WatsonTcpScenarios.TelemetryAuthorizationAllowRecorded), + nameof(WatsonTcpScenarios.TelemetryAuthorizationRejectRecorded), + nameof(WatsonTcpScenarios.TelemetryBlockedIpRejectionRecorded), + nameof(WatsonTcpScenarios.TelemetryTransientAcceptErrorRecorded), + nameof(WatsonTcpScenarios.TelemetryMetricsDisabledProducesNoMeasurements), + nameof(WatsonTcpScenarios.TelemetryTracingDisabledProducesNoSpans), + nameof(WatsonTcpScenarios.TelemetrySendAndReceiveSpansRecorded), + nameof(WatsonTcpScenarios.TelemetryMetricsHaveNoHighCardinalityTags) + }; + private static readonly IReadOnlyList _All = BuildSuites(); + private static readonly string[] _SuiteIds = { "regression", "auth-handshake", "streaming", "telemetry" }; + public static IReadOnlyList All { get { return _All; } } + /// + /// Return only the suite with the supplied identifier (regression, auth-handshake, streaming, telemetry), + /// or all suites when the identifier is null or empty. + /// + public static IReadOnlyList WithId(string id) + { + if (String.IsNullOrEmpty(id)) return _All; + + List result = new List(); + for (int i = 0; i < _All.Count && i < _SuiteIds.Length; i++) + { + if (String.Equals(_SuiteIds[i], id, StringComparison.OrdinalIgnoreCase)) result.Add(_All[i]); + } + + return result; + } + private static IReadOnlyList BuildSuites() { List suites = new List @@ -75,13 +115,17 @@ private static IReadOnlyList BuildSuites() new TestSuiteDescriptor( "streaming", "Streaming", - BuildCases(includeOnlyStreamingScenarios: true)) + BuildCases(includeOnlyStreamingScenarios: true)), + new TestSuiteDescriptor( + "telemetry", + "Telemetry", + BuildCases(includeOnlyTelemetryScenarios: true)) }; return suites; } - private static IReadOnlyList BuildCases(bool excludeAuthorizationScenarios = false, bool includeOnlyAuthorizationScenarios = false, bool includeOnlyStreamingScenarios = false) + private static IReadOnlyList BuildCases(bool excludeAuthorizationScenarios = false, bool includeOnlyAuthorizationScenarios = false, bool includeOnlyStreamingScenarios = false, bool includeOnlyTelemetryScenarios = false) { IEnumerable methods = typeof(WatsonTcpScenarios) .GetMethods(BindingFlags.Public | BindingFlags.Static) @@ -90,7 +134,8 @@ private static IReadOnlyList BuildCases(bool excludeAuthoriz if (excludeAuthorizationScenarios) { - methods = methods.Where(m => !_AuthorizationScenarioNames.Contains(m.Name, StringComparer.Ordinal)); + methods = methods.Where(m => !_AuthorizationScenarioNames.Contains(m.Name, StringComparer.Ordinal) + && !_TelemetryScenarioNames.Contains(m.Name, StringComparer.Ordinal)); } if (includeOnlyAuthorizationScenarios) @@ -103,12 +148,17 @@ private static IReadOnlyList BuildCases(bool excludeAuthoriz methods = methods.Where(m => _StreamingScenarioNames.Contains(m.Name, StringComparer.Ordinal)); } + if (includeOnlyTelemetryScenarios) + { + methods = methods.Where(m => _TelemetryScenarioNames.Contains(m.Name, StringComparer.Ordinal)); + } + return methods .OrderBy(m => m.Name, StringComparer.Ordinal) .Select(m => new TestCaseDescriptor( includeOnlyAuthorizationScenarios ? "auth-handshake" - : (includeOnlyStreamingScenarios ? "streaming" : "regression"), + : (includeOnlyStreamingScenarios ? "streaming" : (includeOnlyTelemetryScenarios ? "telemetry" : "regression")), m.Name, ToDisplayName(m.Name), token => diff --git a/src/WatsonTcp/ClientMetadata.cs b/src/WatsonTcp/ClientMetadata.cs index e67db9e..4fdab50 100644 --- a/src/WatsonTcp/ClientMetadata.cs +++ b/src/WatsonTcp/ClientMetadata.cs @@ -1,6 +1,7 @@ namespace WatsonTcp { using System; + using System.Diagnostics; using System.IO; using System.Net.Security; using System.Net.Sockets; @@ -97,6 +98,7 @@ internal Stream DataStream internal BufferedReadStream ReceiveStream { get; private set; } = null; internal byte[] SendBuffer { get; set; } = new byte[65536]; internal Task DataReceiver { get; set; } = null; + internal Activity SessionActivity { get; set; } = null; internal long LastSeenUtcTicks { get; set; } = 0; internal SemaphoreSlim WriteLock = new SemaphoreSlim(1, 1); diff --git a/src/WatsonTcp/WatsonTcp.csproj b/src/WatsonTcp/WatsonTcp.csproj index 2f88ad9..feec8ba 100644 --- a/src/WatsonTcp/WatsonTcp.csproj +++ b/src/WatsonTcp/WatsonTcp.csproj @@ -7,7 +7,7 @@ true latest-recommended true - 6.3.2 + 6.4.0 Joel Christner Joel Christner A simple C# async TCP server and client with integrated framing for reliable transmission and receipt of data @@ -17,7 +17,7 @@ https://github.com/dotnet/WatsonTcp Github - Fixes a listener reliability issue where transient accept-time connection reset or abort socket errors could stop the server accept loop. + Adds vendor-neutral telemetry via System.Diagnostics.Metrics (Meter) and System.Diagnostics.ActivitySource, both named 'WatsonTcp', for consumption by Radiant, the OpenTelemetry SDK, Prometheus, and others. See TELEMETRY.md. LICENSE.md watson.png @@ -47,6 +47,15 @@ + + + + + Always diff --git a/src/WatsonTcp/WatsonTcp.xml b/src/WatsonTcp/WatsonTcp.xml index a8e07ba..32724fe 100644 --- a/src/WatsonTcp/WatsonTcp.xml +++ b/src/WatsonTcp/WatsonTcp.xml @@ -1338,6 +1338,23 @@ Value must be greater than zero. + + + Enable or disable emission of metrics into the WatsonTcp + . Default is true. + When true, the meter and its instruments are created; recording is a near-free no-op unless a + telemetry host subscribes to the meter by name. When false, no meter is created and there is + zero overhead. See and TELEMETRY.md. + + + + + Enable or disable emission of distributed-tracing spans into the WatsonTcp + . Default is true. + When true, spans are created only if a tracing host subscribes to the activity source by name; + otherwise span creation is a near-free no-op. See and TELEMETRY.md. + + Instantiate. @@ -1379,6 +1396,39 @@ + + + Owns the and a single WatsonTcp client or + server records telemetry into, along with every instrument and span factory. One instance + lives per client/server and is disposed with its owner. All recording is fire-and-forget and + never throws into the caller's send, receive, or connection path. When metrics are disabled the + meter is never created (so no instruments are published); when tracing is disabled span factories + return null. + + + + + Instantiate. + + Emitting side; one of or . + Transport; one of or . + When false, no meter or instruments are created. + When false, span factories return null. + Callback returning the current live connection count, or null to omit that gauge. + Callback returning the current pending connection count, or null to omit that gauge. + Callback returning the current in-flight synchronous request count, or null to omit that gauge. + Callback returning seconds since start, or null to omit that gauge. + + + + Returns fractional seconds elapsed since a reading. + + + + + Dispose of the underlying meter and activity source. + + WatsonTcp keepalive settings. WatsonTcp does not implement keepalives, rather, it relies on the underlying implementation in the operating system and runtime. @@ -1422,6 +1472,337 @@ Instantiate. + + + Well-known names, units, and tag keys that WatsonTcp emits telemetry under. + + WatsonTcp publishes metrics to a and + distributed-tracing spans to a , both named + ("WatsonTcp"). These names are the public contract between + WatsonTcp and any telemetry host (Radiant, the OpenTelemetry SDK, Prometheus, and others). + They are stable across releases; treat them like public API. + + + A host observes WatsonTcp by subscribing to the meter and activity source by name, for + example MeterProviderBuilder.AddMeter(WatsonTcpMetrics.MeterName) or, for a Radiant + host, settings.Sources.AddMeter(WatsonTcpMetrics.MeterName). Metric names are dotted + and lowercase and units are UCUM strings, so the OpenTelemetry Prometheus exporter produces + conventional series names automatically. + + + + + + Name of the WatsonTcp records all metrics + into, and of the WatsonTcp starts all + spans from. Value is "WatsonTcp". Stable across releases. + + + + + Name of the WatsonTcp starts spans from. + Identical to ("WatsonTcp") so a single subscription string + covers both metrics and traces. + + + + + UCUM unit for a count of messages ("{message}"). Annotation units contribute no + Prometheus suffix. + + + + + UCUM unit for a count of connections ("{connection}"). + + + + + UCUM unit for a count of handshakes ("{handshake}"). + + + + + UCUM unit for a count of authentications ("{authentication}"). + + + + + UCUM unit for a count of authorizations ("{authorization}"). + + + + + UCUM unit for a count of synchronous requests ("{request}"). + + + + + UCUM unit for a count of synchronous responses ("{response}"). + + + + + UCUM unit for a count of exceptions ("{exception}"). + + + + + UCUM unit for a count of errors ("{error}"). + + + + + UCUM unit for bytes ("By"). The Prometheus exporter appends a _bytes suffix. + + + + + UCUM unit for seconds ("s"). The Prometheus exporter appends a _seconds suffix. + + + + + Counter of messages written to the wire. Unit . + + + + + Counter of messages read from the wire. Unit . + + + + + Counter of payload bytes sent. Unit . + + + + + Counter of payload bytes received. Unit . + + + + + Histogram of sent message sizes. Unit . + + + + + Histogram of received message sizes. Unit . + + + + + Histogram of the time taken to write a message header and payload to the transport stream. + Unit . + + + + + Observable gauge of currently live connections. Unit . + + + + + Observable gauge of accepted-but-not-yet-admitted connections (server only). + Unit . + + + + + Counter of connection admission outcomes, dimensioned by . + Unit . + + + + + Counter of disconnections, dimensioned by . + Unit . + + + + + Counter of custom-handshake completions, dimensioned by . + Unit . + + + + + Histogram of custom-handshake duration, dimensioned by . + Unit . + + + + + Counter of preshared-key authentication results, dimensioned by . + Unit . + + + + + Counter of connection-authorization results, dimensioned by . + Unit . + + + + + Counter of synchronous requests issued through SendAndWaitAsync. Unit . + + + + + Counter of synchronous responses matched to an outstanding request. Unit . + + + + + Histogram of synchronous round-trip duration, dimensioned by + (completed or timeout). Unit . + + + + + Observable gauge of in-flight synchronous conversations. Unit . + + + + + Counter of expired synchronous requests or responses that were discarded, dimensioned by + (request or response). Unit . + + + + + Counter of exceptions surfaced through the ExceptionEncountered event, dimensioned by + . Unit . + + + + + Counter of recovered transient accept-loop socket errors, dimensioned by + . Unit . + + + + + Counter of unread stream-payload bytes drained after a receive handler returned. + Unit . + + + + + Observable gauge of seconds elapsed since the client or server started. Unit . + + + + + Client span covering a connection attempt through registration and handshake. + + + + + Server span covering the lifetime of an admitted client session. + + + + + Span covering a custom handshake exchange. + + + + + Span covering a single message send. + + + + + Span covering the processing of a single received data message. + + + + + Span covering a synchronous request/response round trip. + + + + + Metric tag key naming which side emitted the measurement. Values: server, client. + + + + + Metric tag key naming the transport. Values: tcp, ssl. + + + + + Metric tag key naming the outcome of a connection, handshake, authentication, authorization, + or synchronous operation. + + + + + Metric tag key naming the reason for a disconnection. Values mirror the + enum member names. + + + + + Metric tag key naming a message classification. Values: data, control, + sync_request, sync_response. + + + + + Metric tag key naming an expired-synchronous discard classification. Values: request, + response. + + + + + Metric tag key naming the short type name of an exception. + + + + + Metric tag key naming a value. + + + + + Span tag key naming the remote server address (client spans). + + + + + Span tag key naming the remote server port (client spans). + + + + + Span tag key naming the remote client address (server spans). + + + + + Span tag key naming the WatsonTcp client GUID (server spans). High cardinality; spans only. + + + + + Span tag key naming a synchronous conversation GUID. High cardinality; spans only. + + + + + Span tag key naming a message payload size in bytes. + + + + + Span tag key indicating whether a message is a synchronous request. + + Watson TCP server, with or without SSL. @@ -1863,6 +2244,23 @@ Value must be greater than zero. + + + Enable or disable emission of metrics into the WatsonTcp + . Default is true. + When true, the meter and its instruments are created; recording is a near-free no-op unless a + telemetry host subscribes to the meter by name. When false, no meter is created and there is + zero overhead. See and TELEMETRY.md. + + + + + Enable or disable emission of distributed-tracing spans into the WatsonTcp + . Default is true. + When true, spans are created only if a tracing host subscribes to the activity source by name; + otherwise span creation is a near-free no-op. See and TELEMETRY.md. + + Instantiate. diff --git a/src/WatsonTcp/WatsonTcpClient.cs b/src/WatsonTcp/WatsonTcpClient.cs index 7490ef7..49e5e2e 100644 --- a/src/WatsonTcp/WatsonTcpClient.cs +++ b/src/WatsonTcp/WatsonTcpClient.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; + using System.Diagnostics; using System.IO; using System.Net; using System.Net.Security; @@ -150,6 +151,8 @@ public ISerializationHelper SerializationHelper private WatsonTcpKeepaliveSettings _Keepalive = new WatsonTcpKeepaliveSettings(); private WatsonTcpClientSslConfiguration _SslConfiguration = new WatsonTcpClientSslConfiguration(); private ISerializationHelper _SerializationHelper = new DefaultSerializationHelper(); + private WatsonTcpInstrumentation _Instrumentation = null; + private Activity _ConnectActivity = null; private Mode _Mode = Mode.Tcp; private TlsVersion _TlsVersion = TlsVersion.Tls12; @@ -158,11 +161,11 @@ public ISerializationHelper SerializationHelper private string _ServerIp = null; private int _ServerPort = 0; - private TcpClient _Client = null; - private Stream _DataStream = null; - private BufferedReadStream _ReceiveStream = null; - private NetworkStream _TcpStream = null; - private SslStream _SslStream = null; + private TcpClient _Client = null; + private Stream _DataStream = null; + private BufferedReadStream _ReceiveStream = null; + private NetworkStream _TcpStream = null; + private SslStream _SslStream = null; private X509Certificate2 _SslCertificate = null; private X509Certificate2Collection _SslCertificateCollection = null; @@ -175,19 +178,19 @@ public ISerializationHelper SerializationHelper private Task _DataReceiver = null; private Task _IdleServerMonitor = null; - private DateTime _LastActivity = DateTime.UtcNow; - private bool _IsTimeout = false; - private bool _TransportConnected = false; - private bool _ServerConnectedRaised = false; - private bool _HandshakeRequired = false; - private TaskCompletionSource _InitializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - private TaskCompletionSource _InitializationReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - private TaskCompletionSource _InitializationFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - private HandshakeSessionTransport _ClientHandshakeTransport = null; - private ClientHandshakeSession _ClientHandshakeSession = null; - private Task _ClientHandshakeTask = null; - - private readonly ConcurrentDictionary> _SyncRequests = new ConcurrentDictionary>(); + private DateTime _LastActivity = DateTime.UtcNow; + private bool _IsTimeout = false; + private bool _TransportConnected = false; + private bool _ServerConnectedRaised = false; + private bool _HandshakeRequired = false; + private TaskCompletionSource _InitializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource _InitializationReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private TaskCompletionSource _InitializationFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private HandshakeSessionTransport _ClientHandshakeTransport = null; + private ClientHandshakeSession _ClientHandshakeSession = null; + private Task _ClientHandshakeTask = null; + + private readonly ConcurrentDictionary> _SyncRequests = new ConcurrentDictionary>(); #endregion @@ -317,50 +320,50 @@ public void Dispose() GC.SuppressFinalize(this); } - /// - /// Connect to the server. - /// - public void Connect() - { - ConnectCoreAsync().GetAwaiter().GetResult(); - } + /// + /// Connect to the server. + /// + public void Connect() + { + ConnectCoreAsync().GetAwaiter().GetResult(); + } /// /// Disconnect from the server. /// /// Flag to indicate whether the server should be notified of the disconnect. This message will not be sent until other send requests have been handled. - public void Disconnect(bool sendNotice = true) - { - if (!Connected && !_TransportConnected) throw new InvalidOperationException("Not connected to the server."); - - _Settings.Logger?.Invoke(Severity.Info, _Header + "disconnecting from " + _ServerIp + ":" + _ServerPort); - - if (_TransportConnected && sendNotice) - { - WatsonMessage msg = new WatsonMessage(); - msg.Status = MessageStatus.Shutdown; - SendInternalAsync(msg, 0, null, default(CancellationToken), true).GetAwaiter().GetResult(); - } - - CloseTransport(true); - _Settings.Logger?.Invoke(Severity.Info, _Header + "disconnected from " + _ServerIp + ":" + _ServerPort); - } + public void Disconnect(bool sendNotice = true) + { + if (!Connected && !_TransportConnected) throw new InvalidOperationException("Not connected to the server."); + + _Settings.Logger?.Invoke(Severity.Info, _Header + "disconnecting from " + _ServerIp + ":" + _ServerPort); + + if (_TransportConnected && sendNotice) + { + WatsonMessage msg = new WatsonMessage(); + msg.Status = MessageStatus.Shutdown; + SendInternalAsync(msg, 0, null, default(CancellationToken), true).GetAwaiter().GetResult(); + } + + CloseTransport(true); + _Settings.Logger?.Invoke(Severity.Info, _Header + "disconnected from " + _ServerIp + ":" + _ServerPort); + } /// /// Send a pre-shared key to the server to authenticate. /// /// Up to 16-character string. /// Cancellation token to cancel the request. - public async Task AuthenticateAsync(string presharedKey, CancellationToken token = default) - { - if (String.IsNullOrEmpty(presharedKey)) throw new ArgumentNullException(nameof(presharedKey)); - if (presharedKey.Length != 16) throw new ArgumentException("Preshared key length must be 16 bytes."); - - WatsonMessage msg = new WatsonMessage(); - msg.Status = MessageStatus.AuthRequested; - msg.PresharedKey = Encoding.UTF8.GetBytes(presharedKey); - await SendInternalAsync(msg, 0, null, token, true).ConfigureAwait(false); - } + public async Task AuthenticateAsync(string presharedKey, CancellationToken token = default) + { + if (String.IsNullOrEmpty(presharedKey)) throw new ArgumentNullException(nameof(presharedKey)); + if (presharedKey.Length != 16) throw new ArgumentException("Preshared key length must be 16 bytes."); + + WatsonMessage msg = new WatsonMessage(); + msg.Status = MessageStatus.AuthRequested; + msg.PresharedKey = Encoding.UTF8.GetBytes(presharedKey); + await SendInternalAsync(msg, 0, null, token, true).ConfigureAwait(false); + } #region SendAsync @@ -478,13 +481,25 @@ public async Task SendAndWaitAsync(int timeoutMs, long contentLeng /// Do not reuse the object after disposal. /// /// Indicate if resources should be disposed. - protected virtual void Dispose(bool disposing) - { - if (disposing) - { - _Settings.Logger?.Invoke(Severity.Info, _Header + "disposing"); - - if (Connected || _TransportConnected) CloseTransport(true); + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + _Settings.Logger?.Invoke(Severity.Info, _Header + "disposing"); + + if (Connected || _TransportConnected) CloseTransport(true); + + if (_ConnectActivity != null) + { + _ConnectActivity.Dispose(); + _ConnectActivity = null; + } + + if (_Instrumentation != null) + { + _Instrumentation.Dispose(); + _Instrumentation = null; + } if (_SslCertificate != null) _SslCertificate.Dispose(); @@ -505,415 +520,453 @@ protected virtual void Dispose(bool disposing) _SourceIp = null; _ServerIp = null; - _Client = null; - _DataStream = null; - _ReceiveStream = null; - _TcpStream = null; - _SslStream = null; + _Client = null; + _DataStream = null; + _ReceiveStream = null; + _TcpStream = null; + _SslStream = null; _SslCertificate = null; _SslCertificateCollection = null; _WriteLock = null; _ReadLock = null; - _DataReceiver = null; - } - } - - private void ResetInitializationState() - { - _ServerConnectedRaised = false; - _HandshakeRequired = false; - _InitializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _InitializationReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - _InitializationFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - _ClientHandshakeTransport?.Dispose(); - _ClientHandshakeTransport = null; - _ClientHandshakeSession = null; - _ClientHandshakeTask = null; - } - - private async Task ConnectCoreAsync() - { - if (Connected || _TransportConnected) throw new InvalidOperationException("Already connected to the server."); - - if (_Settings.LocalPort == 0) - { - _Client = new TcpClient(); - } - else - { - IPEndPoint ipe = new IPEndPoint(IPAddress.Any, _Settings.LocalPort); - _Client = new TcpClient(ipe); - } - - _Client.NoDelay = _Settings.NoDelay; - _Statistics = new WatsonTcpStatistics(); - - ValidateReceiveHandlerConfiguration(); - - try - { - if (_Mode == Mode.Tcp) - { - await ConnectTcpTransportAsync().ConfigureAwait(false); - } - else if (_Mode == Mode.Ssl) - { - await ConnectSslTransportAsync().ConfigureAwait(false); - } - else - { - throw new ArgumentException("Unknown mode: " + _Mode.ToString()); - } - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "exception encountered: " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - throw; - } - - _TransportConnected = true; - ResetInitializationState(); - - _TokenSource = new CancellationTokenSource(); - _Token = _TokenSource.Token; - _MessageBuilder.MaxHeaderSize = _Settings.MaxHeaderSize; - _MessageBuilder.ReadStreamBuffer = _Settings.StreamBufferSize; - - _LastActivity = DateTime.UtcNow; - _IsTimeout = false; - - _DataReceiver = DataReceiver(_Token); - _IdleServerMonitor = IdleServerMonitor(_Token); - - WatsonMessage msg = new WatsonMessage(); - msg.Status = MessageStatus.RegisterClient; - - if (!await SendInternalAsync(msg, 0, null, default(CancellationToken), true).ConfigureAwait(false)) - { - Exception initException = null; - if (_InitializationFailure.Task.IsCompleted - && !_InitializationFailure.Task.IsCanceled - && !_InitializationFailure.Task.IsFaulted) - { - initException = await _InitializationFailure.Task.ConfigureAwait(false); - } - - CloseTransport(false); - if (initException != null) throw initException; - - _Settings.Logger?.Invoke(Severity.Alert, _Header + "unable to register GUID " + _Settings.Guid + " with the server"); - throw new ArgumentException("Server rejected GUID " + _Settings.Guid); - } - - await CompleteConnectionInitializationAsync().ConfigureAwait(false); - } - - private async Task ConnectTcpTransportAsync() - { - _Settings.Logger?.Invoke(Severity.Info, _Header + "connecting to " + _ServerIp + ":" + _ServerPort); - - _Client.LingerState = new LingerOption(true, 0); - await ConnectSocketAsync().ConfigureAwait(false); - - _SourceIp = ((IPEndPoint)_Client.Client.LocalEndPoint).Address.ToString(); - _SourcePort = ((IPEndPoint)_Client.Client.LocalEndPoint).Port; - _TcpStream = _Client.GetStream(); - _DataStream = _TcpStream; - _ReceiveStream = new BufferedReadStream(_DataStream, _Settings.StreamBufferSize); - _SslStream = null; - - if (_Keepalive.EnableTcpKeepAlives) EnableKeepalives(); - } - - private async Task ConnectSslTransportAsync() - { - _Settings.Logger?.Invoke(Severity.Info, _Header + "connecting with SSL to " + _ServerIp + ":" + _ServerPort); - - _Client.LingerState = new LingerOption(true, 0); - await ConnectSocketAsync().ConfigureAwait(false); - - _SourceIp = ((IPEndPoint)_Client.Client.LocalEndPoint).Address.ToString(); - _SourcePort = ((IPEndPoint)_Client.Client.LocalEndPoint).Port; - - if (_Settings.AcceptInvalidCertificates) - _SslStream = new SslStream(_Client.GetStream(), false, _SslConfiguration.ServerCertificateValidationCallback, _SslConfiguration.ClientCertificateSelectionCallback); - else - _SslStream = new SslStream(_Client.GetStream(), false); - - await _SslStream.AuthenticateAsClientAsync(_ServerIp, _SslCertificateCollection, _TlsVersion.ToSslProtocols(), !_Settings.AcceptInvalidCertificates).ConfigureAwait(false); - - if (!_SslStream.IsEncrypted) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream to " + _ServerIp + ":" + _ServerPort + " is not encrypted"); - throw new AuthenticationException("Stream is not encrypted"); - } - - if (!_SslStream.IsAuthenticated) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream to " + _ServerIp + ":" + _ServerPort + " is not authenticated"); - throw new AuthenticationException("Stream is not authenticated"); - } - - if (_Settings.MutuallyAuthenticate && !_SslStream.IsMutuallyAuthenticated) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "mutual authentication with " + _ServerIp + ":" + _ServerPort + " failed"); - throw new AuthenticationException("Mutual authentication failed"); - } - - _DataStream = _SslStream; - _ReceiveStream = new BufferedReadStream(_DataStream, _Settings.StreamBufferSize); - - if (_Keepalive.EnableTcpKeepAlives) EnableKeepalives(); - } - - private async Task ConnectSocketAsync() - { - Task connectTask = _Client.ConnectAsync(_ServerIp, _ServerPort); - Task timeoutTask = Task.Delay(TimeSpan.FromSeconds(_Settings.ConnectTimeoutSeconds)); - Task completedTask = await Task.WhenAny(connectTask, timeoutTask).ConfigureAwait(false); - - if (!ReferenceEquals(completedTask, connectTask)) - { - _Client.Close(); - _Settings.Logger?.Invoke(Severity.Error, _Header + "timeout connecting to " + _ServerIp + ":" + _ServerPort); - throw new TimeoutException("Timeout connecting to " + _ServerIp + ":" + _ServerPort); - } - - await connectTask.ConfigureAwait(false); - } - - private async Task CompleteConnectionInitializationAsync() - { - Task negotiationWindow = Task.Delay(50); - Task winner = await Task.WhenAny( - _InitializationFailure.Task, - _InitializationReady.Task, - _InitializationStarted.Task, - negotiationWindow).ConfigureAwait(false); - - if (ReferenceEquals(winner, _InitializationFailure.Task)) - { - CloseTransport(false); - throw await _InitializationFailure.Task.ConfigureAwait(false); - } - - if (ReferenceEquals(winner, _InitializationReady.Task)) - { - MarkConnected(); - return; - } - - if (ReferenceEquals(winner, _InitializationStarted.Task)) - { - Task result = await Task.WhenAny(_InitializationFailure.Task, _InitializationReady.Task, Task.Delay(_Settings.HandshakeTimeoutMs)).ConfigureAwait(false); - if (ReferenceEquals(result, _InitializationFailure.Task)) - { - CloseTransport(false); - throw await _InitializationFailure.Task.ConfigureAwait(false); - } - - if (ReferenceEquals(result, _InitializationReady.Task)) - { - MarkConnected(); - return; - } - - HandshakeFailedException timeoutException = new HandshakeFailedException("Connection initialization timed out."); - _InitializationFailure.TrySetResult(timeoutException); - CloseTransport(false); - throw timeoutException; - } - - MarkConnected(); - } - - private void MarkConnected() - { - if (Connected) return; - - Connected = true; - _Events.HandleServerConnected(this, new ConnectionEventArgs()); - _ServerConnectedRaised = true; - _Settings.Logger?.Invoke(Severity.Info, _Header + "connected to " + _ServerIp + ":" + _ServerPort); - } - - private void CloseTransport(bool waitForBackgroundTasks) - { - _TransportConnected = false; - Connected = false; - - if (_TokenSource != null) - { - try - { - if (!_TokenSource.IsCancellationRequested) - { - _TokenSource.Cancel(); - } - } - catch (ObjectDisposedException) - { - } - - _Token = default(CancellationToken); - } - - try - { - _SslStream?.Close(); - } - catch (ObjectDisposedException) - { - } - - try - { - _TcpStream?.Close(); - } - catch (ObjectDisposedException) - { - } - - try - { - _Client?.Close(); - } - catch (ObjectDisposedException) - { - } - - if (waitForBackgroundTasks) - { - try - { - if (_DataReceiver != null && !_DataReceiver.IsCompleted) - _DataReceiver.Wait(TimeSpan.FromSeconds(5)); - } - catch (AggregateException) { } - catch (ObjectDisposedException) { } - - try - { - if (_IdleServerMonitor != null && !_IdleServerMonitor.IsCompleted) - _IdleServerMonitor.Wait(TimeSpan.FromSeconds(5)); - } - catch (AggregateException) { } - catch (ObjectDisposedException) { } - } - } - - private async Task SendRegisterMessageAsync(CancellationToken token) - { - WatsonMessage registerMsg = new WatsonMessage(); - registerMsg.Status = MessageStatus.RegisterClient; - await SendInternalAsync(registerMsg, 0, null, token, true).ConfigureAwait(false); - } - - private async Task SendStatusMessageAsync(MessageStatus status, string reason, CancellationToken token) - { - byte[] data = Array.Empty(); - if (!String.IsNullOrEmpty(reason)) data = Encoding.UTF8.GetBytes(reason); - WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); - WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); - msg.Status = status; - await SendInternalAsync(msg, contentLength, stream, token, true).ConfigureAwait(false); - } - - private async Task ReadStatusMessageAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null || msg.ContentLength <= 0) return null; - byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - if (data == null || data.Length < 1) return null; - return Encoding.UTF8.GetString(data); - } - - private async Task SendHandshakeDataAsync(HandshakeMessage handshakeMessage, CancellationToken token) - { - if (handshakeMessage == null) throw new ArgumentNullException(nameof(handshakeMessage)); - - byte[] data = WatsonCommon.SerializeJsonBytes(SerializationHelper, handshakeMessage, false); - WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); - WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); - msg.Status = MessageStatus.HandshakeData; - await SendInternalAsync(msg, contentLength, stream, token, true).ConfigureAwait(false); - } - - private async Task ReadHandshakeMessageAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null) return null; - if (msg.ContentLength <= 0) return new HandshakeMessage(); - - byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - if (data == null || data.Length < 1) return new HandshakeMessage(); - return SerializationHelper.DeserializeJson(Encoding.UTF8.GetString(data)); - } - - private void FailInitialization(Exception exception) - { - if (exception == null) return; - _InitializationFailure.TrySetResult(exception); - } - - private void StartClientHandshake(CancellationToken token) - { - if (_ClientHandshakeTask != null) return; - - _ClientHandshakeTransport = new HandshakeSessionTransport( - async (msg, innerToken) => await SendHandshakeDataAsync(msg, innerToken).ConfigureAwait(false), - async (reason, status, innerToken) => await SendStatusMessageAsync(status, reason, innerToken).ConfigureAwait(false), - token); - _ClientHandshakeSession = new ClientHandshakeSession(_ClientHandshakeTransport); - _ClientHandshakeTask = RunClientHandshakeAsync(token); - } - - private async Task RunClientHandshakeAsync(CancellationToken token) - { - HandshakeResult result = null; - - using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.HandshakeTimeoutMs)) - using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) - { - try - { - result = await _Callbacks.HandshakeAsync(_ClientHandshakeSession, linkedCts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - if (timeoutCts.IsCancellationRequested) - { - result = HandshakeResult.Fail("Handshake timed out."); - } - else - { - result = HandshakeResult.Fail("Handshake canceled."); - } - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "handshake exception: " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - result = HandshakeResult.Fail("Handshake failed: " + e.Message); - } - } - - if (result == null) result = HandshakeResult.Succeed(); - if (result.Success) return; - - HandshakeFailedException exception = new HandshakeFailedException(result.Reason, result.FailureStatus); - _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, result.Reason, result.FailureStatus)); - FailInitialization(exception); - await SendStatusMessageAsync(result.FailureStatus, result.Reason, token).ConfigureAwait(false); - CloseTransport(false); - } - - #region Connection - - private void EnableKeepalives() - { + _DataReceiver = null; + } + } + + private void ResetInitializationState() + { + _ServerConnectedRaised = false; + _HandshakeRequired = false; + _InitializationStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _InitializationReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _InitializationFailure = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + _ClientHandshakeTransport?.Dispose(); + _ClientHandshakeTransport = null; + _ClientHandshakeSession = null; + _ClientHandshakeTask = null; + } + + private async Task ConnectCoreAsync() + { + if (Connected || _TransportConnected) throw new InvalidOperationException("Already connected to the server."); + + if (_Settings.LocalPort == 0) + { + _Client = new TcpClient(); + } + else + { + IPEndPoint ipe = new IPEndPoint(IPAddress.Any, _Settings.LocalPort); + _Client = new TcpClient(ipe); + } + + _Client.NoDelay = _Settings.NoDelay; + _Statistics = new WatsonTcpStatistics(); + + _Instrumentation?.Dispose(); + _Instrumentation = null; + if (_Settings.EnableMetrics || _Settings.EnableTracing) + { + _Instrumentation = new WatsonTcpInstrumentation( + WatsonTcpInstrumentation.RoleClient, + _Mode == Mode.Ssl ? WatsonTcpInstrumentation.ProtocolSsl : WatsonTcpInstrumentation.ProtocolTcp, + _Settings.EnableMetrics, + _Settings.EnableTracing, + () => Connected ? 1L : 0L, + null, + () => _SyncRequests.Count, + () => _Statistics != null ? _Statistics.UpTime.TotalSeconds : 0.0); + } + + _ConnectActivity = _Instrumentation?.StartConnectSpan(_ServerIp, _ServerPort); + + ValidateReceiveHandlerConfiguration(); + + try + { + if (_Mode == Mode.Tcp) + { + await ConnectTcpTransportAsync().ConfigureAwait(false); + } + else if (_Mode == Mode.Ssl) + { + await ConnectSslTransportAsync().ConfigureAwait(false); + } + else + { + throw new ArgumentException("Unknown mode: " + _Mode.ToString()); + } + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "exception encountered: " + e.Message); + HandleException(e); + throw; + } + + _TransportConnected = true; + ResetInitializationState(); + + _TokenSource = new CancellationTokenSource(); + _Token = _TokenSource.Token; + _MessageBuilder.MaxHeaderSize = _Settings.MaxHeaderSize; + _MessageBuilder.ReadStreamBuffer = _Settings.StreamBufferSize; + + _LastActivity = DateTime.UtcNow; + _IsTimeout = false; + + _DataReceiver = DataReceiver(_Token); + _IdleServerMonitor = IdleServerMonitor(_Token); + + WatsonMessage msg = new WatsonMessage(); + msg.Status = MessageStatus.RegisterClient; + + if (!await SendInternalAsync(msg, 0, null, default(CancellationToken), true).ConfigureAwait(false)) + { + Exception initException = null; + if (_InitializationFailure.Task.IsCompleted + && !_InitializationFailure.Task.IsCanceled + && !_InitializationFailure.Task.IsFaulted) + { + initException = await _InitializationFailure.Task.ConfigureAwait(false); + } + + CloseTransport(false); + if (initException != null) throw initException; + + _Settings.Logger?.Invoke(Severity.Alert, _Header + "unable to register GUID " + _Settings.Guid + " with the server"); + throw new ArgumentException("Server rejected GUID " + _Settings.Guid); + } + + await CompleteConnectionInitializationAsync().ConfigureAwait(false); + } + + private async Task ConnectTcpTransportAsync() + { + _Settings.Logger?.Invoke(Severity.Info, _Header + "connecting to " + _ServerIp + ":" + _ServerPort); + + _Client.LingerState = new LingerOption(true, 0); + await ConnectSocketAsync().ConfigureAwait(false); + + _SourceIp = ((IPEndPoint)_Client.Client.LocalEndPoint).Address.ToString(); + _SourcePort = ((IPEndPoint)_Client.Client.LocalEndPoint).Port; + _TcpStream = _Client.GetStream(); + _DataStream = _TcpStream; + _ReceiveStream = new BufferedReadStream(_DataStream, _Settings.StreamBufferSize); + _SslStream = null; + + if (_Keepalive.EnableTcpKeepAlives) EnableKeepalives(); + } + + private async Task ConnectSslTransportAsync() + { + _Settings.Logger?.Invoke(Severity.Info, _Header + "connecting with SSL to " + _ServerIp + ":" + _ServerPort); + + _Client.LingerState = new LingerOption(true, 0); + await ConnectSocketAsync().ConfigureAwait(false); + + _SourceIp = ((IPEndPoint)_Client.Client.LocalEndPoint).Address.ToString(); + _SourcePort = ((IPEndPoint)_Client.Client.LocalEndPoint).Port; + + if (_Settings.AcceptInvalidCertificates) + _SslStream = new SslStream(_Client.GetStream(), false, _SslConfiguration.ServerCertificateValidationCallback, _SslConfiguration.ClientCertificateSelectionCallback); + else + _SslStream = new SslStream(_Client.GetStream(), false); + + await _SslStream.AuthenticateAsClientAsync(_ServerIp, _SslCertificateCollection, _TlsVersion.ToSslProtocols(), !_Settings.AcceptInvalidCertificates).ConfigureAwait(false); + + if (!_SslStream.IsEncrypted) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream to " + _ServerIp + ":" + _ServerPort + " is not encrypted"); + throw new AuthenticationException("Stream is not encrypted"); + } + + if (!_SslStream.IsAuthenticated) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream to " + _ServerIp + ":" + _ServerPort + " is not authenticated"); + throw new AuthenticationException("Stream is not authenticated"); + } + + if (_Settings.MutuallyAuthenticate && !_SslStream.IsMutuallyAuthenticated) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "mutual authentication with " + _ServerIp + ":" + _ServerPort + " failed"); + throw new AuthenticationException("Mutual authentication failed"); + } + + _DataStream = _SslStream; + _ReceiveStream = new BufferedReadStream(_DataStream, _Settings.StreamBufferSize); + + if (_Keepalive.EnableTcpKeepAlives) EnableKeepalives(); + } + + private async Task ConnectSocketAsync() + { + Task connectTask = _Client.ConnectAsync(_ServerIp, _ServerPort); + Task timeoutTask = Task.Delay(TimeSpan.FromSeconds(_Settings.ConnectTimeoutSeconds)); + Task completedTask = await Task.WhenAny(connectTask, timeoutTask).ConfigureAwait(false); + + if (!ReferenceEquals(completedTask, connectTask)) + { + _Client.Close(); + _Settings.Logger?.Invoke(Severity.Error, _Header + "timeout connecting to " + _ServerIp + ":" + _ServerPort); + throw new TimeoutException("Timeout connecting to " + _ServerIp + ":" + _ServerPort); + } + + await connectTask.ConfigureAwait(false); + } + + private async Task CompleteConnectionInitializationAsync() + { + Task negotiationWindow = Task.Delay(50); + Task winner = await Task.WhenAny( + _InitializationFailure.Task, + _InitializationReady.Task, + _InitializationStarted.Task, + negotiationWindow).ConfigureAwait(false); + + if (ReferenceEquals(winner, _InitializationFailure.Task)) + { + CloseTransport(false); + throw await _InitializationFailure.Task.ConfigureAwait(false); + } + + if (ReferenceEquals(winner, _InitializationReady.Task)) + { + MarkConnected(); + return; + } + + if (ReferenceEquals(winner, _InitializationStarted.Task)) + { + Task result = await Task.WhenAny(_InitializationFailure.Task, _InitializationReady.Task, Task.Delay(_Settings.HandshakeTimeoutMs)).ConfigureAwait(false); + if (ReferenceEquals(result, _InitializationFailure.Task)) + { + CloseTransport(false); + throw await _InitializationFailure.Task.ConfigureAwait(false); + } + + if (ReferenceEquals(result, _InitializationReady.Task)) + { + MarkConnected(); + return; + } + + HandshakeFailedException timeoutException = new HandshakeFailedException("Connection initialization timed out."); + _InitializationFailure.TrySetResult(timeoutException); + CloseTransport(false); + throw timeoutException; + } + + MarkConnected(); + } + + private void MarkConnected() + { + if (Connected) return; + + Connected = true; + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeConnected); + _ConnectActivity?.SetTag(WatsonTcpMetrics.TagOutcome, WatsonTcpInstrumentation.OutcomeConnected); + _Events.HandleServerConnected(this, new ConnectionEventArgs()); + _ServerConnectedRaised = true; + _Settings.Logger?.Invoke(Severity.Info, _Header + "connected to " + _ServerIp + ":" + _ServerPort); + } + + private void CloseTransport(bool waitForBackgroundTasks) + { + _TransportConnected = false; + Connected = false; + + if (_TokenSource != null) + { + try + { + if (!_TokenSource.IsCancellationRequested) + { + _TokenSource.Cancel(); + } + } + catch (ObjectDisposedException) + { + } + + _Token = default(CancellationToken); + } + + try + { + _SslStream?.Close(); + } + catch (ObjectDisposedException) + { + } + + try + { + _TcpStream?.Close(); + } + catch (ObjectDisposedException) + { + } + + try + { + _Client?.Close(); + } + catch (ObjectDisposedException) + { + } + + if (waitForBackgroundTasks) + { + try + { + if (_DataReceiver != null && !_DataReceiver.IsCompleted) + _DataReceiver.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) { } + catch (ObjectDisposedException) { } + + try + { + if (_IdleServerMonitor != null && !_IdleServerMonitor.IsCompleted) + _IdleServerMonitor.Wait(TimeSpan.FromSeconds(5)); + } + catch (AggregateException) { } + catch (ObjectDisposedException) { } + } + } + + private async Task SendRegisterMessageAsync(CancellationToken token) + { + WatsonMessage registerMsg = new WatsonMessage(); + registerMsg.Status = MessageStatus.RegisterClient; + await SendInternalAsync(registerMsg, 0, null, token, true).ConfigureAwait(false); + } + + private async Task SendStatusMessageAsync(MessageStatus status, string reason, CancellationToken token) + { + byte[] data = Array.Empty(); + if (!String.IsNullOrEmpty(reason)) data = Encoding.UTF8.GetBytes(reason); + WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); + WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); + msg.Status = status; + await SendInternalAsync(msg, contentLength, stream, token, true).ConfigureAwait(false); + } + + private async Task ReadStatusMessageAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null || msg.ContentLength <= 0) return null; + byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + if (data == null || data.Length < 1) return null; + return Encoding.UTF8.GetString(data); + } + + private async Task SendHandshakeDataAsync(HandshakeMessage handshakeMessage, CancellationToken token) + { + if (handshakeMessage == null) throw new ArgumentNullException(nameof(handshakeMessage)); + + byte[] data = WatsonCommon.SerializeJsonBytes(SerializationHelper, handshakeMessage, false); + WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); + WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); + msg.Status = MessageStatus.HandshakeData; + await SendInternalAsync(msg, contentLength, stream, token, true).ConfigureAwait(false); + } + + private async Task ReadHandshakeMessageAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null) return null; + if (msg.ContentLength <= 0) return new HandshakeMessage(); + + byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + if (data == null || data.Length < 1) return new HandshakeMessage(); + return SerializationHelper.DeserializeJson(Encoding.UTF8.GetString(data)); + } + + private void FailInitialization(Exception exception) + { + if (exception == null) return; + _InitializationFailure.TrySetResult(exception); + } + + private void HandleException(Exception e) + { + _Instrumentation?.ExceptionRecorded(e); + _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + } + + private void StartClientHandshake(CancellationToken token) + { + if (_ClientHandshakeTask != null) return; + + _ClientHandshakeTransport = new HandshakeSessionTransport( + async (msg, innerToken) => await SendHandshakeDataAsync(msg, innerToken).ConfigureAwait(false), + async (reason, status, innerToken) => await SendStatusMessageAsync(status, reason, innerToken).ConfigureAwait(false), + token); + _ClientHandshakeSession = new ClientHandshakeSession(_ClientHandshakeTransport); + _ClientHandshakeTask = RunClientHandshakeAsync(token); + } + + private async Task RunClientHandshakeAsync(CancellationToken token) + { + HandshakeResult result = null; + long handshakeStartTimestamp = Stopwatch.GetTimestamp(); + string handshakeOutcome = WatsonTcpInstrumentation.OutcomeSuccess; + + using (Activity handshakeSpan = _Instrumentation?.StartHandshakeSpan()) + using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.HandshakeTimeoutMs)) + using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) + { + try + { + result = await _Callbacks.HandshakeAsync(_ClientHandshakeSession, linkedCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (timeoutCts.IsCancellationRequested) + { + handshakeOutcome = WatsonTcpInstrumentation.OutcomeTimeout; + result = HandshakeResult.Fail("Handshake timed out."); + } + else + { + handshakeOutcome = WatsonTcpInstrumentation.OutcomeCanceled; + result = HandshakeResult.Fail("Handshake canceled."); + } + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "handshake exception: " + e.Message); + HandleException(e); + handshakeOutcome = WatsonTcpInstrumentation.OutcomeFailure; + result = HandshakeResult.Fail("Handshake failed: " + e.Message); + } + } + + if (result == null) result = HandshakeResult.Succeed(); + if (result.Success) + { + _Instrumentation?.HandshakeCompleted(WatsonTcpInstrumentation.OutcomeSuccess, handshakeStartTimestamp); + return; + } + + if (handshakeOutcome == WatsonTcpInstrumentation.OutcomeSuccess) handshakeOutcome = WatsonTcpInstrumentation.OutcomeFailure; + _Instrumentation?.HandshakeCompleted(handshakeOutcome, handshakeStartTimestamp); + + HandshakeFailedException exception = new HandshakeFailedException(result.Reason, result.FailureStatus); + _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, result.Reason, result.FailureStatus)); + FailInitialization(exception); + await SendStatusMessageAsync(result.FailureStatus, result.Reason, token).ConfigureAwait(false); + CloseTransport(false); + } + + #region Connection + + private void EnableKeepalives() + { // issues with definitions: https://github.com/dotnet/sdk/issues/14540 try @@ -950,99 +1003,101 @@ private void EnableKeepalives() { _Settings.Logger?.Invoke(Severity.Error, _Header + "keepalives not supported on this platform, disabled"); _Keepalive.EnableTcpKeepAlives = false; - } - } - - private void ValidateReceiveHandlerConfiguration() - { - bool usingMessages = _Events.IsUsingMessages; - bool usingSyncStreams = _Events.IsUsingStreams; - bool usingAsyncStreams = _Callbacks.StreamReceivedAsync != null; - - if (!usingMessages && !usingSyncStreams && !usingAsyncStreams) - { - throw new InvalidOperationException("One of either 'MessageReceived', 'StreamReceived', or 'Callbacks.StreamReceivedAsync' must first be set."); - } - - if (usingMessages && usingAsyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and Callbacks.StreamReceivedAsync are both configured; MessageReceived will be used."); - } - - if (usingMessages && usingSyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and StreamReceived are both configured; MessageReceived will be used."); - } - - if (!usingMessages && usingAsyncStreams && usingSyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "Callbacks.StreamReceivedAsync and StreamReceived are both configured; Callbacks.StreamReceivedAsync will be used."); - } - } - - private async Task HandleStreamPayloadAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null) throw new ArgumentNullException(nameof(msg)); - - bool useAsyncCallback = _Callbacks.StreamReceivedAsync != null; - bool useSyncEvent = _Events.IsUsingStreams; - - if (!useAsyncCallback && !useSyncEvent) - { - throw new InvalidOperationException("Receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync."); - } - - bool useBufferedStream = msg.ContentLength < _Settings.MaxProxiedStreamSize; - Stream payloadStream = msg.DataStream; - - if (useBufferedStream) - { - payloadStream = await WatsonCommon.DataStreamToMemoryStream(msg.ContentLength, msg.DataStream, _Settings.StreamBufferSize, token).ConfigureAwait(false); - } - - WatsonStream watsonStream = new WatsonStream(msg.ContentLength, payloadStream); - StreamReceivedEventArgs args = new StreamReceivedEventArgs(null, msg.Metadata, msg.ContentLength, watsonStream); - bool preserveOriginalException = false; - - try - { - if (useAsyncCallback) - { - await _Callbacks.StreamReceivedAsync(args, token).ConfigureAwait(false); - } - else if (useBufferedStream) - { - await Task.Run(() => _Events.HandleStreamReceived(this, args), token).ConfigureAwait(false); - } - else - { - _Events.HandleStreamReceived(this, args); - } - } - catch (Exception e) when (!(e is OperationCanceledException)) - { - preserveOriginalException = true; - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream receive handler exception for " + _ServerIp + ":" + _ServerPort + ": " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - throw; - } - finally - { - if (watsonStream.RemainingBytes > 0 && !token.IsCancellationRequested) - { - try - { - await watsonStream.DrainAsync(_Settings.StreamBufferSize, token).ConfigureAwait(false); - } - catch (Exception e) when (preserveOriginalException) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "failed draining unread stream bytes after handler exception: " + e.Message); - } - } - } - } - - #endregion + } + } + + private void ValidateReceiveHandlerConfiguration() + { + bool usingMessages = _Events.IsUsingMessages; + bool usingSyncStreams = _Events.IsUsingStreams; + bool usingAsyncStreams = _Callbacks.StreamReceivedAsync != null; + + if (!usingMessages && !usingSyncStreams && !usingAsyncStreams) + { + throw new InvalidOperationException("One of either 'MessageReceived', 'StreamReceived', or 'Callbacks.StreamReceivedAsync' must first be set."); + } + + if (usingMessages && usingAsyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and Callbacks.StreamReceivedAsync are both configured; MessageReceived will be used."); + } + + if (usingMessages && usingSyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and StreamReceived are both configured; MessageReceived will be used."); + } + + if (!usingMessages && usingAsyncStreams && usingSyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "Callbacks.StreamReceivedAsync and StreamReceived are both configured; Callbacks.StreamReceivedAsync will be used."); + } + } + + private async Task HandleStreamPayloadAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null) throw new ArgumentNullException(nameof(msg)); + + bool useAsyncCallback = _Callbacks.StreamReceivedAsync != null; + bool useSyncEvent = _Events.IsUsingStreams; + + if (!useAsyncCallback && !useSyncEvent) + { + throw new InvalidOperationException("Receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync."); + } + + bool useBufferedStream = msg.ContentLength < _Settings.MaxProxiedStreamSize; + Stream payloadStream = msg.DataStream; + + if (useBufferedStream) + { + payloadStream = await WatsonCommon.DataStreamToMemoryStream(msg.ContentLength, msg.DataStream, _Settings.StreamBufferSize, token).ConfigureAwait(false); + } + + WatsonStream watsonStream = new WatsonStream(msg.ContentLength, payloadStream); + StreamReceivedEventArgs args = new StreamReceivedEventArgs(null, msg.Metadata, msg.ContentLength, watsonStream); + bool preserveOriginalException = false; + + try + { + if (useAsyncCallback) + { + await _Callbacks.StreamReceivedAsync(args, token).ConfigureAwait(false); + } + else if (useBufferedStream) + { + await Task.Run(() => _Events.HandleStreamReceived(this, args), token).ConfigureAwait(false); + } + else + { + _Events.HandleStreamReceived(this, args); + } + } + catch (Exception e) when (!(e is OperationCanceledException)) + { + preserveOriginalException = true; + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream receive handler exception for " + _ServerIp + ":" + _ServerPort + ": " + e.Message); + HandleException(e); + throw; + } + finally + { + if (watsonStream.RemainingBytes > 0 && !token.IsCancellationRequested) + { + long drained = watsonStream.RemainingBytes; + try + { + await watsonStream.DrainAsync(_Settings.StreamBufferSize, token).ConfigureAwait(false); + _Instrumentation?.StreamDrained(drained); + } + catch (Exception e) when (preserveOriginalException) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "failed draining unread stream bytes after handler exception: " + e.Message); + } + } + } + } + + #endregion #region Read @@ -1050,24 +1105,24 @@ private async Task DataReceiver(CancellationToken token) { DisconnectReason reason = DisconnectReason.Normal; - while (true) - { - bool readLockHeld = false; - - try - { - token.ThrowIfCancellationRequested(); - - #region Check-for-Connection - #endregion - - #region Read-Message - - await _ReadLock.WaitAsync(token).ConfigureAwait(false); - readLockHeld = true; - WatsonMessage msg = await _MessageBuilder.BuildFromStream(_ReceiveStream, token).ConfigureAwait(false); - - _LastActivity = DateTime.UtcNow; + while (true) + { + bool readLockHeld = false; + + try + { + token.ThrowIfCancellationRequested(); + + #region Check-for-Connection + #endregion + + #region Read-Message + + await _ReadLock.WaitAsync(token).ConfigureAwait(false); + readLockHeld = true; + WatsonMessage msg = await _MessageBuilder.BuildFromStream(_ReceiveStream, token).ConfigureAwait(false); + + _LastActivity = DateTime.UtcNow; #endregion @@ -1085,54 +1140,56 @@ private async Task DataReceiver(CancellationToken token) reason = DisconnectReason.Shutdown; break; } - else if (msg.Status == MessageStatus.Timeout) - { - _Settings?.Logger?.Invoke(Severity.Info, _Header + "disconnect due to timeout"); - reason = DisconnectReason.Timeout; - break; - } - else if (msg.Status == MessageStatus.ConnectionRejected) - { - string rejectionReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - if (String.IsNullOrEmpty(rejectionReason)) rejectionReason = "Connection rejected."; - _Settings.Logger?.Invoke(Severity.Error, _Header + rejectionReason); - reason = DisconnectReason.ConnectionRejected; - ConnectionRejectedException exception = new ConnectionRejectedException(rejectionReason, MessageStatus.ConnectionRejected); - _Events.HandleConnectionRejected(this, new ConnectionRejectedEventArgs(null, rejectionReason, MessageStatus.ConnectionRejected)); - FailInitialization(exception); - break; - } - else if (msg.Status == MessageStatus.AuthSuccess) - { - await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - _Settings.Logger?.Invoke(Severity.Debug, _Header + "authentication successful"); - _InitializationStarted.TrySetResult(true); - Task unawaited = Task.Run(() => _Events.HandleAuthenticationSucceeded(this, EventArgs.Empty), token); - await SendRegisterMessageAsync(token).ConfigureAwait(false); - if (!_HandshakeRequired) - { - _InitializationReady.TrySetResult(true); - } - - continue; - } - else if (msg.Status == MessageStatus.AuthFailure) - { - string authFailureReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - if (String.IsNullOrEmpty(authFailureReason)) authFailureReason = "Authentication failed."; - _Settings.Logger?.Invoke(Severity.Error, _Header + authFailureReason); - reason = DisconnectReason.AuthFailure; - Task unawaited = Task.Run(() => _Events.HandleAuthenticationFailure(this, EventArgs.Empty), token); - FailInitialization(new ConnectionRejectedException(authFailureReason, MessageStatus.AuthFailure)); - break; - } - else if (msg.Status == MessageStatus.AuthRequired) - { - await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - _InitializationStarted.TrySetResult(true); - _Settings.Logger?.Invoke(Severity.Info, _Header + "authentication required by server; please authenticate using pre-shared key"); - - string psk = null; + else if (msg.Status == MessageStatus.Timeout) + { + _Settings?.Logger?.Invoke(Severity.Info, _Header + "disconnect due to timeout"); + reason = DisconnectReason.Timeout; + break; + } + else if (msg.Status == MessageStatus.ConnectionRejected) + { + string rejectionReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + if (String.IsNullOrEmpty(rejectionReason)) rejectionReason = "Connection rejected."; + _Settings.Logger?.Invoke(Severity.Error, _Header + rejectionReason); + reason = DisconnectReason.ConnectionRejected; + ConnectionRejectedException exception = new ConnectionRejectedException(rejectionReason, MessageStatus.ConnectionRejected); + _Events.HandleConnectionRejected(this, new ConnectionRejectedEventArgs(null, rejectionReason, MessageStatus.ConnectionRejected)); + FailInitialization(exception); + break; + } + else if (msg.Status == MessageStatus.AuthSuccess) + { + await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + _Settings.Logger?.Invoke(Severity.Debug, _Header + "authentication successful"); + _Instrumentation?.Authentication(WatsonTcpInstrumentation.OutcomeSuccess); + _InitializationStarted.TrySetResult(true); + Task unawaited = Task.Run(() => _Events.HandleAuthenticationSucceeded(this, EventArgs.Empty), token); + await SendRegisterMessageAsync(token).ConfigureAwait(false); + if (!_HandshakeRequired) + { + _InitializationReady.TrySetResult(true); + } + + continue; + } + else if (msg.Status == MessageStatus.AuthFailure) + { + string authFailureReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + if (String.IsNullOrEmpty(authFailureReason)) authFailureReason = "Authentication failed."; + _Settings.Logger?.Invoke(Severity.Error, _Header + authFailureReason); + _Instrumentation?.Authentication(WatsonTcpInstrumentation.OutcomeFailure); + reason = DisconnectReason.AuthFailure; + Task unawaited = Task.Run(() => _Events.HandleAuthenticationFailure(this, EventArgs.Empty), token); + FailInitialization(new ConnectionRejectedException(authFailureReason, MessageStatus.AuthFailure)); + break; + } + else if (msg.Status == MessageStatus.AuthRequired) + { + await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + _InitializationStarted.TrySetResult(true); + _Settings.Logger?.Invoke(Severity.Info, _Header + "authentication required by server; please authenticate using pre-shared key"); + + string psk = null; // First check if pre-shared key is set in settings if (!String.IsNullOrEmpty(_Settings.PresharedKey)) @@ -1153,62 +1210,62 @@ private async Task DataReceiver(CancellationToken token) if (!String.IsNullOrEmpty(psk)) { await AuthenticateAsync(psk, token); - } - else - { - // No pre-shared key available - neither in settings nor from callback - _Settings.Logger?.Invoke(Severity.Error, _Header + "authentication required by server but no pre-shared key available"); - ConnectionRejectedException exception = new ConnectionRejectedException("Server requires authentication but no pre-shared key is configured. Set Settings.PresharedKey or implement Callbacks.AuthenticationRequested.", MessageStatus.AuthRequired); - FailInitialization(exception); - throw exception; - } - continue; - } - else if (msg.Status == MessageStatus.HandshakeBegin) - { - await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - _InitializationStarted.TrySetResult(true); - _HandshakeRequired = true; - - if (_Callbacks.HandshakeAsync == null) - { - HandshakeFailedException exception = new HandshakeFailedException("Server requested a handshake but no handshake callback is configured."); - _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, exception.Message, MessageStatus.HandshakeFailure)); - FailInitialization(exception); - await SendStatusMessageAsync(MessageStatus.HandshakeFailure, exception.Message, token).ConfigureAwait(false); - reason = DisconnectReason.HandshakeFailure; - break; - } - - StartClientHandshake(token); - continue; - } - else if (msg.Status == MessageStatus.HandshakeData) - { - HandshakeMessage handshakeMsg = await ReadHandshakeMessageAsync(msg, token).ConfigureAwait(false); - _ClientHandshakeTransport?.Enqueue(handshakeMsg); - continue; - } - else if (msg.Status == MessageStatus.HandshakeSuccess) - { - await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - _Settings.Logger?.Invoke(Severity.Debug, _Header + "handshake successful"); - _Events.HandleHandshakeSucceeded(this, new HandshakeSucceededEventArgs()); - await SendRegisterMessageAsync(token).ConfigureAwait(false); - _InitializationReady.TrySetResult(true); - continue; - } - else if (msg.Status == MessageStatus.HandshakeFailure) - { - string handshakeFailureReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - if (String.IsNullOrEmpty(handshakeFailureReason)) handshakeFailureReason = "Handshake failed."; - _Settings.Logger?.Invoke(Severity.Error, _Header + handshakeFailureReason); - reason = DisconnectReason.HandshakeFailure; - HandshakeFailedException exception = new HandshakeFailedException(handshakeFailureReason, MessageStatus.HandshakeFailure); - _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, handshakeFailureReason, MessageStatus.HandshakeFailure)); - FailInitialization(exception); - break; - } + } + else + { + // No pre-shared key available - neither in settings nor from callback + _Settings.Logger?.Invoke(Severity.Error, _Header + "authentication required by server but no pre-shared key available"); + ConnectionRejectedException exception = new ConnectionRejectedException("Server requires authentication but no pre-shared key is configured. Set Settings.PresharedKey or implement Callbacks.AuthenticationRequested.", MessageStatus.AuthRequired); + FailInitialization(exception); + throw exception; + } + continue; + } + else if (msg.Status == MessageStatus.HandshakeBegin) + { + await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + _InitializationStarted.TrySetResult(true); + _HandshakeRequired = true; + + if (_Callbacks.HandshakeAsync == null) + { + HandshakeFailedException exception = new HandshakeFailedException("Server requested a handshake but no handshake callback is configured."); + _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, exception.Message, MessageStatus.HandshakeFailure)); + FailInitialization(exception); + await SendStatusMessageAsync(MessageStatus.HandshakeFailure, exception.Message, token).ConfigureAwait(false); + reason = DisconnectReason.HandshakeFailure; + break; + } + + StartClientHandshake(token); + continue; + } + else if (msg.Status == MessageStatus.HandshakeData) + { + HandshakeMessage handshakeMsg = await ReadHandshakeMessageAsync(msg, token).ConfigureAwait(false); + _ClientHandshakeTransport?.Enqueue(handshakeMsg); + continue; + } + else if (msg.Status == MessageStatus.HandshakeSuccess) + { + await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + _Settings.Logger?.Invoke(Severity.Debug, _Header + "handshake successful"); + _Events.HandleHandshakeSucceeded(this, new HandshakeSucceededEventArgs()); + await SendRegisterMessageAsync(token).ConfigureAwait(false); + _InitializationReady.TrySetResult(true); + continue; + } + else if (msg.Status == MessageStatus.HandshakeFailure) + { + string handshakeFailureReason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + if (String.IsNullOrEmpty(handshakeFailureReason)) handshakeFailureReason = "Handshake failed."; + _Settings.Logger?.Invoke(Severity.Error, _Header + handshakeFailureReason); + reason = DisconnectReason.HandshakeFailure; + HandshakeFailedException exception = new HandshakeFailedException(handshakeFailureReason, MessageStatus.HandshakeFailure); + _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(null, handshakeFailureReason, MessageStatus.HandshakeFailure)); + FailInitialization(exception); + break; + } #endregion @@ -1264,6 +1321,7 @@ private async Task DataReceiver(CancellationToken token) } else { + _Instrumentation?.SyncExpiredMessage(WatsonTcpInstrumentation.KindRequest); _Settings.Logger?.Invoke(Severity.Debug, _Header + "expired synchronous request received and discarded"); } } @@ -1279,6 +1337,7 @@ private async Task DataReceiver(CancellationToken token) TaskCompletionSource tcs; if (_SyncRequests.TryRemove(msg.ConversationGuid, out tcs)) { + _Instrumentation?.SyncResponseReceived(); SyncResponse syncResp = new SyncResponse(msg.ConversationGuid, msg.ExpirationUtc.Value, msg.Metadata, msgData); tcs.TrySetResult(syncResp); } @@ -1289,6 +1348,7 @@ private async Task DataReceiver(CancellationToken token) } else { + _Instrumentation?.SyncExpiredMessage(WatsonTcpInstrumentation.KindResponse); _Settings.Logger?.Invoke(Severity.Debug, _Header + "expired synchronous response received and discarded"); TaskCompletionSource tcs; _SyncRequests.TryRemove(msg.ConversationGuid, out tcs); @@ -1296,144 +1356,162 @@ private async Task DataReceiver(CancellationToken token) } else { - byte[] msgData = null; - - if (_Events.IsUsingMessages) + using (Activity receiveSpan = _Instrumentation?.StartReceiveSpan(msg.ContentLength, null)) { - msgData = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - MessageReceivedEventArgs args = new MessageReceivedEventArgs(null, msg.Metadata, msgData); - await Task.Run(() => _Events.HandleMessageReceived(this, args), token); + byte[] msgData = null; + + if (_Events.IsUsingMessages) + { + msgData = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + MessageReceivedEventArgs args = new MessageReceivedEventArgs(null, msg.Metadata, msgData); + await Task.Run(() => _Events.HandleMessageReceived(this, args), token); + } + else if (_Callbacks.StreamReceivedAsync != null || _Events.IsUsingStreams) + { + await HandleStreamPayloadAsync(msg, token).ConfigureAwait(false); + } + else + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync"); + break; + } } - else if (_Callbacks.StreamReceivedAsync != null || _Events.IsUsingStreams) - { - await HandleStreamPayloadAsync(msg, token).ConfigureAwait(false); - } - else - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync"); - break; - } } #endregion _Statistics.IncrementReceivedMessages(); _Statistics.AddReceivedBytes(msg.ContentLength); + _Instrumentation?.MessageReceived(msg, msg.ContentLength); } catch (ObjectDisposedException ode) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "object disposed exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(ode)); + HandleException(ode); break; } catch (TaskCanceledException tce) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "task canceled exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(tce)); + HandleException(tce); break; } catch (OperationCanceledException oce) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "operation canceled exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(oce)); + HandleException(oce); break; } catch (IOException ioe) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "IO exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(ioe)); + HandleException(ioe); break; } catch (Exception e) { _Settings?.Logger?.Invoke(Severity.Error, _Header + "data receiver exception for " + _ServerIp + ":" + _ServerPort + ": " + e.Message + Environment.NewLine); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + HandleException(e); break; } - finally - { - if (_ReadLock != null && readLockHeld) _ReadLock.Release(); - } - } - - try - { - _SslStream?.Close(); - } - catch (ObjectDisposedException) - { - } - - try - { - _TcpStream?.Close(); - } - catch (ObjectDisposedException) - { - } - - try - { - _Client?.Close(); - } - catch (ObjectDisposedException) - { - } - - _TransportConnected = false; - Connected = false; - - if (_IsTimeout) reason = DisconnectReason.Timeout; - - _Settings?.Logger?.Invoke(Severity.Debug, _Header + "data receiver terminated for " + _ServerIp + ":" + _ServerPort); - if (_ServerConnectedRaised) - { - _Events?.HandleServerDisconnected(this, new DisconnectionEventArgs(null, reason)); - } - } + finally + { + if (_ReadLock != null && readLockHeld) _ReadLock.Release(); + } + } + + try + { + _SslStream?.Close(); + } + catch (ObjectDisposedException) + { + } + + try + { + _TcpStream?.Close(); + } + catch (ObjectDisposedException) + { + } + + try + { + _Client?.Close(); + } + catch (ObjectDisposedException) + { + } + + _TransportConnected = false; + Connected = false; + + if (_IsTimeout) reason = DisconnectReason.Timeout; + + Activity connectActivity = _ConnectActivity; + _ConnectActivity = null; + if (connectActivity != null) + { + connectActivity.SetTag(WatsonTcpMetrics.TagReason, reason.ToString()); + connectActivity.Dispose(); + } + + _Settings?.Logger?.Invoke(Severity.Debug, _Header + "data receiver terminated for " + _ServerIp + ":" + _ServerPort); + if (_ServerConnectedRaised) + { + _Instrumentation?.Disconnection(reason); + _Events?.HandleServerDisconnected(this, new DisconnectionEventArgs(null, reason)); + } + } #endregion #region Send - private async Task SendInternalAsync(WatsonMessage msg, long contentLength, Stream stream, CancellationToken token, bool allowWhilePending = false) - { - if (msg == null) throw new ArgumentNullException(nameof(msg)); - if (!Connected && !(allowWhilePending && _TransportConnected)) return false; + private async Task SendInternalAsync(WatsonMessage msg, long contentLength, Stream stream, CancellationToken token, bool allowWhilePending = false) + { + if (msg == null) throw new ArgumentNullException(nameof(msg)); + if (!Connected && !(allowWhilePending && _TransportConnected)) return false; if (contentLength > 0 && (stream == null || !stream.CanRead)) { throw new ArgumentException("Cannot read from supplied stream."); } - CancellationTokenSource linkedCts = null; - if (token == default(CancellationToken)) - { - token = _Token; - } - else if (_Token.CanBeCanceled) - { - linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _Token); - token = linkedCts.Token; - } - - bool disconnectDetected = false; - - if (_Client == null || _DataStream == null) - { - linkedCts?.Dispose(); - return false; - } + CancellationTokenSource linkedCts = null; + if (token == default(CancellationToken)) + { + token = _Token; + } + else if (_Token.CanBeCanceled) + { + linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _Token); + token = linkedCts.Token; + } + + bool disconnectDetected = false; + + if (_Client == null || _DataStream == null) + { + linkedCts?.Dispose(); + return false; + } await _WriteLock.WaitAsync(token).ConfigureAwait(false); - try - { - await SendMessageAsync(msg, contentLength, stream, token).ConfigureAwait(false); - - _Statistics.IncrementSentMessages(); - _Statistics.AddSentBytes(contentLength); + try + { + long sendStartTimestamp = Stopwatch.GetTimestamp(); + using (Activity sendSpan = _Instrumentation?.StartSendSpan(contentLength, msg.SyncRequest, null)) + { + await SendMessageAsync(msg, contentLength, stream, token).ConfigureAwait(false); + } + + _Statistics.IncrementSentMessages(); + _Statistics.AddSentBytes(contentLength); + _Instrumentation?.MessageSent(msg, contentLength, sendStartTimestamp); return true; } catch (TaskCanceledException) @@ -1459,17 +1537,17 @@ private async Task SendInternalAsync(WatsonMessage msg, long contentLength } finally { - _WriteLock.Release(); - linkedCts?.Dispose(); - - if (disconnectDetected) - { - _TransportConnected = false; - Connected = false; - CloseTransport(false); - } - } - } + _WriteLock.Release(); + linkedCts?.Dispose(); + + if (disconnectDetected) + { + _TransportConnected = false; + Connected = false; + CloseTransport(false); + } + } + } private async Task SendAndWaitInternalAsync(WatsonMessage msg, int timeoutMs, long contentLength, Stream stream, CancellationToken token) { @@ -1481,25 +1559,32 @@ private async Task SendAndWaitInternalAsync(WatsonMessage msg, int bool disconnectDetected = false; - if (_Client == null || _DataStream == null) - { - disconnectDetected = true; - throw new InvalidOperationException("Client is not connected to the server."); + if (_Client == null || _DataStream == null) + { + disconnectDetected = true; + throw new InvalidOperationException("Client is not connected to the server."); } // Register a TaskCompletionSource for this conversation before sending TaskCompletionSource tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _SyncRequests[msg.ConversationGuid] = tcs; + long syncStartTimestamp = Stopwatch.GetTimestamp(); + + using (Activity syncSpan = _Instrumentation?.StartSyncSpan(msg.ConversationGuid, contentLength)) + { await _WriteLock.WaitAsync(token).ConfigureAwait(false); try { - await SendMessageAsync(msg, contentLength, stream, token).ConfigureAwait(false); - _Settings.Logger?.Invoke(Severity.Debug, _Header + "synchronous request sent: " + msg.ConversationGuid); + long sendStartTimestamp = Stopwatch.GetTimestamp(); + await SendMessageAsync(msg, contentLength, stream, token).ConfigureAwait(false); + _Settings.Logger?.Invoke(Severity.Debug, _Header + "synchronous request sent: " + msg.ConversationGuid); _Statistics.IncrementSentMessages(); _Statistics.AddSentBytes(contentLength); + _Instrumentation?.MessageSent(msg, contentLength, sendStartTimestamp); + _Instrumentation?.SyncRequestSent(); } catch (TaskCanceledException) { @@ -1538,6 +1623,7 @@ private async Task SendAndWaitInternalAsync(WatsonMessage msg, int { linkedCts.Token.Register(() => tcs.TrySetCanceled()); SyncResponse ret = await tcs.Task.ConfigureAwait(false); + _Instrumentation?.SyncCompleted(WatsonTcpInstrumentation.OutcomeCompleted, syncStartTimestamp); return ret; } catch (TaskCanceledException) @@ -1546,6 +1632,7 @@ private async Task SendAndWaitInternalAsync(WatsonMessage msg, int if (timeoutCts.IsCancellationRequested) { + _Instrumentation?.SyncCompleted(WatsonTcpInstrumentation.OutcomeTimeout, syncStartTimestamp); _Settings.Logger?.Invoke(Severity.Error, _Header + "synchronous response not received within the timeout window"); throw new TimeoutException("A response to a synchronous request was not received within the timeout window."); } @@ -1554,14 +1641,15 @@ private async Task SendAndWaitInternalAsync(WatsonMessage msg, int } } } + } } - private async Task SendMessageAsync(WatsonMessage msg, long contentLength, Stream stream, CancellationToken token) - { - msg.SenderGuid = _Settings.Guid; - byte[] headerBytes = _MessageBuilder.GetHeaderBytes(msg); - await WatsonCommon.WriteMessageAsync(_DataStream, headerBytes, contentLength, stream, _Settings.StreamBufferSize, token).ConfigureAwait(false); - } + private async Task SendMessageAsync(WatsonMessage msg, long contentLength, Stream stream, CancellationToken token) + { + msg.SenderGuid = _Settings.Guid; + byte[] headerBytes = _MessageBuilder.GetHeaderBytes(msg); + await WatsonCommon.WriteMessageAsync(_DataStream, headerBytes, contentLength, stream, _Settings.StreamBufferSize, token).ConfigureAwait(false); + } #endregion @@ -1597,7 +1685,7 @@ private async Task IdleServerMonitor(CancellationToken token) catch (Exception e) { _Settings.Logger?.Invoke(Severity.Warn, _Header + "exception encountered while monitoring for idle server connection: " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + HandleException(e); } } } diff --git a/src/WatsonTcp/WatsonTcpClientSettings.cs b/src/WatsonTcp/WatsonTcpClientSettings.cs index ff661c7..e9e4a0a 100644 --- a/src/WatsonTcp/WatsonTcpClientSettings.cs +++ b/src/WatsonTcp/WatsonTcpClientSettings.cs @@ -165,35 +165,52 @@ public int LocalPort /// Headers larger than this value will be rejected to prevent memory exhaustion from malformed or malicious data. /// Value must be greater than 24. /// - public int MaxHeaderSize - { - get - { - return _MaxHeaderSize; + public int MaxHeaderSize + { + get + { + return _MaxHeaderSize; } set { if (value < 25) throw new ArgumentException("MaxHeaderSize must be greater than 24."); - _MaxHeaderSize = value; - } - } - - /// - /// Number of milliseconds to wait for a custom handshake to complete before failing the connection attempt. - /// Value must be greater than zero. - /// - public int HandshakeTimeoutMs - { - get - { - return _HandshakeTimeoutMs; - } - set - { - if (value < 1) throw new ArgumentException("HandshakeTimeoutMs must be greater than zero."); - _HandshakeTimeoutMs = value; - } - } + _MaxHeaderSize = value; + } + } + + /// + /// Number of milliseconds to wait for a custom handshake to complete before failing the connection attempt. + /// Value must be greater than zero. + /// + public int HandshakeTimeoutMs + { + get + { + return _HandshakeTimeoutMs; + } + set + { + if (value < 1) throw new ArgumentException("HandshakeTimeoutMs must be greater than zero."); + _HandshakeTimeoutMs = value; + } + } + + /// + /// Enable or disable emission of metrics into the WatsonTcp + /// . Default is true. + /// When true, the meter and its instruments are created; recording is a near-free no-op unless a + /// telemetry host subscribes to the meter by name. When false, no meter is created and there is + /// zero overhead. See and TELEMETRY.md. + /// + public bool EnableMetrics { get; set; } = true; + + /// + /// Enable or disable emission of distributed-tracing spans into the WatsonTcp + /// . Default is true. + /// When true, spans are created only if a tracing host subscribes to the activity source by name; + /// otherwise span creation is a near-free no-op. See and TELEMETRY.md. + /// + public bool EnableTracing { get; set; } = true; #endregion @@ -203,10 +220,10 @@ public int HandshakeTimeoutMs private int _MaxProxiedStreamSize = 67108864; private int _ConnectTimeoutSeconds = 5; private int _IdleServerTimeoutMs = 0; - private int _IdleServerEvaluationIntervalMs = 1000; - private int _LocalPort = 0; - private int _MaxHeaderSize = 262144; - private int _HandshakeTimeoutMs = 10000; + private int _IdleServerEvaluationIntervalMs = 1000; + private int _LocalPort = 0; + private int _MaxHeaderSize = 262144; + private int _HandshakeTimeoutMs = 10000; #endregion diff --git a/src/WatsonTcp/WatsonTcpInstrumentation.cs b/src/WatsonTcp/WatsonTcpInstrumentation.cs new file mode 100644 index 0000000..9972f03 --- /dev/null +++ b/src/WatsonTcp/WatsonTcpInstrumentation.cs @@ -0,0 +1,489 @@ +namespace WatsonTcp +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Diagnostics.Metrics; + using System.Net.Sockets; + + /// + /// Owns the and a single WatsonTcp client or + /// server records telemetry into, along with every instrument and span factory. One instance + /// lives per client/server and is disposed with its owner. All recording is fire-and-forget and + /// never throws into the caller's send, receive, or connection path. When metrics are disabled the + /// meter is never created (so no instruments are published); when tracing is disabled span factories + /// return null. + /// + internal sealed class WatsonTcpInstrumentation : IDisposable + { + #region Internal-Constants + + internal const string RoleServer = "server"; + internal const string RoleClient = "client"; + + internal const string ProtocolTcp = "tcp"; + internal const string ProtocolSsl = "ssl"; + + internal const string OutcomeAccepted = "accepted"; + internal const string OutcomeConnected = "connected"; + internal const string OutcomeRejectedMaxConnections = "rejected_maxconnections"; + internal const string OutcomeRejectedNotPermitted = "rejected_notpermitted"; + internal const string OutcomeRejectedBlocked = "rejected_blocked"; + internal const string OutcomeRejectedAuthorization = "rejected_authorization"; + internal const string OutcomeFailed = "failed"; + + internal const string OutcomeSuccess = "success"; + internal const string OutcomeFailure = "failure"; + internal const string OutcomeTimeout = "timeout"; + internal const string OutcomeCanceled = "canceled"; + + internal const string OutcomeCompleted = "completed"; + + internal const string MessageKindData = "data"; + internal const string MessageKindControl = "control"; + internal const string MessageKindSyncRequest = "sync_request"; + internal const string MessageKindSyncResponse = "sync_response"; + + internal const string KindRequest = "request"; + internal const string KindResponse = "response"; + + #endregion + + #region Private-Members + + // Latency histogram buckets, in seconds, mirroring Radiant's LatencyBuckets.Network preset. + private static readonly double[] _NetworkLatencyBuckets = + new double[] { 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0 }; + + private readonly string _Role; + private readonly string _Protocol; + private readonly bool _MetricsEnabled; + private readonly bool _TracingEnabled; + + private readonly Meter _Meter; + private readonly ActivitySource _ActivitySource; + private readonly KeyValuePair[] _BaseTagArray; + + private readonly Counter _MessagesSent; + private readonly Counter _MessagesReceived; + private readonly Counter _BytesSent; + private readonly Counter _BytesReceived; + private readonly Histogram _MessageSentSize; + private readonly Histogram _MessageReceivedSize; + private readonly Histogram _MessageSendDuration; + private readonly Counter _ConnectionsTotal; + private readonly Counter _DisconnectionsTotal; + private readonly Counter _HandshakesTotal; + private readonly Histogram _HandshakeDuration; + private readonly Counter _AuthenticationsTotal; + private readonly Counter _AuthorizationsTotal; + private readonly Counter _SyncRequestsSent; + private readonly Counter _SyncResponsesReceived; + private readonly Histogram _SyncDuration; + private readonly Counter _SyncExpired; + private readonly Counter _ExceptionsTotal; + private readonly Counter _ListenerTransientErrors; + private readonly Counter _StreamDrainedBytes; + + #endregion + + #region Constructors-and-Factories + + /// + /// Instantiate. + /// + /// Emitting side; one of or . + /// Transport; one of or . + /// When false, no meter or instruments are created. + /// When false, span factories return null. + /// Callback returning the current live connection count, or null to omit that gauge. + /// Callback returning the current pending connection count, or null to omit that gauge. + /// Callback returning the current in-flight synchronous request count, or null to omit that gauge. + /// Callback returning seconds since start, or null to omit that gauge. + internal WatsonTcpInstrumentation( + string role, + string protocol, + bool metricsEnabled, + bool tracingEnabled, + Func activeConnectionsCallback, + Func pendingConnectionsCallback, + Func pendingSyncRequestsCallback, + Func uptimeSecondsCallback) + { + if (String.IsNullOrEmpty(role)) throw new ArgumentNullException(nameof(role)); + if (String.IsNullOrEmpty(protocol)) throw new ArgumentNullException(nameof(protocol)); + + _Role = role; + _Protocol = protocol; + _MetricsEnabled = metricsEnabled; + _TracingEnabled = tracingEnabled; + + _BaseTagArray = new KeyValuePair[] + { + new KeyValuePair(WatsonTcpMetrics.TagRole, _Role), + new KeyValuePair(WatsonTcpMetrics.TagProtocol, _Protocol) + }; + + if (_TracingEnabled) + { + _ActivitySource = new ActivitySource(WatsonTcpMetrics.ActivitySourceName); + } + + if (!_MetricsEnabled) return; + + _Meter = new Meter(WatsonTcpMetrics.MeterName); + + _MessagesSent = _Meter.CreateCounter(WatsonTcpMetrics.MessagesSent, WatsonTcpMetrics.UnitMessage, "Messages written to the wire."); + _MessagesReceived = _Meter.CreateCounter(WatsonTcpMetrics.MessagesReceived, WatsonTcpMetrics.UnitMessage, "Messages read from the wire."); + _BytesSent = _Meter.CreateCounter(WatsonTcpMetrics.BytesSent, WatsonTcpMetrics.UnitBytes, "Payload bytes sent."); + _BytesReceived = _Meter.CreateCounter(WatsonTcpMetrics.BytesReceived, WatsonTcpMetrics.UnitBytes, "Payload bytes received."); + _MessageSentSize = _Meter.CreateHistogram(WatsonTcpMetrics.MessageSentSize, WatsonTcpMetrics.UnitBytes, "Distribution of sent message sizes."); + _MessageReceivedSize = _Meter.CreateHistogram(WatsonTcpMetrics.MessageReceivedSize, WatsonTcpMetrics.UnitBytes, "Distribution of received message sizes."); + _MessageSendDuration = _Meter.CreateHistogram(WatsonTcpMetrics.MessageSendDuration, WatsonTcpMetrics.UnitSeconds, "Time to write a message to the transport stream."); + _ConnectionsTotal = _Meter.CreateCounter(WatsonTcpMetrics.ConnectionsTotal, WatsonTcpMetrics.UnitConnection, "Connection admission outcomes."); + _DisconnectionsTotal = _Meter.CreateCounter(WatsonTcpMetrics.DisconnectionsTotal, WatsonTcpMetrics.UnitConnection, "Disconnections by reason."); + _HandshakesTotal = _Meter.CreateCounter(WatsonTcpMetrics.HandshakesTotal, WatsonTcpMetrics.UnitHandshake, "Custom-handshake completions."); + _HandshakeDuration = _Meter.CreateHistogram(WatsonTcpMetrics.HandshakeDuration, WatsonTcpMetrics.UnitSeconds, "Custom-handshake duration."); + _AuthenticationsTotal = _Meter.CreateCounter(WatsonTcpMetrics.AuthenticationsTotal, WatsonTcpMetrics.UnitAuthentication, "Preshared-key authentication results."); + _AuthorizationsTotal = _Meter.CreateCounter(WatsonTcpMetrics.AuthorizationsTotal, WatsonTcpMetrics.UnitAuthorization, "Connection-authorization results."); + _SyncRequestsSent = _Meter.CreateCounter(WatsonTcpMetrics.SyncRequestsSent, WatsonTcpMetrics.UnitRequest, "Synchronous requests issued."); + _SyncResponsesReceived = _Meter.CreateCounter(WatsonTcpMetrics.SyncResponsesReceived, WatsonTcpMetrics.UnitResponse, "Synchronous responses matched to a request."); + _SyncDuration = _Meter.CreateHistogram(WatsonTcpMetrics.SyncDuration, WatsonTcpMetrics.UnitSeconds, "Synchronous round-trip duration."); + _SyncExpired = _Meter.CreateCounter(WatsonTcpMetrics.SyncExpired, WatsonTcpMetrics.UnitMessage, "Expired synchronous requests or responses discarded."); + _ExceptionsTotal = _Meter.CreateCounter(WatsonTcpMetrics.ExceptionsTotal, WatsonTcpMetrics.UnitException, "Exceptions surfaced through ExceptionEncountered."); + _ListenerTransientErrors = _Meter.CreateCounter(WatsonTcpMetrics.ListenerTransientErrors, WatsonTcpMetrics.UnitError, "Recovered transient accept-loop socket errors."); + _StreamDrainedBytes = _Meter.CreateCounter(WatsonTcpMetrics.StreamDrainedBytes, WatsonTcpMetrics.UnitBytes, "Unread stream-payload bytes drained after a handler."); + + if (activeConnectionsCallback != null) + { + _Meter.CreateObservableGauge( + WatsonTcpMetrics.ConnectionsActive, + () => new Measurement(SafeSampleLong(activeConnectionsCallback), _BaseTagArray), + WatsonTcpMetrics.UnitConnection, + "Current live connections."); + } + + if (pendingConnectionsCallback != null) + { + _Meter.CreateObservableGauge( + WatsonTcpMetrics.ConnectionsPending, + () => new Measurement(SafeSampleLong(pendingConnectionsCallback), _BaseTagArray), + WatsonTcpMetrics.UnitConnection, + "Accepted-but-not-yet-admitted connections."); + } + + if (pendingSyncRequestsCallback != null) + { + _Meter.CreateObservableGauge( + WatsonTcpMetrics.SyncPending, + () => new Measurement(SafeSampleLong(pendingSyncRequestsCallback), _BaseTagArray), + WatsonTcpMetrics.UnitRequest, + "In-flight synchronous conversations."); + } + + if (uptimeSecondsCallback != null) + { + _Meter.CreateObservableGauge( + WatsonTcpMetrics.Uptime, + () => new Measurement(SafeSampleDouble(uptimeSecondsCallback), _BaseTagArray), + WatsonTcpMetrics.UnitSeconds, + "Seconds since the client or server started."); + } + } + + #endregion + + #region Internal-Methods + + /// + /// Returns fractional seconds elapsed since a reading. + /// + internal static double SecondsSince(long startTimestamp) + { + long now = Stopwatch.GetTimestamp(); + if (now <= startTimestamp) return 0.0; + return (now - startTimestamp) / (double)Stopwatch.Frequency; + } + + internal void MessageSent(WatsonMessage msg, long bytes, long sendStartTimestamp) + { + if (!_MetricsEnabled) return; + + TagList tags = KindTags(msg); + _MessagesSent.Add(1, tags); + if (bytes > 0) + { + _BytesSent.Add(bytes, BaseTags()); + _MessageSentSize.Record(bytes, BaseTags()); + } + + _MessageSendDuration.Record(SecondsSince(sendStartTimestamp), BaseTags()); + } + + internal void MessageReceived(WatsonMessage msg, long bytes) + { + if (!_MetricsEnabled) return; + + TagList tags = KindTags(msg); + _MessagesReceived.Add(1, tags); + if (bytes > 0) + { + _BytesReceived.Add(bytes, BaseTags()); + _MessageReceivedSize.Record(bytes, BaseTags()); + } + } + + internal void ConnectionOutcome(string outcome) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagOutcome, outcome); + _ConnectionsTotal.Add(1, tags); + } + + internal void Disconnection(DisconnectReason reason) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagReason, reason.ToString()); + _DisconnectionsTotal.Add(1, tags); + } + + internal void HandshakeCompleted(string outcome, long handshakeStartTimestamp) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagOutcome, outcome); + _HandshakesTotal.Add(1, tags); + _HandshakeDuration.Record(SecondsSince(handshakeStartTimestamp), tags); + } + + internal void Authentication(string outcome) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagOutcome, outcome); + _AuthenticationsTotal.Add(1, tags); + } + + internal void Authorization(string outcome) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagOutcome, outcome); + _AuthorizationsTotal.Add(1, tags); + } + + internal void SyncRequestSent() + { + if (!_MetricsEnabled) return; + _SyncRequestsSent.Add(1, BaseTags()); + } + + internal void SyncResponseReceived() + { + if (!_MetricsEnabled) return; + _SyncResponsesReceived.Add(1, BaseTags()); + } + + internal void SyncCompleted(string outcome, long syncStartTimestamp) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagOutcome, outcome); + _SyncDuration.Record(SecondsSince(syncStartTimestamp), tags); + } + + internal void SyncExpiredMessage(string kind) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagKind, kind); + _SyncExpired.Add(1, tags); + } + + internal void ExceptionRecorded(Exception e) + { + if (!_MetricsEnabled || e == null) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagExceptionType, e.GetType().Name); + _ExceptionsTotal.Add(1, tags); + } + + internal void TransientAcceptError(SocketError error) + { + if (!_MetricsEnabled) return; + + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagSocketError, error.ToString()); + _ListenerTransientErrors.Add(1, tags); + } + + internal void StreamDrained(long bytes) + { + if (!_MetricsEnabled || bytes <= 0) return; + _StreamDrainedBytes.Add(bytes, BaseTags()); + } + + internal Activity StartConnectSpan(string serverAddress, int serverPort) + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanConnect, ActivityKind.Client); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagServerAddress, serverAddress); + activity.SetTag(WatsonTcpMetrics.TagServerPort, serverPort); + activity.SetTag(WatsonTcpMetrics.TagProtocol, _Protocol); + } + + return activity; + } + + internal Activity StartSessionSpan(string clientAddress, Guid clientGuid) + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanSession, ActivityKind.Server); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagClientAddress, clientAddress); + activity.SetTag(WatsonTcpMetrics.TagClientGuid, clientGuid.ToString()); + activity.SetTag(WatsonTcpMetrics.TagProtocol, _Protocol); + } + + return activity; + } + + internal Activity StartHandshakeSpan() + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanHandshake, ActivityKind.Internal); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagRole, _Role); + } + + return activity; + } + + internal Activity StartSendSpan(long bytes, bool sync, Guid? clientGuid) + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanSend, ActivityKind.Producer); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagMessageBytes, bytes); + activity.SetTag(WatsonTcpMetrics.TagMessageSync, sync); + if (clientGuid.HasValue) activity.SetTag(WatsonTcpMetrics.TagClientGuid, clientGuid.Value.ToString()); + } + + return activity; + } + + internal Activity StartReceiveSpan(long bytes, Guid? clientGuid) + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanReceive, ActivityKind.Consumer); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagMessageBytes, bytes); + if (clientGuid.HasValue) activity.SetTag(WatsonTcpMetrics.TagClientGuid, clientGuid.Value.ToString()); + } + + return activity; + } + + internal Activity StartSyncSpan(Guid conversationGuid, long bytes) + { + if (!_TracingEnabled) return null; + + Activity activity = _ActivitySource.StartActivity(WatsonTcpMetrics.SpanSync, ActivityKind.Client); + if (activity != null) + { + activity.SetTag(WatsonTcpMetrics.TagConversationGuid, conversationGuid.ToString()); + activity.SetTag(WatsonTcpMetrics.TagMessageBytes, bytes); + } + + return activity; + } + + #endregion + + #region IDisposable + + /// + /// Dispose of the underlying meter and activity source. + /// + public void Dispose() + { + _ActivitySource?.Dispose(); + _Meter?.Dispose(); + } + + #endregion + + #region Private-Methods + + private TagList BaseTags() + { + TagList tags = new TagList(); + tags.Add(WatsonTcpMetrics.TagRole, _Role); + tags.Add(WatsonTcpMetrics.TagProtocol, _Protocol); + return tags; + } + + private TagList KindTags(WatsonMessage msg) + { + TagList tags = BaseTags(); + tags.Add(WatsonTcpMetrics.TagMessageKind, ClassifyMessage(msg)); + return tags; + } + + private static string ClassifyMessage(WatsonMessage msg) + { + if (msg == null) return MessageKindData; + if (msg.SyncRequest) return MessageKindSyncRequest; + if (msg.SyncResponse) return MessageKindSyncResponse; + if (msg.Status != MessageStatus.Normal) return MessageKindControl; + return MessageKindData; + } + + private static long SafeSampleLong(Func callback) + { + try + { + return callback(); + } + catch (Exception) + { + return 0L; + } + } + + private static double SafeSampleDouble(Func callback) + { + try + { + return callback(); + } + catch (Exception) + { + return 0.0; + } + } + + #endregion + } +} diff --git a/src/WatsonTcp/WatsonTcpMetrics.cs b/src/WatsonTcp/WatsonTcpMetrics.cs new file mode 100644 index 0000000..fc488b9 --- /dev/null +++ b/src/WatsonTcp/WatsonTcpMetrics.cs @@ -0,0 +1,358 @@ +namespace WatsonTcp +{ + /// + /// Well-known names, units, and tag keys that WatsonTcp emits telemetry under. + /// + /// WatsonTcp publishes metrics to a and + /// distributed-tracing spans to a , both named + /// ("WatsonTcp"). These names are the public contract between + /// WatsonTcp and any telemetry host (Radiant, the OpenTelemetry SDK, Prometheus, and others). + /// They are stable across releases; treat them like public API. + /// + /// + /// A host observes WatsonTcp by subscribing to the meter and activity source by name, for + /// example MeterProviderBuilder.AddMeter(WatsonTcpMetrics.MeterName) or, for a Radiant + /// host, settings.Sources.AddMeter(WatsonTcpMetrics.MeterName). Metric names are dotted + /// and lowercase and units are UCUM strings, so the OpenTelemetry Prometheus exporter produces + /// conventional series names automatically. + /// + /// + public static class WatsonTcpMetrics + { + #region Sources + + /// + /// Name of the WatsonTcp records all metrics + /// into, and of the WatsonTcp starts all + /// spans from. Value is "WatsonTcp". Stable across releases. + /// + public const string MeterName = "WatsonTcp"; + + /// + /// Name of the WatsonTcp starts spans from. + /// Identical to ("WatsonTcp") so a single subscription string + /// covers both metrics and traces. + /// + public const string ActivitySourceName = "WatsonTcp"; + + #endregion + + #region Units + + /// + /// UCUM unit for a count of messages ("{message}"). Annotation units contribute no + /// Prometheus suffix. + /// + public const string UnitMessage = "{message}"; + + /// + /// UCUM unit for a count of connections ("{connection}"). + /// + public const string UnitConnection = "{connection}"; + + /// + /// UCUM unit for a count of handshakes ("{handshake}"). + /// + public const string UnitHandshake = "{handshake}"; + + /// + /// UCUM unit for a count of authentications ("{authentication}"). + /// + public const string UnitAuthentication = "{authentication}"; + + /// + /// UCUM unit for a count of authorizations ("{authorization}"). + /// + public const string UnitAuthorization = "{authorization}"; + + /// + /// UCUM unit for a count of synchronous requests ("{request}"). + /// + public const string UnitRequest = "{request}"; + + /// + /// UCUM unit for a count of synchronous responses ("{response}"). + /// + public const string UnitResponse = "{response}"; + + /// + /// UCUM unit for a count of exceptions ("{exception}"). + /// + public const string UnitException = "{exception}"; + + /// + /// UCUM unit for a count of errors ("{error}"). + /// + public const string UnitError = "{error}"; + + /// + /// UCUM unit for bytes ("By"). The Prometheus exporter appends a _bytes suffix. + /// + public const string UnitBytes = "By"; + + /// + /// UCUM unit for seconds ("s"). The Prometheus exporter appends a _seconds suffix. + /// + public const string UnitSeconds = "s"; + + #endregion + + #region Metric-Names + + /// + /// Counter of messages written to the wire. Unit . + /// + public const string MessagesSent = "watsontcp.messages.sent"; + + /// + /// Counter of messages read from the wire. Unit . + /// + public const string MessagesReceived = "watsontcp.messages.received"; + + /// + /// Counter of payload bytes sent. Unit . + /// + public const string BytesSent = "watsontcp.bytes.sent"; + + /// + /// Counter of payload bytes received. Unit . + /// + public const string BytesReceived = "watsontcp.bytes.received"; + + /// + /// Histogram of sent message sizes. Unit . + /// + public const string MessageSentSize = "watsontcp.message.sent.size"; + + /// + /// Histogram of received message sizes. Unit . + /// + public const string MessageReceivedSize = "watsontcp.message.received.size"; + + /// + /// Histogram of the time taken to write a message header and payload to the transport stream. + /// Unit . + /// + public const string MessageSendDuration = "watsontcp.message.send.duration"; + + /// + /// Observable gauge of currently live connections. Unit . + /// + public const string ConnectionsActive = "watsontcp.connections.active"; + + /// + /// Observable gauge of accepted-but-not-yet-admitted connections (server only). + /// Unit . + /// + public const string ConnectionsPending = "watsontcp.connections.pending"; + + /// + /// Counter of connection admission outcomes, dimensioned by . + /// Unit . + /// + public const string ConnectionsTotal = "watsontcp.connections.total"; + + /// + /// Counter of disconnections, dimensioned by . + /// Unit . + /// + public const string DisconnectionsTotal = "watsontcp.disconnections.total"; + + /// + /// Counter of custom-handshake completions, dimensioned by . + /// Unit . + /// + public const string HandshakesTotal = "watsontcp.handshakes.total"; + + /// + /// Histogram of custom-handshake duration, dimensioned by . + /// Unit . + /// + public const string HandshakeDuration = "watsontcp.handshake.duration"; + + /// + /// Counter of preshared-key authentication results, dimensioned by . + /// Unit . + /// + public const string AuthenticationsTotal = "watsontcp.authentications.total"; + + /// + /// Counter of connection-authorization results, dimensioned by . + /// Unit . + /// + public const string AuthorizationsTotal = "watsontcp.authorizations.total"; + + /// + /// Counter of synchronous requests issued through SendAndWaitAsync. Unit . + /// + public const string SyncRequestsSent = "watsontcp.sync.requests.sent"; + + /// + /// Counter of synchronous responses matched to an outstanding request. Unit . + /// + public const string SyncResponsesReceived = "watsontcp.sync.responses.received"; + + /// + /// Histogram of synchronous round-trip duration, dimensioned by + /// (completed or timeout). Unit . + /// + public const string SyncDuration = "watsontcp.sync.duration"; + + /// + /// Observable gauge of in-flight synchronous conversations. Unit . + /// + public const string SyncPending = "watsontcp.sync.pending"; + + /// + /// Counter of expired synchronous requests or responses that were discarded, dimensioned by + /// (request or response). Unit . + /// + public const string SyncExpired = "watsontcp.sync.expired"; + + /// + /// Counter of exceptions surfaced through the ExceptionEncountered event, dimensioned by + /// . Unit . + /// + public const string ExceptionsTotal = "watsontcp.exceptions.total"; + + /// + /// Counter of recovered transient accept-loop socket errors, dimensioned by + /// . Unit . + /// + public const string ListenerTransientErrors = "watsontcp.listener.transient_errors"; + + /// + /// Counter of unread stream-payload bytes drained after a receive handler returned. + /// Unit . + /// + public const string StreamDrainedBytes = "watsontcp.stream.drained_bytes"; + + /// + /// Observable gauge of seconds elapsed since the client or server started. Unit . + /// + public const string Uptime = "watsontcp.uptime"; + + #endregion + + #region Span-Names + + /// + /// Client span covering a connection attempt through registration and handshake. + /// + public const string SpanConnect = "watsontcp.connect"; + + /// + /// Server span covering the lifetime of an admitted client session. + /// + public const string SpanSession = "watsontcp.session"; + + /// + /// Span covering a custom handshake exchange. + /// + public const string SpanHandshake = "watsontcp.handshake"; + + /// + /// Span covering a single message send. + /// + public const string SpanSend = "watsontcp.send"; + + /// + /// Span covering the processing of a single received data message. + /// + public const string SpanReceive = "watsontcp.receive"; + + /// + /// Span covering a synchronous request/response round trip. + /// + public const string SpanSync = "watsontcp.sync"; + + #endregion + + #region Metric-Tag-Keys + + /// + /// Metric tag key naming which side emitted the measurement. Values: server, client. + /// + public const string TagRole = "role"; + + /// + /// Metric tag key naming the transport. Values: tcp, ssl. + /// + public const string TagProtocol = "protocol"; + + /// + /// Metric tag key naming the outcome of a connection, handshake, authentication, authorization, + /// or synchronous operation. + /// + public const string TagOutcome = "outcome"; + + /// + /// Metric tag key naming the reason for a disconnection. Values mirror the + /// enum member names. + /// + public const string TagReason = "reason"; + + /// + /// Metric tag key naming a message classification. Values: data, control, + /// sync_request, sync_response. + /// + public const string TagMessageKind = "message.kind"; + + /// + /// Metric tag key naming an expired-synchronous discard classification. Values: request, + /// response. + /// + public const string TagKind = "kind"; + + /// + /// Metric tag key naming the short type name of an exception. + /// + public const string TagExceptionType = "exception.type"; + + /// + /// Metric tag key naming a value. + /// + public const string TagSocketError = "socket.error"; + + #endregion + + #region Span-Tag-Keys + + /// + /// Span tag key naming the remote server address (client spans). + /// + public const string TagServerAddress = "server.address"; + + /// + /// Span tag key naming the remote server port (client spans). + /// + public const string TagServerPort = "server.port"; + + /// + /// Span tag key naming the remote client address (server spans). + /// + public const string TagClientAddress = "client.address"; + + /// + /// Span tag key naming the WatsonTcp client GUID (server spans). High cardinality; spans only. + /// + public const string TagClientGuid = "client.guid"; + + /// + /// Span tag key naming a synchronous conversation GUID. High cardinality; spans only. + /// + public const string TagConversationGuid = "conversation.guid"; + + /// + /// Span tag key naming a message payload size in bytes. + /// + public const string TagMessageBytes = "message.bytes"; + + /// + /// Span tag key indicating whether a message is a synchronous request. + /// + public const string TagMessageSync = "message.sync"; + + #endregion + } +} diff --git a/src/WatsonTcp/WatsonTcpServer.cs b/src/WatsonTcp/WatsonTcpServer.cs index 23aa8ef..f31d7a6 100644 --- a/src/WatsonTcp/WatsonTcpServer.cs +++ b/src/WatsonTcp/WatsonTcpServer.cs @@ -1,13 +1,14 @@ namespace WatsonTcp { using System; - using System.Buffers; - using System.Collections.Concurrent; - using System.Collections.Generic; - using System.IO; - using System.Net; - using System.Net.Security; - using System.Net.Sockets; + using System.Buffers; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Net; + using System.Net.Security; + using System.Net.Sockets; #if NET5_0_OR_GREATER using System.Runtime.InteropServices; #endif @@ -134,24 +135,24 @@ public ISerializationHelper SerializationHelper /// /// Retrieve the number of current connected clients. /// - public int Connections - { - get - { - return _Connections; - } - } - - /// - /// Retrieve the number of pending clients that have not yet completed admission. - /// - public int PendingConnections - { - get - { - return _ClientManager?.PendingClientCount() ?? 0; - } - } + public int Connections + { + get + { + return _Connections; + } + } + + /// + /// Retrieve the number of pending clients that have not yet completed admission. + /// + public int PendingConnections + { + get + { + return _ClientManager?.PendingClientCount() ?? 0; + } + } /// /// Flag to indicate if Watson TCP is listening for incoming TCP connections. @@ -178,11 +179,12 @@ public bool IsListening private WatsonTcpServerSslConfiguration _SslConfiguration = new WatsonTcpServerSslConfiguration(); private ClientMetadataManager _ClientManager = new ClientMetadataManager(); private ISerializationHelper _SerializationHelper = new DefaultSerializationHelper(); + private WatsonTcpInstrumentation _Instrumentation = null; private int _Connections = 0; - private bool _IsListening = false; - private HashSet _PermittedIpsSnapshot = null; - private HashSet _BlockedIpsSnapshot = null; + private bool _IsListening = false; + private HashSet _PermittedIpsSnapshot = null; + private HashSet _BlockedIpsSnapshot = null; private Mode _Mode; private TlsVersion _TlsVersion = TlsVersion.Tls12; @@ -382,7 +384,22 @@ public void Start() _Statistics = new WatsonTcpStatistics(); _Listener = new TcpListener(_ListenerIpAddress, _ListenerPort); - ValidateReceiveHandlerConfiguration(); + _Instrumentation?.Dispose(); + _Instrumentation = null; + if (_Settings.EnableMetrics || _Settings.EnableTracing) + { + _Instrumentation = new WatsonTcpInstrumentation( + WatsonTcpInstrumentation.RoleServer, + _Mode == Mode.Ssl ? WatsonTcpInstrumentation.ProtocolSsl : WatsonTcpInstrumentation.ProtocolTcp, + _Settings.EnableMetrics, + _Settings.EnableTracing, + () => _Connections, + () => _ClientManager?.PendingClientCount() ?? 0, + () => _SyncRequests.Count, + () => _Statistics != null ? _Statistics.UpTime.TotalSeconds : 0.0); + } + + ValidateReceiveHandlerConfiguration(); if (_Mode == Mode.Tcp) { @@ -397,14 +414,14 @@ public void Start() throw new ArgumentException("Unknown mode: " + _Mode.ToString()); } - _MessageBuilder.MaxHeaderSize = _Settings.MaxHeaderSize; - _MessageBuilder.ReadStreamBuffer = _Settings.StreamBufferSize; - _PermittedIpsSnapshot = _Settings.PermittedIPs.Count > 0 ? new HashSet(_Settings.PermittedIPs) : null; - _BlockedIpsSnapshot = _Settings.BlockedIPs.Count > 0 ? new HashSet(_Settings.BlockedIPs) : null; - _Listener.Start(); - _AcceptConnections = AcceptConnections(_Token); // sets _IsListening - _MonitorClients = MonitorForIdleClients(_Token); - _Events.HandleServerStarted(this, EventArgs.Empty); + _MessageBuilder.MaxHeaderSize = _Settings.MaxHeaderSize; + _MessageBuilder.ReadStreamBuffer = _Settings.StreamBufferSize; + _PermittedIpsSnapshot = _Settings.PermittedIPs.Count > 0 ? new HashSet(_Settings.PermittedIPs) : null; + _BlockedIpsSnapshot = _Settings.BlockedIPs.Count > 0 ? new HashSet(_Settings.BlockedIPs) : null; + _Listener.Start(); + _AcceptConnections = AcceptConnections(_Token); // sets _IsListening + _MonitorClients = MonitorForIdleClients(_Token); + _Events.HandleServerStarted(this, EventArgs.Empty); } /// @@ -560,13 +577,13 @@ public bool IsClientConnected(Guid guid) /// Retrieve the client metadata associated with each connected client. /// /// An enumerable collection of client metadata. - public IEnumerable ListClients() - { - foreach (ClientMetadata client in _ClientManager.EnumerateClients()) - { - yield return client; - } - } + public IEnumerable ListClients() + { + foreach (ClientMetadata client in _ClientManager.EnumerateClients()) + { + yield return client; + } + } /// /// Disconnects the specified client. @@ -575,29 +592,29 @@ public IEnumerable ListClients() /// Reason for the disconnect. This is conveyed to the client. /// Flag to indicate whether the client should be notified of the disconnect. This message will not be sent until other send requests have been handled. /// Cancellation token to cancel the request. - public async Task DisconnectClientAsync(Guid guid, MessageStatus status = MessageStatus.Removed, bool sendNotice = true, CancellationToken token = default) - { - ClientMetadata client = _ClientManager.GetTrackedClient(guid); - if (client == null) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "unable to find client " + guid.ToString()); - } - else - { - if (!_ClientManager.ExistsClientTimedout(guid)) _ClientManager.AddClientKicked(guid); - - if (sendNotice) - { - WatsonMessage removeMsg = new WatsonMessage(); - removeMsg.Status = status; - await SendInternalAsync(client, removeMsg, 0, null, token).ConfigureAwait(false); - } - - client.Phase = ConnectionPhase.Disconnected; - client.Dispose(); - _ClientManager.Remove(guid); - } - } + public async Task DisconnectClientAsync(Guid guid, MessageStatus status = MessageStatus.Removed, bool sendNotice = true, CancellationToken token = default) + { + ClientMetadata client = _ClientManager.GetTrackedClient(guid); + if (client == null) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "unable to find client " + guid.ToString()); + } + else + { + if (!_ClientManager.ExistsClientTimedout(guid)) _ClientManager.AddClientKicked(guid); + + if (sendNotice) + { + WatsonMessage removeMsg = new WatsonMessage(); + removeMsg.Status = status; + await SendInternalAsync(client, removeMsg, 0, null, token).ConfigureAwait(false); + } + + client.Phase = ConnectionPhase.Disconnected; + client.Dispose(); + _ClientManager.Remove(guid); + } + } /// /// Disconnects all connected clients. @@ -605,18 +622,18 @@ public async Task DisconnectClientAsync(Guid guid, MessageStatus status = Messag /// Reason for the disconnect. This is conveyed to each client. /// Flag to indicate whether the client should be notified of the disconnect. This message will not be sent until other send requests have been handled. /// Cancellation token to cancel the request. - public async Task DisconnectClientsAsync(MessageStatus status = MessageStatus.Removed, bool sendNotice = true, CancellationToken token = default) - { - foreach (ClientMetadata client in _ClientManager.EnumerateClients()) - { - await DisconnectClientAsync(client.Guid, status, sendNotice, token).ConfigureAwait(false); - } - - foreach (ClientMetadata client in _ClientManager.EnumeratePendingClients()) - { - await DisconnectClientAsync(client.Guid, status, sendNotice, token).ConfigureAwait(false); - } - } + public async Task DisconnectClientsAsync(MessageStatus status = MessageStatus.Removed, bool sendNotice = true, CancellationToken token = default) + { + foreach (ClientMetadata client in _ClientManager.EnumerateClients()) + { + await DisconnectClientAsync(client.Guid, status, sendNotice, token).ConfigureAwait(false); + } + + foreach (ClientMetadata client in _ClientManager.EnumeratePendingClients()) + { + await DisconnectClientAsync(client.Guid, status, sendNotice, token).ConfigureAwait(false); + } + } #endregion @@ -656,6 +673,12 @@ protected virtual void Dispose(bool disposing) _ClientManager.Dispose(); } + if (_Instrumentation != null) + { + _Instrumentation.Dispose(); + _Instrumentation = null; + } + Settings = null; _Events = null; _Callbacks = null; @@ -680,10 +703,10 @@ protected virtual void Dispose(bool disposing) #region Connection - private void EnableKeepalives(TcpClient client) - { - // issues with definitions: https://github.com/dotnet/sdk/issues/14540 - + private void EnableKeepalives(TcpClient client) + { + // issues with definitions: https://github.com/dotnet/sdk/issues/14540 + try { #if NET6_0_OR_GREATER @@ -717,28 +740,28 @@ private void EnableKeepalives(TcpClient client) catch (Exception) { _Settings.Logger?.Invoke(Severity.Error, _Header + "keepalives not supported on this platform, disabled"); - _Keepalive.EnableTcpKeepAlives = false; - } - } - - /// - /// Accept a TCP client from the listener. - /// - /// Cancellation token used to stop accepting clients. - /// The accepted TCP client. - protected virtual async Task AcceptTcpClientAsync(CancellationToken token) - { -#if NET6_0_OR_GREATER - return await _Listener.AcceptTcpClientAsync(token).ConfigureAwait(false); -#else - _ = token; - return await _Listener.AcceptTcpClientAsync().ConfigureAwait(false); -#endif - } - - private async Task AcceptConnections(CancellationToken token) - { - _IsListening = true; + _Keepalive.EnableTcpKeepAlives = false; + } + } + + /// + /// Accept a TCP client from the listener. + /// + /// Cancellation token used to stop accepting clients. + /// The accepted TCP client. + protected virtual async Task AcceptTcpClientAsync(CancellationToken token) + { +#if NET6_0_OR_GREATER + return await _Listener.AcceptTcpClientAsync(token).ConfigureAwait(false); +#else + _ = token; + return await _Listener.AcceptTcpClientAsync().ConfigureAwait(false); +#endif + } + + private async Task AcceptConnections(CancellationToken token) + { + _IsListening = true; while (true) { @@ -759,29 +782,31 @@ private async Task AcceptConnections(CancellationToken token) _IsListening = true; } - #endregion - - #region Accept-and-Validate - - TcpClient tcpClient; - try - { - tcpClient = await AcceptTcpClientAsync(token).ConfigureAwait(false); - } - catch (SocketException e) when (_IsListening && !token.IsCancellationRequested && IsTransientAcceptSocketException(e)) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "transient listener exception while accepting connection, continuing: " + e.SocketErrorCode + " (" + e.Message + ")"); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - continue; - } - - tcpClient.LingerState.Enabled = false; - tcpClient.NoDelay = _Settings.NoDelay; + #endregion + + #region Accept-and-Validate + + TcpClient tcpClient; + try + { + tcpClient = await AcceptTcpClientAsync(token).ConfigureAwait(false); + } + catch (SocketException e) when (_IsListening && !token.IsCancellationRequested && IsTransientAcceptSocketException(e)) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "transient listener exception while accepting connection, continuing: " + e.SocketErrorCode + " (" + e.Message + ")"); + _Instrumentation?.TransientAcceptError(e.SocketErrorCode); + HandleException(e); + continue; + } + + tcpClient.LingerState.Enabled = false; + tcpClient.NoDelay = _Settings.NoDelay; // Enforce max connections - reject if at capacity if (_Connections >= _Settings.MaxConnections && _Settings.EnforceMaxConnections) { _Settings.Logger?.Invoke(Severity.Info, _Header + "rejecting connection, maximum connections " + _Settings.MaxConnections + " reached (currently " + _Connections + " connections)"); + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeRejectedMaxConnections); tcpClient.Close(); continue; } @@ -789,24 +814,26 @@ private async Task AcceptConnections(CancellationToken token) if (_Keepalive.EnableTcpKeepAlives) EnableKeepalives(tcpClient); string clientIp = ((IPEndPoint)tcpClient.Client.RemoteEndPoint).Address.ToString(); - if (_PermittedIpsSnapshot != null && !_PermittedIpsSnapshot.Contains(clientIp)) - { - _Settings.Logger?.Invoke(Severity.Info, _Header + "rejecting connection from " + clientIp + " (not permitted)"); - tcpClient.Close(); - continue; - } - - if (_BlockedIpsSnapshot != null && _BlockedIpsSnapshot.Contains(clientIp)) - { - _Settings.Logger?.Invoke(Severity.Info, _Header + "rejecting connection from " + clientIp + " (blocked)"); - tcpClient.Close(); + if (_PermittedIpsSnapshot != null && !_PermittedIpsSnapshot.Contains(clientIp)) + { + _Settings.Logger?.Invoke(Severity.Info, _Header + "rejecting connection from " + clientIp + " (not permitted)"); + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeRejectedNotPermitted); + tcpClient.Close(); continue; - } - - ClientMetadata client = new ClientMetadata(tcpClient); - _ClientManager.AddPendingClient(client.Guid, client); - - CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(_Token, client.Token); + } + + if (_BlockedIpsSnapshot != null && _BlockedIpsSnapshot.Contains(clientIp)) + { + _Settings.Logger?.Invoke(Severity.Info, _Header + "rejecting connection from " + clientIp + " (blocked)"); + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeRejectedBlocked); + tcpClient.Close(); + continue; + } + + ClientMetadata client = new ClientMetadata(tcpClient); + _ClientManager.AddPendingClient(client.Guid, client); + + CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(_Token, client.Token); #endregion @@ -828,14 +855,14 @@ private async Task AcceptConnections(CancellationToken token) #region Initialize-Client - if (_Mode == Mode.Tcp) - { - _ = ProcessAcceptedClientAsync(client, linkedCts.Token); - } - else if (_Mode == Mode.Ssl) - { - if (_Settings.AcceptInvalidCertificates) - { + if (_Mode == Mode.Tcp) + { + _ = ProcessAcceptedClientAsync(client, linkedCts.Token); + } + else if (_Mode == Mode.Ssl) + { + if (_Settings.AcceptInvalidCertificates) + { client.SslStream = new SslStream(client.NetworkStream, false, _SslConfiguration.ClientCertificateValidationCallback); } else @@ -843,10 +870,10 @@ private async Task AcceptConnections(CancellationToken token) client.SslStream = new SslStream(client.NetworkStream, false); } - _ = Task.Run(() => InitializeAcceptedSslClientAsync(client, linkedCts.Token), linkedCts.Token); - } - else - { + _ = Task.Run(() => InitializeAcceptedSslClientAsync(client, linkedCts.Token), linkedCts.Token); + } + else + { throw new ArgumentException("Unknown mode: " + _Mode.ToString()); } @@ -858,461 +885,495 @@ private async Task AcceptConnections(CancellationToken token) { break; } - catch (ObjectDisposedException) - { - break; - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "listener exception: " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + catch (ObjectDisposedException) + { break; } - } - } - - private static bool IsTransientAcceptSocketException(SocketException e) - { - return e != null - && (e.SocketErrorCode == SocketError.ConnectionReset - || e.SocketErrorCode == SocketError.ConnectionAborted); - } - - private async Task InitializeAcceptedSslClientAsync(ClientMetadata client, CancellationToken token) - { - bool success = await StartTls(client, token).ConfigureAwait(false); - if (success) - { - client.Phase = ConnectionPhase.TlsEstablished; - await ProcessAcceptedClientAsync(client, token).ConfigureAwait(false); - } - } - - private async Task StartTls(ClientMetadata client, CancellationToken token) - { - try - { - token.ThrowIfCancellationRequested(); + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "listener exception: " + e.Message); + HandleException(e); + break; + } + } + } + + private static bool IsTransientAcceptSocketException(SocketException e) + { + return e != null + && (e.SocketErrorCode == SocketError.ConnectionReset + || e.SocketErrorCode == SocketError.ConnectionAborted); + } + + private void HandleException(Exception e) + { + _Instrumentation?.ExceptionRecorded(e); + _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + } + + private async Task InitializeAcceptedSslClientAsync(ClientMetadata client, CancellationToken token) + { + bool success = await StartTls(client, token).ConfigureAwait(false); + if (success) + { + client.Phase = ConnectionPhase.TlsEstablished; + await ProcessAcceptedClientAsync(client, token).ConfigureAwait(false); + } + } + + private async Task StartTls(ClientMetadata client, CancellationToken token) + { + try + { + token.ThrowIfCancellationRequested(); await client.SslStream.AuthenticateAsServerAsync(_SslCertificate, _SslConfiguration.ClientCertificateRequired, _TlsVersion.ToSslProtocols(), !_Settings.AcceptInvalidCertificates).ConfigureAwait(false); - if (!client.SslStream.IsEncrypted) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream from " + client.ToString() + " not encrypted"); - CleanupPendingClient(client); - return false; - } - - if (!client.SslStream.IsAuthenticated) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream from " + client.ToString() + " not authenticated"); - CleanupPendingClient(client); - return false; - } - - if (_Settings.MutuallyAuthenticate && !client.SslStream.IsMutuallyAuthenticated) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + $"mutual authentication with {client.ToString()} ({_TlsVersion}) failed"); - CleanupPendingClient(client); - return false; - } - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + $"disconnected during SSL/TLS establishment with {client.ToString()} ({_TlsVersion}): " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - - CleanupPendingClient(client); - return false; - } - - return true; - } - - private async Task ProcessAcceptedClientAsync(ClientMetadata client, CancellationToken token) - { - if (client == null) throw new ArgumentNullException(nameof(client)); - - bool authorized = await AuthorizePendingClientAsync(client, token).ConfigureAwait(false); - if (!authorized) return; - - _Settings.Logger?.Invoke(Severity.Debug, _Header + "starting data receiver for " + client.ToString()); - client.DataReceiver = Task.Run(() => DataReceiver(client, token), token); - - if (!String.IsNullOrEmpty(_Settings.PresharedKey)) - { - client.Phase = ConnectionPhase.PresharedKeyPending; - _Settings.Logger?.Invoke(Severity.Debug, _Header + "requesting authentication material from " + client.ToString()); - _ClientManager.AddUnauthenticatedClient(client.Guid); - WatsonMessage authMsg = new WatsonMessage(); - authMsg.Status = MessageStatus.AuthRequired; - await SendInternalAsync(client, authMsg, 0, null, token).ConfigureAwait(false); - return; - } - - await BeginPostAuthenticationFlowAsync(client, token).ConfigureAwait(false); - } - - private async Task AuthorizePendingClientAsync(ClientMetadata client, CancellationToken token) - { - if (_Callbacks.AuthorizeConnectionAsync == null) return true; - - client.Phase = ConnectionPhase.Authorizing; - ConnectionAuthorizationResult result = null; - - using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.AuthorizationTimeoutMs)) - using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) - { - try - { - X509Certificate clientCertificate = client.SslStream?.RemoteCertificate; - ConnectionAuthorizationContext context = new ConnectionAuthorizationContext(client, _Mode == Mode.Ssl, clientCertificate); - result = await _Callbacks.AuthorizeConnectionAsync(context, linkedCts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - if (timeoutCts.IsCancellationRequested) - { - result = ConnectionAuthorizationResult.Reject("Connection authorization timed out."); - } - else - { - result = ConnectionAuthorizationResult.Reject("Connection authorization canceled."); - } - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "connection authorization exception for " + client.ToString() + ": " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - result = ConnectionAuthorizationResult.Reject("Connection authorization failed: " + e.Message); - } - } - - if (result == null) result = ConnectionAuthorizationResult.Allow(); - if (result.Allowed) return true; - - await RejectPendingClientBeforeReceiverAsync(client, result.Reason, result.RejectionStatus, token).ConfigureAwait(false); - return false; - } - - private async Task BeginPostAuthenticationFlowAsync(ClientMetadata client, CancellationToken token) - { - if (_Callbacks.HandshakeAsync != null) - { - await StartHandshakePhaseAsync(client, token).ConfigureAwait(false); - } - else - { - client.Phase = ConnectionPhase.AwaitingRegistration; - } - } - - private async Task StartHandshakePhaseAsync(ClientMetadata client, CancellationToken token) - { - client.HandshakeRequired = true; - client.Phase = ConnectionPhase.HandshakePending; - client.HandshakeTransport = new HandshakeSessionTransport( - async (msg, innerToken) => await SendHandshakeDataAsync(client, msg, innerToken).ConfigureAwait(false), - async (reason, status, innerToken) => await SendStatusMessageAsync(client, status, reason, innerToken).ConfigureAwait(false), - token); - client.ServerHandshakeSession = new ServerHandshakeSession(client, client.HandshakeTransport); - - await SendStatusMessageAsync(client, MessageStatus.HandshakeBegin, "Handshake required", token).ConfigureAwait(false); - client.HandshakeTask = RunHandshakePhaseAsync(client, token); - } - - private async Task RunHandshakePhaseAsync(ClientMetadata client, CancellationToken token) - { - HandshakeResult result = null; - - using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.HandshakeTimeoutMs)) - using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) - { - try - { - result = await _Callbacks.HandshakeAsync(client.ServerHandshakeSession, linkedCts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - if (timeoutCts.IsCancellationRequested) - { - result = HandshakeResult.Fail("Handshake timed out."); - } - else - { - result = HandshakeResult.Fail("Handshake canceled."); - } - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "handshake exception for " + client.ToString() + ": " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - result = HandshakeResult.Fail("Handshake failed: " + e.Message); - } - } - - if (result == null) result = HandshakeResult.Succeed(); - - if (result.Success) - { - client.HandshakeCompleted = true; - client.Phase = ConnectionPhase.AwaitingRegistration; - _Events.HandleHandshakeSucceeded(this, new HandshakeSucceededEventArgs(client)); - await SendStatusMessageAsync(client, MessageStatus.HandshakeSuccess, "Handshake successful", token).ConfigureAwait(false); - } - else - { - client.HandshakeFailed = true; - client.FailureReason = result.Reason; - client.FailureStatus = result.FailureStatus; - _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(client, result.Reason, result.FailureStatus)); - await SendStatusMessageAsync(client, result.FailureStatus, result.Reason, token).ConfigureAwait(false); - client.Dispose(); - } - } - - private async Task ActivateClientAsync(ClientMetadata client, Guid requestedGuid) - { - if (client == null) throw new ArgumentNullException(nameof(client)); - - _Settings.Logger?.Invoke(Severity.Debug, _Header + "client " + client.ToString() + " attempting to register GUID " + requestedGuid.ToString()); - _ClientManager.ReplaceGuid(client.Guid, requestedGuid); - _ClientManager.RemovePendingClient(requestedGuid); - _ClientManager.AddClient(requestedGuid, client); - _ClientManager.AddClientLastSeen(requestedGuid); - _Settings.Logger?.Invoke(Severity.Debug, _Header + "updated client GUID from " + client.Guid + " to " + requestedGuid); - - client.Guid = requestedGuid; - client.Registered = true; - client.Phase = ConnectionPhase.Connected; - _Events.HandleClientConnected(this, new ConnectionEventArgs(client)); - } - - private void CleanupPendingClient(ClientMetadata client) - { - if (client == null) return; - - _ClientManager.Remove(client.Guid); - Interlocked.Decrement(ref _Connections); - client.Phase = ConnectionPhase.Disconnected; - client.Dispose(); - } - - private async Task RejectPendingClientBeforeReceiverAsync(ClientMetadata client, string reason, MessageStatus status, CancellationToken token) - { - if (client == null) return; - - client.ConnectionRejected = true; - client.FailureReason = String.IsNullOrEmpty(reason) ? "Connection rejected." : reason; - client.FailureStatus = status; - client.Phase = ConnectionPhase.Rejected; - - _Events.HandleConnectionRejected(this, new ConnectionRejectedEventArgs(client, client.FailureReason, status)); - await SendStatusMessageAsync(client, status, client.FailureReason, token).ConfigureAwait(false); - CleanupPendingClient(client); - } - - private async Task SendStatusMessageAsync(ClientMetadata client, MessageStatus status, string reason, CancellationToken token) - { - if (client == null) throw new ArgumentNullException(nameof(client)); - - byte[] data = Array.Empty(); - if (!String.IsNullOrEmpty(reason)) data = Encoding.UTF8.GetBytes(reason); - WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); - WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); - msg.Status = status; - await SendInternalAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); - } - - private async Task ReadStatusMessageAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null) return null; - if (msg.ContentLength <= 0) return null; - byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - if (data == null || data.Length < 1) return null; - return Encoding.UTF8.GetString(data); - } - - private async Task SendHandshakeDataAsync(ClientMetadata client, HandshakeMessage handshakeMessage, CancellationToken token) - { - if (client == null) throw new ArgumentNullException(nameof(client)); - if (handshakeMessage == null) throw new ArgumentNullException(nameof(handshakeMessage)); - - byte[] data = WatsonCommon.SerializeJsonBytes(SerializationHelper, handshakeMessage, false); - WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); - WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); - msg.Status = MessageStatus.HandshakeData; - await SendInternalAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); - } - - private async Task ReadHandshakeMessageAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null) return null; - if (msg.ContentLength <= 0) return new HandshakeMessage(); - - byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - if (data == null || data.Length < 1) return new HandshakeMessage(); - return SerializationHelper.DeserializeJson(Encoding.UTF8.GetString(data)); - } - - private async Task DrainMessageAsync(WatsonMessage msg, CancellationToken token) - { - if (msg == null || msg.ContentLength <= 0) return; - await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - } - - private void ValidateReceiveHandlerConfiguration() - { - bool usingMessages = _Events.IsUsingMessages; - bool usingSyncStreams = _Events.IsUsingStreams; - bool usingAsyncStreams = _Callbacks.StreamReceivedAsync != null; - - if (!usingMessages && !usingSyncStreams && !usingAsyncStreams) - { - throw new InvalidOperationException("One of either 'MessageReceived', 'StreamReceived', or 'Callbacks.StreamReceivedAsync' must first be set."); - } - - if (usingMessages && usingAsyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and Callbacks.StreamReceivedAsync are both configured; MessageReceived will be used."); - } - - if (usingMessages && usingSyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and StreamReceived are both configured; MessageReceived will be used."); - } - - if (!usingMessages && usingAsyncStreams && usingSyncStreams) - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "Callbacks.StreamReceivedAsync and StreamReceived are both configured; Callbacks.StreamReceivedAsync will be used."); - } - } - - private async Task HandleStreamPayloadAsync(ClientMetadata client, WatsonMessage msg, CancellationToken token) - { - if (msg == null) throw new ArgumentNullException(nameof(msg)); - - bool useAsyncCallback = _Callbacks.StreamReceivedAsync != null; - bool useSyncEvent = _Events.IsUsingStreams; - - if (!useAsyncCallback && !useSyncEvent) - { - throw new InvalidOperationException("Receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync."); - } - - bool useBufferedStream = msg.ContentLength < _Settings.MaxProxiedStreamSize; - Stream payloadStream = msg.DataStream; - - if (useBufferedStream) - { - payloadStream = await WatsonCommon.DataStreamToMemoryStream(msg.ContentLength, msg.DataStream, _Settings.StreamBufferSize, token).ConfigureAwait(false); - } - - WatsonStream watsonStream = new WatsonStream(msg.ContentLength, payloadStream); - StreamReceivedEventArgs args = new StreamReceivedEventArgs(client, msg.Metadata, msg.ContentLength, watsonStream); - bool preserveOriginalException = false; - - try - { - if (useAsyncCallback) - { - await _Callbacks.StreamReceivedAsync(args, token).ConfigureAwait(false); - } - else if (useBufferedStream) - { - await Task.Run(() => _Events.HandleStreamReceived(this, args), token).ConfigureAwait(false); - } - else - { - _Events.HandleStreamReceived(this, args); - } - } - catch (Exception e) when (!(e is OperationCanceledException)) - { - preserveOriginalException = true; - _Settings.Logger?.Invoke(Severity.Error, _Header + "stream receive handler exception for " + client.ToString() + ": " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - throw; - } - finally - { - if (watsonStream.RemainingBytes > 0 && !token.IsCancellationRequested) - { - try - { - await watsonStream.DrainAsync(_Settings.StreamBufferSize, token).ConfigureAwait(false); - } - catch (Exception e) when (preserveOriginalException) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "failed draining unread stream bytes from " + client.ToString() + " after handler exception: " + e.Message); - } - } - } - } - - private async Task DataReceiver(ClientMetadata client, CancellationToken token) - { - while (true) - { - try - { - token.ThrowIfCancellationRequested(); - WatsonMessage msg = await _MessageBuilder.BuildFromStream(client.ReceiveStream, token).ConfigureAwait(false); - - if (!String.IsNullOrEmpty(_Settings.PresharedKey)) - { - if (_ClientManager.ExistsUnauthenticatedClient(client.Guid)) - { - _Settings.Logger?.Invoke(Severity.Debug, _Header + "message received from unauthenticated endpoint " + client.ToString()); - - if (msg.Status == MessageStatus.AuthRequested) - { - // check preshared key - if (msg.PresharedKey != null && msg.PresharedKey.Length > 0) + if (!client.SslStream.IsEncrypted) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream from " + client.ToString() + " not encrypted"); + CleanupPendingClient(client); + return false; + } + + if (!client.SslStream.IsAuthenticated) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream from " + client.ToString() + " not authenticated"); + CleanupPendingClient(client); + return false; + } + + if (_Settings.MutuallyAuthenticate && !client.SslStream.IsMutuallyAuthenticated) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + $"mutual authentication with {client.ToString()} ({_TlsVersion}) failed"); + CleanupPendingClient(client); + return false; + } + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + $"disconnected during SSL/TLS establishment with {client.ToString()} ({_TlsVersion}): " + e.Message); + HandleException(e); + + CleanupPendingClient(client); + return false; + } + + return true; + } + + private async Task ProcessAcceptedClientAsync(ClientMetadata client, CancellationToken token) + { + if (client == null) throw new ArgumentNullException(nameof(client)); + + bool authorized = await AuthorizePendingClientAsync(client, token).ConfigureAwait(false); + if (!authorized) return; + + _Settings.Logger?.Invoke(Severity.Debug, _Header + "starting data receiver for " + client.ToString()); + client.DataReceiver = Task.Run(() => DataReceiver(client, token), token); + + if (!String.IsNullOrEmpty(_Settings.PresharedKey)) + { + client.Phase = ConnectionPhase.PresharedKeyPending; + _Settings.Logger?.Invoke(Severity.Debug, _Header + "requesting authentication material from " + client.ToString()); + _ClientManager.AddUnauthenticatedClient(client.Guid); + WatsonMessage authMsg = new WatsonMessage(); + authMsg.Status = MessageStatus.AuthRequired; + await SendInternalAsync(client, authMsg, 0, null, token).ConfigureAwait(false); + return; + } + + await BeginPostAuthenticationFlowAsync(client, token).ConfigureAwait(false); + } + + private async Task AuthorizePendingClientAsync(ClientMetadata client, CancellationToken token) + { + if (_Callbacks.AuthorizeConnectionAsync == null) return true; + + client.Phase = ConnectionPhase.Authorizing; + ConnectionAuthorizationResult result = null; + string authorizationOutcome = WatsonTcpInstrumentation.OutcomeSuccess; + + using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.AuthorizationTimeoutMs)) + using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) + { + try + { + X509Certificate clientCertificate = client.SslStream?.RemoteCertificate; + ConnectionAuthorizationContext context = new ConnectionAuthorizationContext(client, _Mode == Mode.Ssl, clientCertificate); + result = await _Callbacks.AuthorizeConnectionAsync(context, linkedCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (timeoutCts.IsCancellationRequested) + { + authorizationOutcome = WatsonTcpInstrumentation.OutcomeTimeout; + result = ConnectionAuthorizationResult.Reject("Connection authorization timed out."); + } + else + { + authorizationOutcome = WatsonTcpInstrumentation.OutcomeCanceled; + result = ConnectionAuthorizationResult.Reject("Connection authorization canceled."); + } + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "connection authorization exception for " + client.ToString() + ": " + e.Message); + HandleException(e); + authorizationOutcome = WatsonTcpInstrumentation.OutcomeFailure; + result = ConnectionAuthorizationResult.Reject("Connection authorization failed: " + e.Message); + } + } + + if (result == null) result = ConnectionAuthorizationResult.Allow(); + if (result.Allowed) + { + _Instrumentation?.Authorization(WatsonTcpInstrumentation.OutcomeSuccess); + return true; + } + + if (authorizationOutcome == WatsonTcpInstrumentation.OutcomeSuccess) authorizationOutcome = WatsonTcpInstrumentation.OutcomeFailure; + _Instrumentation?.Authorization(authorizationOutcome); + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeRejectedAuthorization); + + await RejectPendingClientBeforeReceiverAsync(client, result.Reason, result.RejectionStatus, token).ConfigureAwait(false); + return false; + } + + private async Task BeginPostAuthenticationFlowAsync(ClientMetadata client, CancellationToken token) + { + if (_Callbacks.HandshakeAsync != null) + { + await StartHandshakePhaseAsync(client, token).ConfigureAwait(false); + } + else + { + client.Phase = ConnectionPhase.AwaitingRegistration; + } + } + + private async Task StartHandshakePhaseAsync(ClientMetadata client, CancellationToken token) + { + client.HandshakeRequired = true; + client.Phase = ConnectionPhase.HandshakePending; + client.HandshakeTransport = new HandshakeSessionTransport( + async (msg, innerToken) => await SendHandshakeDataAsync(client, msg, innerToken).ConfigureAwait(false), + async (reason, status, innerToken) => await SendStatusMessageAsync(client, status, reason, innerToken).ConfigureAwait(false), + token); + client.ServerHandshakeSession = new ServerHandshakeSession(client, client.HandshakeTransport); + + await SendStatusMessageAsync(client, MessageStatus.HandshakeBegin, "Handshake required", token).ConfigureAwait(false); + client.HandshakeTask = RunHandshakePhaseAsync(client, token); + } + + private async Task RunHandshakePhaseAsync(ClientMetadata client, CancellationToken token) + { + HandshakeResult result = null; + long handshakeStartTimestamp = Stopwatch.GetTimestamp(); + string handshakeOutcome = WatsonTcpInstrumentation.OutcomeSuccess; + + using (Activity handshakeSpan = _Instrumentation?.StartHandshakeSpan()) + using (CancellationTokenSource timeoutCts = new CancellationTokenSource(_Settings.HandshakeTimeoutMs)) + using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) + { + try + { + result = await _Callbacks.HandshakeAsync(client.ServerHandshakeSession, linkedCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (timeoutCts.IsCancellationRequested) + { + handshakeOutcome = WatsonTcpInstrumentation.OutcomeTimeout; + result = HandshakeResult.Fail("Handshake timed out."); + } + else + { + handshakeOutcome = WatsonTcpInstrumentation.OutcomeCanceled; + result = HandshakeResult.Fail("Handshake canceled."); + } + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "handshake exception for " + client.ToString() + ": " + e.Message); + HandleException(e); + handshakeOutcome = WatsonTcpInstrumentation.OutcomeFailure; + result = HandshakeResult.Fail("Handshake failed: " + e.Message); + } + } + + if (result == null) result = HandshakeResult.Succeed(); + + if (result.Success) + { + _Instrumentation?.HandshakeCompleted(WatsonTcpInstrumentation.OutcomeSuccess, handshakeStartTimestamp); + client.HandshakeCompleted = true; + client.Phase = ConnectionPhase.AwaitingRegistration; + _Events.HandleHandshakeSucceeded(this, new HandshakeSucceededEventArgs(client)); + await SendStatusMessageAsync(client, MessageStatus.HandshakeSuccess, "Handshake successful", token).ConfigureAwait(false); + } + else + { + if (handshakeOutcome == WatsonTcpInstrumentation.OutcomeSuccess) handshakeOutcome = WatsonTcpInstrumentation.OutcomeFailure; + _Instrumentation?.HandshakeCompleted(handshakeOutcome, handshakeStartTimestamp); + client.HandshakeFailed = true; + client.FailureReason = result.Reason; + client.FailureStatus = result.FailureStatus; + _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(client, result.Reason, result.FailureStatus)); + await SendStatusMessageAsync(client, result.FailureStatus, result.Reason, token).ConfigureAwait(false); + client.Dispose(); + } + } + + private async Task ActivateClientAsync(ClientMetadata client, Guid requestedGuid) + { + if (client == null) throw new ArgumentNullException(nameof(client)); + + _Settings.Logger?.Invoke(Severity.Debug, _Header + "client " + client.ToString() + " attempting to register GUID " + requestedGuid.ToString()); + _ClientManager.ReplaceGuid(client.Guid, requestedGuid); + _ClientManager.RemovePendingClient(requestedGuid); + _ClientManager.AddClient(requestedGuid, client); + _ClientManager.AddClientLastSeen(requestedGuid); + _Settings.Logger?.Invoke(Severity.Debug, _Header + "updated client GUID from " + client.Guid + " to " + requestedGuid); + + client.Guid = requestedGuid; + client.Registered = true; + client.Phase = ConnectionPhase.Connected; + _Instrumentation?.ConnectionOutcome(WatsonTcpInstrumentation.OutcomeAccepted); + client.SessionActivity = _Instrumentation?.StartSessionSpan(client.IpPort, client.Guid); + _Events.HandleClientConnected(this, new ConnectionEventArgs(client)); + } + + private void CleanupPendingClient(ClientMetadata client) + { + if (client == null) return; + + _ClientManager.Remove(client.Guid); + Interlocked.Decrement(ref _Connections); + client.Phase = ConnectionPhase.Disconnected; + client.Dispose(); + } + + private async Task RejectPendingClientBeforeReceiverAsync(ClientMetadata client, string reason, MessageStatus status, CancellationToken token) + { + if (client == null) return; + + client.ConnectionRejected = true; + client.FailureReason = String.IsNullOrEmpty(reason) ? "Connection rejected." : reason; + client.FailureStatus = status; + client.Phase = ConnectionPhase.Rejected; + + _Events.HandleConnectionRejected(this, new ConnectionRejectedEventArgs(client, client.FailureReason, status)); + await SendStatusMessageAsync(client, status, client.FailureReason, token).ConfigureAwait(false); + CleanupPendingClient(client); + } + + private async Task SendStatusMessageAsync(ClientMetadata client, MessageStatus status, string reason, CancellationToken token) + { + if (client == null) throw new ArgumentNullException(nameof(client)); + + byte[] data = Array.Empty(); + if (!String.IsNullOrEmpty(reason)) data = Encoding.UTF8.GetBytes(reason); + WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); + WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); + msg.Status = status; + await SendInternalAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); + } + + private async Task ReadStatusMessageAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null) return null; + if (msg.ContentLength <= 0) return null; + byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + if (data == null || data.Length < 1) return null; + return Encoding.UTF8.GetString(data); + } + + private async Task SendHandshakeDataAsync(ClientMetadata client, HandshakeMessage handshakeMessage, CancellationToken token) + { + if (client == null) throw new ArgumentNullException(nameof(client)); + if (handshakeMessage == null) throw new ArgumentNullException(nameof(handshakeMessage)); + + byte[] data = WatsonCommon.SerializeJsonBytes(SerializationHelper, handshakeMessage, false); + WatsonCommon.BytesToStream(data, 0, out int contentLength, out Stream stream); + WatsonMessage msg = _MessageBuilder.ConstructNew(contentLength, stream, false, false, null, null); + msg.Status = MessageStatus.HandshakeData; + await SendInternalAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); + } + + private async Task ReadHandshakeMessageAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null) return null; + if (msg.ContentLength <= 0) return new HandshakeMessage(); + + byte[] data = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + if (data == null || data.Length < 1) return new HandshakeMessage(); + return SerializationHelper.DeserializeJson(Encoding.UTF8.GetString(data)); + } + + private async Task DrainMessageAsync(WatsonMessage msg, CancellationToken token) + { + if (msg == null || msg.ContentLength <= 0) return; + await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + } + + private void ValidateReceiveHandlerConfiguration() + { + bool usingMessages = _Events.IsUsingMessages; + bool usingSyncStreams = _Events.IsUsingStreams; + bool usingAsyncStreams = _Callbacks.StreamReceivedAsync != null; + + if (!usingMessages && !usingSyncStreams && !usingAsyncStreams) + { + throw new InvalidOperationException("One of either 'MessageReceived', 'StreamReceived', or 'Callbacks.StreamReceivedAsync' must first be set."); + } + + if (usingMessages && usingAsyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and Callbacks.StreamReceivedAsync are both configured; MessageReceived will be used."); + } + + if (usingMessages && usingSyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "MessageReceived and StreamReceived are both configured; MessageReceived will be used."); + } + + if (!usingMessages && usingAsyncStreams && usingSyncStreams) + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "Callbacks.StreamReceivedAsync and StreamReceived are both configured; Callbacks.StreamReceivedAsync will be used."); + } + } + + private async Task HandleStreamPayloadAsync(ClientMetadata client, WatsonMessage msg, CancellationToken token) + { + if (msg == null) throw new ArgumentNullException(nameof(msg)); + + bool useAsyncCallback = _Callbacks.StreamReceivedAsync != null; + bool useSyncEvent = _Events.IsUsingStreams; + + if (!useAsyncCallback && !useSyncEvent) + { + throw new InvalidOperationException("Receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync."); + } + + bool useBufferedStream = msg.ContentLength < _Settings.MaxProxiedStreamSize; + Stream payloadStream = msg.DataStream; + + if (useBufferedStream) + { + payloadStream = await WatsonCommon.DataStreamToMemoryStream(msg.ContentLength, msg.DataStream, _Settings.StreamBufferSize, token).ConfigureAwait(false); + } + + WatsonStream watsonStream = new WatsonStream(msg.ContentLength, payloadStream); + StreamReceivedEventArgs args = new StreamReceivedEventArgs(client, msg.Metadata, msg.ContentLength, watsonStream); + bool preserveOriginalException = false; + + try + { + if (useAsyncCallback) + { + await _Callbacks.StreamReceivedAsync(args, token).ConfigureAwait(false); + } + else if (useBufferedStream) + { + await Task.Run(() => _Events.HandleStreamReceived(this, args), token).ConfigureAwait(false); + } + else + { + _Events.HandleStreamReceived(this, args); + } + } + catch (Exception e) when (!(e is OperationCanceledException)) + { + preserveOriginalException = true; + _Settings.Logger?.Invoke(Severity.Error, _Header + "stream receive handler exception for " + client.ToString() + ": " + e.Message); + HandleException(e); + throw; + } + finally + { + if (watsonStream.RemainingBytes > 0 && !token.IsCancellationRequested) + { + long drained = watsonStream.RemainingBytes; + try + { + await watsonStream.DrainAsync(_Settings.StreamBufferSize, token).ConfigureAwait(false); + _Instrumentation?.StreamDrained(drained); + } + catch (Exception e) when (preserveOriginalException) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "failed draining unread stream bytes from " + client.ToString() + " after handler exception: " + e.Message); + } + } + } + } + + private async Task DataReceiver(ClientMetadata client, CancellationToken token) + { + while (true) + { + try + { + token.ThrowIfCancellationRequested(); + WatsonMessage msg = await _MessageBuilder.BuildFromStream(client.ReceiveStream, token).ConfigureAwait(false); + + if (!String.IsNullOrEmpty(_Settings.PresharedKey)) + { + if (_ClientManager.ExistsUnauthenticatedClient(client.Guid)) + { + _Settings.Logger?.Invoke(Severity.Debug, _Header + "message received from unauthenticated endpoint " + client.ToString()); + + if (msg.Status == MessageStatus.AuthRequested) + { + // check preshared key + if (msg.PresharedKey != null && msg.PresharedKey.Length > 0) { string clientPsk = Encoding.UTF8.GetString(msg.PresharedKey).Trim(); - if (_Settings.PresharedKey.Trim().Equals(clientPsk, StringComparison.Ordinal)) - { - _Settings.Logger?.Invoke(Severity.Debug, _Header + "accepted authentication for " + client.ToString()); - _ClientManager.RemoveUnauthenticatedClient(client.Guid); - _Events.HandleAuthenticationSucceeded(this, new AuthenticationSucceededEventArgs(client)); - - await SendStatusMessageAsync(client, MessageStatus.AuthSuccess, "Authentication successful", token).ConfigureAwait(false); - await BeginPostAuthenticationFlowAsync(client, token).ConfigureAwait(false); - continue; - } - else - { - _Settings.Logger?.Invoke(Severity.Warn, _Header + "declined authentication for " + client.ToString()); - _Events.HandleAuthenticationFailed(this, new AuthenticationFailedEventArgs(client.IpPort)); - client.HandshakeFailed = false; - client.FailureReason = "Authentication failed."; - client.FailureStatus = MessageStatus.AuthFailure; - await SendStatusMessageAsync(client, MessageStatus.AuthFailure, client.FailureReason, token).ConfigureAwait(false); - break; - } - } - else - { - // AuthRequested message with no pre-shared key - decline and terminate - _Settings.Logger?.Invoke(Severity.Warn, _Header + "no authentication material for " + client.ToString()); - _Events.HandleAuthenticationFailed(this, new AuthenticationFailedEventArgs(client.IpPort)); - client.FailureReason = "Authentication failed."; - client.FailureStatus = MessageStatus.AuthFailure; - await SendStatusMessageAsync(client, MessageStatus.AuthFailure, client.FailureReason, token).ConfigureAwait(false); - break; - } - } - else - { - // Non-auth message from unauthenticated client - ignore and wait for auth - _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring message from unauthenticated client " + client.ToString() + " (waiting for authentication)"); - await DrainMessageAsync(msg, token).ConfigureAwait(false); - await Task.Delay(30, token).ConfigureAwait(false); - continue; - } - } - } + if (_Settings.PresharedKey.Trim().Equals(clientPsk, StringComparison.Ordinal)) + { + _Settings.Logger?.Invoke(Severity.Debug, _Header + "accepted authentication for " + client.ToString()); + _Instrumentation?.Authentication(WatsonTcpInstrumentation.OutcomeSuccess); + _ClientManager.RemoveUnauthenticatedClient(client.Guid); + _Events.HandleAuthenticationSucceeded(this, new AuthenticationSucceededEventArgs(client)); + + await SendStatusMessageAsync(client, MessageStatus.AuthSuccess, "Authentication successful", token).ConfigureAwait(false); + await BeginPostAuthenticationFlowAsync(client, token).ConfigureAwait(false); + continue; + } + else + { + _Settings.Logger?.Invoke(Severity.Warn, _Header + "declined authentication for " + client.ToString()); + _Instrumentation?.Authentication(WatsonTcpInstrumentation.OutcomeFailure); + _Events.HandleAuthenticationFailed(this, new AuthenticationFailedEventArgs(client.IpPort)); + client.HandshakeFailed = false; + client.FailureReason = "Authentication failed."; + client.FailureStatus = MessageStatus.AuthFailure; + await SendStatusMessageAsync(client, MessageStatus.AuthFailure, client.FailureReason, token).ConfigureAwait(false); + break; + } + } + else + { + // AuthRequested message with no pre-shared key - decline and terminate + _Settings.Logger?.Invoke(Severity.Warn, _Header + "no authentication material for " + client.ToString()); + _Instrumentation?.Authentication(WatsonTcpInstrumentation.OutcomeFailure); + _Events.HandleAuthenticationFailed(this, new AuthenticationFailedEventArgs(client.IpPort)); + client.FailureReason = "Authentication failed."; + client.FailureStatus = MessageStatus.AuthFailure; + await SendStatusMessageAsync(client, MessageStatus.AuthFailure, client.FailureReason, token).ConfigureAwait(false); + break; + } + } + else + { + // Non-auth message from unauthenticated client - ignore and wait for auth + _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring message from unauthenticated client " + client.ToString() + " (waiting for authentication)"); + await DrainMessageAsync(msg, token).ConfigureAwait(false); + await Task.Delay(30, token).ConfigureAwait(false); + continue; + } + } + } if (msg.Status == MessageStatus.Shutdown) { @@ -1320,45 +1381,45 @@ private async Task DataReceiver(ClientMetadata client, CancellationToken token) break; } else if (msg.Status == MessageStatus.Removed) - { - _Settings.Logger?.Invoke(Severity.Debug, _Header + "sent disconnect notice to " + client.ToString()); - break; - } - else if (client.HandshakeRequired && !client.HandshakeCompleted) - { - if (msg.Status == MessageStatus.HandshakeData) - { - HandshakeMessage handshakeMsg = await ReadHandshakeMessageAsync(msg, token).ConfigureAwait(false); - client.HandshakeTransport?.Enqueue(handshakeMsg); - continue; - } - else if (msg.Status == MessageStatus.HandshakeFailure) - { - string reason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); - client.HandshakeFailed = true; - client.FailureReason = reason; - client.FailureStatus = MessageStatus.HandshakeFailure; - _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(client, reason, MessageStatus.HandshakeFailure)); - break; - } - else if (msg.Status == MessageStatus.RegisterClient) - { - _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring registration from " + client.ToString() + " while handshake is pending"); - await DrainMessageAsync(msg, token).ConfigureAwait(false); - continue; - } - else - { - _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring message from " + client.ToString() + " while handshake is pending"); - await DrainMessageAsync(msg, token).ConfigureAwait(false); - continue; - } - } - else if (msg.Status == MessageStatus.RegisterClient) - { - await ActivateClientAsync(client, msg.SenderGuid).ConfigureAwait(false); - continue; - } + { + _Settings.Logger?.Invoke(Severity.Debug, _Header + "sent disconnect notice to " + client.ToString()); + break; + } + else if (client.HandshakeRequired && !client.HandshakeCompleted) + { + if (msg.Status == MessageStatus.HandshakeData) + { + HandshakeMessage handshakeMsg = await ReadHandshakeMessageAsync(msg, token).ConfigureAwait(false); + client.HandshakeTransport?.Enqueue(handshakeMsg); + continue; + } + else if (msg.Status == MessageStatus.HandshakeFailure) + { + string reason = await ReadStatusMessageAsync(msg, token).ConfigureAwait(false); + client.HandshakeFailed = true; + client.FailureReason = reason; + client.FailureStatus = MessageStatus.HandshakeFailure; + _Events.HandleHandshakeFailed(this, new HandshakeFailedEventArgs(client, reason, MessageStatus.HandshakeFailure)); + break; + } + else if (msg.Status == MessageStatus.RegisterClient) + { + _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring registration from " + client.ToString() + " while handshake is pending"); + await DrainMessageAsync(msg, token).ConfigureAwait(false); + continue; + } + else + { + _Settings.Logger?.Invoke(Severity.Debug, _Header + "ignoring message from " + client.ToString() + " while handshake is pending"); + await DrainMessageAsync(msg, token).ConfigureAwait(false); + continue; + } + } + else if (msg.Status == MessageStatus.RegisterClient) + { + await ActivateClientAsync(client, msg.SenderGuid).ConfigureAwait(false); + continue; + } if (msg.SyncRequest) { @@ -1409,6 +1470,7 @@ private async Task DataReceiver(ClientMetadata client, CancellationToken token) } else { + _Instrumentation?.SyncExpiredMessage(WatsonTcpInstrumentation.KindRequest); _Settings.Logger?.Invoke(Severity.Debug, _Header + "expired synchronous request received and discarded from " + client.ToString()); } } @@ -1424,6 +1486,7 @@ private async Task DataReceiver(ClientMetadata client, CancellationToken token) TaskCompletionSource tcs; if (_SyncRequests.TryRemove(msg.ConversationGuid, out tcs)) { + _Instrumentation?.SyncResponseReceived(); SyncResponse syncResp = new SyncResponse(msg.ConversationGuid, msg.ExpirationUtc.Value, msg.Metadata, msgData); tcs.TrySetResult(syncResp); } @@ -1434,6 +1497,7 @@ private async Task DataReceiver(ClientMetadata client, CancellationToken token) } else { + _Instrumentation?.SyncExpiredMessage(WatsonTcpInstrumentation.KindResponse); _Settings.Logger?.Invoke(Severity.Debug, _Header + "expired synchronous response received and discarded from " + client.ToString()); TaskCompletionSource tcs; _SyncRequests.TryRemove(msg.ConversationGuid, out tcs); @@ -1441,82 +1505,94 @@ private async Task DataReceiver(ClientMetadata client, CancellationToken token) } else { - byte[] msgData = null; - - if (_Events.IsUsingMessages) + using (Activity receiveSpan = _Instrumentation?.StartReceiveSpan(msg.ContentLength, client.Guid)) { - msgData = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); - MessageReceivedEventArgs mr = new MessageReceivedEventArgs(client, msg.Metadata, msgData); - await Task.Run(() => _Events.HandleMessageReceived(this, mr), token); + byte[] msgData = null; + + if (_Events.IsUsingMessages) + { + msgData = await WatsonCommon.ReadMessageDataAsync(msg, _Settings.StreamBufferSize, token).ConfigureAwait(false); + MessageReceivedEventArgs mr = new MessageReceivedEventArgs(client, msg.Metadata, msgData); + await Task.Run(() => _Events.HandleMessageReceived(this, mr), token); + } + else if (_Callbacks.StreamReceivedAsync != null || _Events.IsUsingStreams) + { + await HandleStreamPayloadAsync(client, msg, token).ConfigureAwait(false); + } + else + { + _Settings.Logger?.Invoke(Severity.Error, _Header + "receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync"); + break; + } } - else if (_Callbacks.StreamReceivedAsync != null || _Events.IsUsingStreams) - { - await HandleStreamPayloadAsync(client, msg, token).ConfigureAwait(false); - } - else - { - _Settings.Logger?.Invoke(Severity.Error, _Header + "receive handler not set for MessageReceived, StreamReceived, or Callbacks.StreamReceivedAsync"); - break; - } } - - _Statistics.IncrementReceivedMessages(); - _Statistics.AddReceivedBytes(msg.ContentLength); - if (client.Registered) - { - _ClientManager.UpdateClientLastSeen(client.Guid, DateTime.UtcNow); - } - } - catch (ObjectDisposedException ode) - { - _Settings?.Logger?.Invoke(Severity.Debug, _Header + "object disposed exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(ode)); + + _Statistics.IncrementReceivedMessages(); + _Statistics.AddReceivedBytes(msg.ContentLength); + _Instrumentation?.MessageReceived(msg, msg.ContentLength); + if (client.Registered) + { + _ClientManager.UpdateClientLastSeen(client.Guid, DateTime.UtcNow); + } + } + catch (ObjectDisposedException ode) + { + _Settings?.Logger?.Invoke(Severity.Debug, _Header + "object disposed exception encountered"); + HandleException(ode); break; } catch (TaskCanceledException tce) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "task canceled exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(tce)); + HandleException(tce); break; } catch (OperationCanceledException oce) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "operation canceled exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(oce)); + HandleException(oce); break; } catch (IOException ioe) { _Settings?.Logger?.Invoke(Severity.Debug, _Header + "IO exception encountered"); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(ioe)); + HandleException(ioe); break; } catch (Exception e) { _Settings?.Logger?.Invoke(Severity.Error, _Header + "data receiver exception for " + client.ToString() + ": " + e.Message); - _Events?.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + HandleException(e); break; } } - - if (_Settings != null && _Events != null) - { - DisconnectReason reason = DisconnectReason.Normal; - if (client.ConnectionRejected) reason = DisconnectReason.ConnectionRejected; - else if (client.HandshakeFailed && client.FailureStatus == MessageStatus.HandshakeFailure) reason = DisconnectReason.HandshakeFailure; - else if (_ClientManager.ExistsClientKicked(client.Guid)) reason = DisconnectReason.Removed; - else if (_ClientManager.ExistsClientTimedout(client.Guid)) reason = DisconnectReason.Timeout; - else if (client.FailureStatus == MessageStatus.AuthFailure) reason = DisconnectReason.AuthFailure; - - if (client.Registered) - { - _Events.HandleClientDisconnected(this, new DisconnectionEventArgs(client, reason)); - } - - _ClientManager.Remove(client.Guid); - Interlocked.Decrement(ref _Connections); - - _Settings?.Logger?.Invoke(Severity.Debug, _Header + "client " + client.ToString() + " disconnected"); + + if (_Settings != null && _Events != null) + { + DisconnectReason reason = DisconnectReason.Normal; + if (client.ConnectionRejected) reason = DisconnectReason.ConnectionRejected; + else if (client.HandshakeFailed && client.FailureStatus == MessageStatus.HandshakeFailure) reason = DisconnectReason.HandshakeFailure; + else if (_ClientManager.ExistsClientKicked(client.Guid)) reason = DisconnectReason.Removed; + else if (_ClientManager.ExistsClientTimedout(client.Guid)) reason = DisconnectReason.Timeout; + else if (client.FailureStatus == MessageStatus.AuthFailure) reason = DisconnectReason.AuthFailure; + + if (client.Registered) + { + _Instrumentation?.Disconnection(reason); + _Events.HandleClientDisconnected(this, new DisconnectionEventArgs(client, reason)); + } + + if (client.SessionActivity != null) + { + client.SessionActivity.SetTag(WatsonTcpMetrics.TagReason, reason.ToString()); + client.SessionActivity.Dispose(); + client.SessionActivity = null; + } + + _ClientManager.Remove(client.Guid); + Interlocked.Decrement(ref _Connections); + + _Settings?.Logger?.Invoke(Severity.Debug, _Header + "client " + client.ToString() + " disconnected"); client.Dispose(); } } @@ -1538,25 +1614,30 @@ private async Task SendInternalAsync(ClientMetadata client, WatsonMessage } } - CancellationTokenSource linkedCts = null; - if (token == default(CancellationToken)) - { - token = _Token; - } - else if (_Token.CanBeCanceled) - { - linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _Token); - token = linkedCts.Token; - } - - await client.WriteLock.WaitAsync(token).ConfigureAwait(false); - - try - { - await SendMessageAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); - - _Statistics.IncrementSentMessages(); - _Statistics.AddSentBytes(contentLength); + CancellationTokenSource linkedCts = null; + if (token == default(CancellationToken)) + { + token = _Token; + } + else if (_Token.CanBeCanceled) + { + linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, _Token); + token = linkedCts.Token; + } + + await client.WriteLock.WaitAsync(token).ConfigureAwait(false); + + try + { + long sendStartTimestamp = Stopwatch.GetTimestamp(); + using (Activity sendSpan = _Instrumentation?.StartSendSpan(contentLength, msg.SyncRequest, client.Guid)) + { + await SendMessageAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); + } + + _Statistics.IncrementSentMessages(); + _Statistics.AddSentBytes(contentLength); + _Instrumentation?.MessageSent(msg, contentLength, sendStartTimestamp); return true; } catch (TaskCanceledException) @@ -1572,15 +1653,15 @@ private async Task SendInternalAsync(ClientMetadata client, WatsonMessage catch (Exception e) { _Settings.Logger?.Invoke(Severity.Error, _Header + "failed to write message to " + client.ToString() + ": " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); + HandleException(e); return false; } - finally - { - linkedCts?.Dispose(); - if (client != null) client.WriteLock.Release(); - } - } + finally + { + linkedCts?.Dispose(); + if (client != null) client.WriteLock.Release(); + } + } private async Task SendAndWaitInternalAsync(ClientMetadata client, WatsonMessage msg, int timeoutMs, long contentLength, Stream stream, CancellationToken token) { @@ -1599,60 +1680,70 @@ private async Task SendAndWaitInternalAsync(ClientMetadata client, TaskCompletionSource tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _SyncRequests[msg.ConversationGuid] = tcs; - await client.WriteLock.WaitAsync(token); + long syncStartTimestamp = Stopwatch.GetTimestamp(); - try + using (Activity syncSpan = _Instrumentation?.StartSyncSpan(msg.ConversationGuid, contentLength)) { - await SendMessageAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); - _Settings.Logger?.Invoke(Severity.Debug, _Header + client.ToString() + " synchronous request sent: " + msg.ConversationGuid); + await client.WriteLock.WaitAsync(token); - _Statistics.IncrementSentMessages(); - _Statistics.AddSentBytes(contentLength); - } - catch (Exception e) - { - _Settings.Logger?.Invoke(Severity.Error, _Header + client.ToString() + " failed to write message: " + e.Message); - _Events.HandleExceptionEncountered(this, new ExceptionEventArgs(e)); - _SyncRequests.TryRemove(msg.ConversationGuid, out _); - throw; - } - finally - { - if (client != null) client.WriteLock.Release(); - } + try + { + long sendStartTimestamp = Stopwatch.GetTimestamp(); + await SendMessageAsync(client, msg, contentLength, stream, token).ConfigureAwait(false); + _Settings.Logger?.Invoke(Severity.Debug, _Header + client.ToString() + " synchronous request sent: " + msg.ConversationGuid); + + _Statistics.IncrementSentMessages(); + _Statistics.AddSentBytes(contentLength); + _Instrumentation?.MessageSent(msg, contentLength, sendStartTimestamp); + _Instrumentation?.SyncRequestSent(); + } + catch (Exception e) + { + _Settings.Logger?.Invoke(Severity.Error, _Header + client.ToString() + " failed to write message: " + e.Message); + HandleException(e); + _SyncRequests.TryRemove(msg.ConversationGuid, out _); + throw; + } + finally + { + if (client != null) client.WriteLock.Release(); + } - // Wait for the response with timeout - using (CancellationTokenSource timeoutCts = new CancellationTokenSource(timeoutMs)) - { - using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) + // Wait for the response with timeout + using (CancellationTokenSource timeoutCts = new CancellationTokenSource(timeoutMs)) { - try + using (CancellationTokenSource linkedCts = CancellationTokenSource.CreateLinkedTokenSource(token, timeoutCts.Token)) { - linkedCts.Token.Register(() => tcs.TrySetCanceled()); - SyncResponse ret = await tcs.Task.ConfigureAwait(false); - return ret; - } - catch (TaskCanceledException) - { - _SyncRequests.TryRemove(msg.ConversationGuid, out _); - - if (timeoutCts.IsCancellationRequested) + try { - _Settings.Logger?.Invoke(Severity.Error, _Header + "synchronous response not received within the timeout window"); - throw new TimeoutException("A response to a synchronous request was not received within the timeout window."); + linkedCts.Token.Register(() => tcs.TrySetCanceled()); + SyncResponse ret = await tcs.Task.ConfigureAwait(false); + _Instrumentation?.SyncCompleted(WatsonTcpInstrumentation.OutcomeCompleted, syncStartTimestamp); + return ret; } + catch (TaskCanceledException) + { + _SyncRequests.TryRemove(msg.ConversationGuid, out _); + + if (timeoutCts.IsCancellationRequested) + { + _Instrumentation?.SyncCompleted(WatsonTcpInstrumentation.OutcomeTimeout, syncStartTimestamp); + _Settings.Logger?.Invoke(Severity.Error, _Header + "synchronous response not received within the timeout window"); + throw new TimeoutException("A response to a synchronous request was not received within the timeout window."); + } - throw; + throw; + } } } } } - private async Task SendMessageAsync(ClientMetadata client, WatsonMessage msg, long contentLength, Stream stream, CancellationToken token) - { - byte[] headerBytes = _MessageBuilder.GetHeaderBytes(msg); - await WatsonCommon.WriteMessageAsync(client.DataStream, headerBytes, contentLength, stream, _Settings.StreamBufferSize, token).ConfigureAwait(false); - } + private async Task SendMessageAsync(ClientMetadata client, WatsonMessage msg, long contentLength, Stream stream, CancellationToken token) + { + byte[] headerBytes = _MessageBuilder.GetHeaderBytes(msg); + await WatsonCommon.WriteMessageAsync(client.DataStream, headerBytes, contentLength, stream, _Settings.StreamBufferSize, token).ConfigureAwait(false); + } #endregion @@ -1664,26 +1755,26 @@ private async Task MonitorForIdleClients(CancellationToken token) try { - while (true) - { - token.ThrowIfCancellationRequested(); + while (true) + { + token.ThrowIfCancellationRequested(); await Task.Delay(5000, _Token).ConfigureAwait(false); - if (_Settings.IdleClientTimeoutSeconds > 0) - { - long idleCutoffTicks = DateTime.UtcNow.AddSeconds(-1 * _Settings.IdleClientTimeoutSeconds).Ticks; - - foreach (ClientMetadata client in _ClientManager.EnumerateClients()) - { - if (client.LastSeenUtcTicks > 0 && client.LastSeenUtcTicks < idleCutoffTicks) - { - _ClientManager.AddClientTimedout(client.Guid); - _Settings.Logger?.Invoke(Severity.Debug, _Header + "disconnecting client " + client.Guid + " due to idle timeout"); - await DisconnectClientAsync(client.Guid, MessageStatus.Timeout, true, token).ConfigureAwait(false); - } - } - } + if (_Settings.IdleClientTimeoutSeconds > 0) + { + long idleCutoffTicks = DateTime.UtcNow.AddSeconds(-1 * _Settings.IdleClientTimeoutSeconds).Ticks; + + foreach (ClientMetadata client in _ClientManager.EnumerateClients()) + { + if (client.LastSeenUtcTicks > 0 && client.LastSeenUtcTicks < idleCutoffTicks) + { + _ClientManager.AddClientTimedout(client.Guid); + _Settings.Logger?.Invoke(Severity.Debug, _Header + "disconnecting client " + client.Guid + " due to idle timeout"); + await DisconnectClientAsync(client.Guid, MessageStatus.Timeout, true, token).ConfigureAwait(false); + } + } + } // Purge stale kicked/timed-out records every ~60 seconds (12 iterations * 5s) purgeCounter++; diff --git a/src/WatsonTcp/WatsonTcpServerSettings.cs b/src/WatsonTcp/WatsonTcpServerSettings.cs index cad1a92..c738610 100644 --- a/src/WatsonTcp/WatsonTcpServerSettings.cs +++ b/src/WatsonTcp/WatsonTcpServerSettings.cs @@ -167,46 +167,63 @@ public int MaxHeaderSize } } - /// - /// Enable or disable enforcement of the MaxConnections setting. - /// When true (default), new connections will be rejected when MaxConnections is reached. - /// When false, connections will be accepted beyond MaxConnections (legacy behavior) with a warning logged. - /// - public bool EnforceMaxConnections { get; set; } = true; - - /// - /// Number of milliseconds to wait for the connection authorization callback before rejecting the connection. - /// Value must be greater than zero. - /// - public int AuthorizationTimeoutMs - { - get - { - return _AuthorizationTimeoutMs; - } - set - { - if (value < 1) throw new ArgumentException("AuthorizationTimeoutMs must be greater than zero."); - _AuthorizationTimeoutMs = value; - } - } - - /// - /// Number of milliseconds to wait for a custom handshake to complete before rejecting the connection. - /// Value must be greater than zero. - /// - public int HandshakeTimeoutMs - { - get - { - return _HandshakeTimeoutMs; - } - set - { - if (value < 1) throw new ArgumentException("HandshakeTimeoutMs must be greater than zero."); - _HandshakeTimeoutMs = value; - } - } + /// + /// Enable or disable enforcement of the MaxConnections setting. + /// When true (default), new connections will be rejected when MaxConnections is reached. + /// When false, connections will be accepted beyond MaxConnections (legacy behavior) with a warning logged. + /// + public bool EnforceMaxConnections { get; set; } = true; + + /// + /// Number of milliseconds to wait for the connection authorization callback before rejecting the connection. + /// Value must be greater than zero. + /// + public int AuthorizationTimeoutMs + { + get + { + return _AuthorizationTimeoutMs; + } + set + { + if (value < 1) throw new ArgumentException("AuthorizationTimeoutMs must be greater than zero."); + _AuthorizationTimeoutMs = value; + } + } + + /// + /// Number of milliseconds to wait for a custom handshake to complete before rejecting the connection. + /// Value must be greater than zero. + /// + public int HandshakeTimeoutMs + { + get + { + return _HandshakeTimeoutMs; + } + set + { + if (value < 1) throw new ArgumentException("HandshakeTimeoutMs must be greater than zero."); + _HandshakeTimeoutMs = value; + } + } + + /// + /// Enable or disable emission of metrics into the WatsonTcp + /// . Default is true. + /// When true, the meter and its instruments are created; recording is a near-free no-op unless a + /// telemetry host subscribes to the meter by name. When false, no meter is created and there is + /// zero overhead. See and TELEMETRY.md. + /// + public bool EnableMetrics { get; set; } = true; + + /// + /// Enable or disable emission of distributed-tracing spans into the WatsonTcp + /// . Default is true. + /// When true, spans are created only if a tracing host subscribes to the activity source by name; + /// otherwise span creation is a near-free no-op. See and TELEMETRY.md. + /// + public bool EnableTracing { get; set; } = true; #endregion @@ -217,11 +234,11 @@ public int HandshakeTimeoutMs private int _MaxConnections = 4096; private int _IdleClientTimeoutSeconds = 0; - private List _PermittedIPs = new List(); - private List _BlockedIPs = new List(); - private int _MaxHeaderSize = 262144; - private int _AuthorizationTimeoutMs = 5000; - private int _HandshakeTimeoutMs = 10000; + private List _PermittedIPs = new List(); + private List _BlockedIPs = new List(); + private int _MaxHeaderSize = 262144; + private int _AuthorizationTimeoutMs = 5000; + private int _HandshakeTimeoutMs = 10000; #endregion