Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
integration tests. The 13.x line is the .NET 10 generation of Aspire;
9.x pulls a vulnerable KubernetesClient 17.0.4 (GHSA-w7r3-mgwf-4mqq)
that trips NU1902 under our TreatWarningsAsErrors policy. -->
<AspireHostingVersion>13.6.4</AspireHostingVersion>
<AspireHostingVersion>13.6.5</AspireHostingVersion>
<CommunityToolkitAspireVersion>13.6.1</CommunityToolkitAspireVersion>
<ModelContextProtocolVersion>2.2.0</ModelContextProtocolVersion>
</PropertyGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,13 @@ public static Props CreateProps(
/// <summary>Discord's start does not imply readiness; wait up to 30s for Discord.Net's READY event.</summary>
protected override TimeSpan? ReadySignalTimeout => TimeSpan.FromSeconds(30);

/// <summary>
/// Discord.Net's <c>StopAsync</c> awaits a WebSocket close handshake. After
/// a socket-exhaustion event the handshake can never complete, and the
/// actor would otherwise sit in Disconnecting forever with no retry armed.
/// </summary>
protected override TimeSpan? StopTimeout => TimeSpan.FromSeconds(30);

protected override bool IsTransportConnected => _client.ConnectionState == ConnectionState.Connected;

protected override bool HasBotIdentity => _botUserId is not null && _botMentionTag is not null;
Expand Down
101 changes: 96 additions & 5 deletions src/Netclaw.Channels/ChannelLifecycleActor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@ public abstract class ChannelLifecycleActor<TSnapshot, TConnectCommand> : Receiv
private TimeSpan _retryDelay;
private ICancelable? _retryTimer;

/// <summary>
/// Monotonic stamp identifying the current stop cycle. A stop that times out
/// leaves its transport call running; when that call finally completes it
/// must not be mistaken for the result of a later stop cycle.
/// </summary>
private long _stopGeneration;
private ICancelable? _stopTimer;

/// <remarks>
/// The constructor calls <see cref="ActorBase.Become"/>, which invokes the
/// subclass's Register* hooks before the subclass constructor body has run.
Expand Down Expand Up @@ -125,6 +133,22 @@ protected string? HealthDetail
/// </summary>
protected virtual TimeSpan? ReadySignalTimeout => null;

/// <summary>
/// How long the actor waits in Disconnecting for the transport stop to
/// complete before treating the stop as failed and scheduling the next
/// auto-reconnect attempt. Null (the default) waits indefinitely.
/// </summary>
/// <remarks>
/// A transport stop that hangs — a half-open socket, a wedged WebSocket
/// close handshake, or a client whose internal reconnect loop never
/// settles — otherwise parks the actor in Disconnecting forever. That
/// state has no retry timer and discards transport recovery signals, so
/// the channel never reconnects until the process restarts. Discord's
/// <c>StopAsync</c> is the known offender: after a socket-exhaustion
/// event it can await a close handshake that never completes.
/// </remarks>
protected virtual TimeSpan? StopTimeout => null;

protected override void PreStart()
{
SelfRef = Self;
Expand All @@ -137,6 +161,7 @@ protected override void PostStop()
UnsubscribeTransportEvents();
CancelRetryTimer();
CancelReadySignalTimer();
CancelStopTimer();
base.PostStop();
}

Expand Down Expand Up @@ -205,6 +230,7 @@ private void Disconnecting()
_channelDisplayName + " gateway disconnect is already in progress."))));
Receive<StopSucceeded>(HandleStopSucceeded);
Receive<StopFailed>(HandleStopFailed);
Receive<StopTimedOut>(HandleStopTimedOut);
RegisterDisconnectingChannelHandlers();
ReceiveNotReadyIngress();
ReceiveUnexpected(nameof(Disconnecting));
Expand Down Expand Up @@ -245,6 +271,7 @@ private void StartConnecting(TConnectCommand command, IActorRef replyTo)
ResetIdentityState();
_cleanReconnectEmitted = false;
_readyEnteredAt = null;
CancelStopTimer();

// Normalize Nobody to null: "no pending caller" must have exactly one
// representation. The auto-retry path connects with ActorRefs.Nobody,
Expand Down Expand Up @@ -375,28 +402,82 @@ private void StartDisconnecting(IActorRef replyTo, bool preserveAutoReconnect =
private void BeginStop(IActorRef replyTo)
{
var self = Self;
var generation = ++_stopGeneration;
Task stopTask;
try
{
stopTask = StopTransportAsync();
}
catch (Exception ex)
{
self.Tell(new StopFailed(replyTo, ex));
self.Tell(new StopFailed(replyTo, ex, generation));
return;
}

ArmStopTimer(generation);

stopTask.ContinueWith(
task => self.Tell(task.IsCompletedSuccessfully
? new StopSucceeded(replyTo)
: new StopFailed(replyTo, UnwrapTaskException(task))),
? new StopSucceeded(replyTo, generation)
: new StopFailed(replyTo, UnwrapTaskException(task), generation)),
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}

/// <summary>
/// Arms the stop-completion deadline. When it fires, the actor treats the
/// stop as failed and moves on rather than waiting forever on a transport
/// call that may never return.
/// </summary>
private void ArmStopTimer(long generation)
{
CancelStopTimer();
if (StopTimeout is not { } timeout)
return;

_stopTimer = Context.System.Scheduler.ScheduleTellOnceCancelable(
timeout,
Self,
new StopTimedOut(generation, new ChannelConnectException(
ChannelConnectFailureKind.Transient,
_channelDisplayName + " gateway did not stop within " +
$"{timeout.TotalSeconds:F0} seconds.")),
ActorRefs.NoSender);
}

private void CancelStopTimer()
{
_stopTimer?.Cancel();
_stopTimer = null;
}

private void HandleStopTimedOut(StopTimedOut timedOut)
{
if (timedOut.Generation != _stopGeneration)
return;

// The transport call is abandoned, not cancelled: it may still be
// awaiting a close handshake. Its eventual completion carries the old
// generation and is ignored, so it cannot disturb the next cycle.
Logger.LogWarning(
"{Channel} gateway stop did not complete within the stop timeout; " +
"abandoning the stop and scheduling a reconnect. {Reason}",
_channelDisplayName, timedOut.Exception.Message);

_healthDetail = timedOut.Exception.Message;
CancelStopTimer();
ResetIdentityState();
ScheduleRetryIfEnabled();
Become(Disconnected);
}

private void HandleStopSucceeded(StopSucceeded stopped)
{
if (stopped.Generation != _stopGeneration)
return;

CancelStopTimer();
_healthDetail = _disconnectedDetail;
ResetIdentityState();
ScheduleRetryIfEnabled();
Expand All @@ -407,6 +488,10 @@ private void HandleStopSucceeded(StopSucceeded stopped)

private void HandleStopFailed(StopFailed failed)
{
if (failed.Generation != _stopGeneration)
return;

CancelStopTimer();
_healthDetail = failed.Exception.Message;
ScheduleRetryIfEnabled();
Become(Disconnected);
Expand Down Expand Up @@ -765,9 +850,15 @@ private sealed record StartFailed(long Attempt, Exception Exception) : IGatewayC
/// </summary>
private sealed record ReadySignalTimedOut(long Attempt, Exception Exception) : IGatewayConnectFailure;

private sealed record StopSucceeded(IActorRef ReplyTo) : IGatewayStopWork;
private sealed record StopSucceeded(IActorRef ReplyTo, long Generation) : IGatewayStopWork;

private sealed record StopFailed(IActorRef ReplyTo, Exception Exception, long Generation) : IGatewayStopFailure;

private sealed record StopFailed(IActorRef ReplyTo, Exception Exception) : IGatewayStopFailure;
/// <summary>
/// Fires when a transport stop exceeds <see cref="StopTimeout"/>. Carries the
/// stop generation so a late timer from an abandoned cycle is ignored.
/// </summary>
private sealed record StopTimedOut(long Generation, Exception Exception) : IGatewayInternalMessage;

private sealed record RetryConnect : IGatewayInternalMessage
{
Expand Down
Loading