diff --git a/README.md b/README.md index 78da6bd3..806d6144 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The general architecture of the core application goes along these lines: * For this general template, the job logic runner implementation is currently a stub that shall sleep for the requested number of seconds. * In the event that the job source implementation requires application-level heartbeats for long-running messages, a - heartbeat maintainer (`IHeartbeatMaintainer`) worker thread shall periodically heartbeat messages according to the job + heartbeat monitor (`IHeartbeatMonitor`) worker thread shall periodically heartbeat messages according to the job source's recommendation. * Generally, job sources that require heartbeats are told to recommend a heartbeat interval of 75% of the maximum in-flight time for a message without heartbeats. For example, an SQS consumer configured with a visibility timeout diff --git a/src/RedShirt.Example.JobWorker.Core/Extensions/ServiceCollectionExtensions.cs b/src/RedShirt.Example.JobWorker.Core/Extensions/ServiceCollectionExtensions.cs index a73fe76f..7b3b43ee 100644 --- a/src/RedShirt.Example.JobWorker.Core/Extensions/ServiceCollectionExtensions.cs +++ b/src/RedShirt.Example.JobWorker.Core/Extensions/ServiceCollectionExtensions.cs @@ -28,24 +28,23 @@ public static IServiceCollection AddCoreJobManagement(this IServiceCollection se services = services .AddCommon() + // Execution end arbiters + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() // General .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() - .AddSingleton(provider => - provider.GetRequiredService()) - .AddSingleton(provider => - provider.GetRequiredService()) - .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() .AddSingleton() .AddSingleton() .Configure(coreSection) diff --git a/src/RedShirt.Example.JobWorker.Core/Models/JobRepositoryEntry.cs b/src/RedShirt.Example.JobWorker.Core/Models/JobRepositoryEntry.cs index fd797432..abc34fe6 100644 --- a/src/RedShirt.Example.JobWorker.Core/Models/JobRepositoryEntry.cs +++ b/src/RedShirt.Example.JobWorker.Core/Models/JobRepositoryEntry.cs @@ -8,96 +8,192 @@ internal interface ISortableJobWrapper IJobModel JobModel { get; } } -internal interface IJobRepositoryEntry : ISortableJobWrapper +internal interface IJobRepositoryEntry : ISortableJobWrapper, IDisposable { + bool IsDisposed { get; } IRawJobModel RawJobModel { get; } - bool CanHeartbeat { get; } - DateTime LastHeartbeatTime { get; } - JobState State { get; } - Task SetAsCannotHeartbeatAsync(CancellationToken cancellationToken = default); - Task SetLastHeartbeatTimeAsync(DateTime lastHeartbeatTime, CancellationToken cancellationToken = default); - Task SetStateAsync(JobState state, CancellationToken cancellationToken = default); + /// + /// Whether this job can still receive heartbeats. + /// May only be set to false. + /// + bool CanHeartbeat { get; set; } + + DateTime LastHeartbeatTime { get; set; } + + /// + /// Current processing state of this job. + /// May not be set to null. + /// + /// Thrown when the setter is given null. + JobState? State { get; set; } } internal sealed class JobRepositoryEntry : IJobRepositoryEntry { /// - /// Thread-safety for mutable field access from maintainer, executor, and repository threads. + /// Thread-safety for mutable field access from executor, monitor, and repository threads. /// private readonly Lock _lock = new(); - private bool _canHeartbeat = true; - private DateTime _lastHeartbeatTime; - private JobState _state = JobState.Inactive; + private bool _disposed; - public required IRawJobModel RawJobModel { get; init; } - public required IJobModel JobModel { get; init; } + private Action? _stateCallbacks; - public bool CanHeartbeat + private void Dispose(bool disposing) { - get + if (!disposing) { - lock (_lock) + return; + } + + lock (_lock) + { + if (_disposed) { - return _canHeartbeat; + return; } + + _disposed = true; + } + + // Confirm that the job that we're removing is marked as complete, + // for the sake of subscriber callbacks in the underlying JobRepositoryEntry. + State = JobState.Complete; + + lock (_lock) + { + _stateCallbacks = null; } } - public DateTime LastHeartbeatTime + /// + /// Threadsafe indicator of being disposed. + /// + public bool IsDisposed { get { lock (_lock) { - return _lastHeartbeatTime; + return _disposed; } } } - public JobState State + public void Dispose() + { + Dispose(true); + // ReSharper disable once GCSuppressFinalizeForTypeWithoutDestructor + GC.SuppressFinalize(this); + } + + public required IRawJobModel RawJobModel { get; init; } + public required IJobModel JobModel { get; init; } + + public bool CanHeartbeat { get { lock (_lock) { - return _state; + return field; } } - } + set + { + if (value) + { + throw new ArgumentException("CanHeartbeat can only be set to false.", nameof(value)); + } + + lock (_lock) + { + field = false; + } + } + } = true; - public Task SetAsCannotHeartbeatAsync(CancellationToken cancellationToken = default) + public required DateTime LastHeartbeatTime { - cancellationToken.ThrowIfCancellationRequested(); - lock (_lock) + get { - _canHeartbeat = false; + lock (_lock) + { + return field; + } + } + set + { + lock (_lock) + { + field = value; + } } - - return Task.CompletedTask; } - public Task SetLastHeartbeatTimeAsync(DateTime lastHeartbeatTime, - CancellationToken cancellationToken = default) + public required JobState? State { - cancellationToken.ThrowIfCancellationRequested(); - lock (_lock) + get { - _lastHeartbeatTime = lastHeartbeatTime; + lock (_lock) + { + return field; + } } + set + { + if (value is not { } newState) + { + throw new ArgumentNullException(nameof(value)); + } - return Task.CompletedTask; - } + Action? callbacks; + JobState? original; + lock (_lock) + { + if (field == newState) + { + return; + } + + original = field; + field = newState; + callbacks = _stateCallbacks; + } + + callbacks?.Invoke(this, original, newState); + } + } = JobState.Inactive; - public Task SetStateAsync(JobState state, CancellationToken cancellationToken = default) + /// + /// Register a callback invoked with this entry plus the original and current . + /// Invoked immediately on subscribe when the current state is not null; + /// the original state is null for that first invocation. + /// + /// + /// Receives this entry, the original state (possibly null), and the current non-null state. + /// + /// Thrown when this entry has already been disposed. + public void SubscribeToState(Action action) { - cancellationToken.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(action); + + JobState? current; lock (_lock) { - _state = state; + if (_disposed) + { + throw new ObjectDisposedException(nameof(JobRepositoryEntry)); + } + + _stateCallbacks += action; + current = State; } - return Task.CompletedTask; + if (current is { } state) + { + action(this, null, state); + } } } \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/AppliedExecutionEndArbiter.cs b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/AppliedExecutionEndArbiter.cs deleted file mode 100644 index de95db5c..00000000 --- a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/AppliedExecutionEndArbiter.cs +++ /dev/null @@ -1,225 +0,0 @@ -using Microsoft.Extensions.Logging; -using RedShirt.Example.JobWorker.Common.Services.Utility; -using RedShirt.Example.JobWorker.Core.Services.Jobs; -using RedShirt.Example.JobWorker.Core.Utility; - -namespace RedShirt.Example.JobWorker.Core.Services.ExecutionState; - -/// -/// Dictates if maintainer workers should continue running. -/// Extends the functionality of the base IExecutionEndArbiter by accessing the job repository. -/// Initially written as a test-friendly alternative to `while(true){}` -/// -internal interface IAppliedMaintainerExecutionEndArbiter -{ - /// - /// Delays for , honouring both and - /// an internal interrupt signal that triggers when the worker is stopping. - /// If there are no watched jobs to monitor, then also delays until there are watched jobs to monitor before - /// delaying for . - /// Intended for maintainer workers only. - /// Cancellation caused by the internal interrupt signal is ignored and treated as a completed delay. - /// - /// How long to wait when the skip-wait event is not set. - /// Label for future wait-related log messages (unused for now). - /// Description for future wait-related log messages (unused for now). - /// Caller cancellation. - Task MaintainerDelayWaitAsync(TimeSpan delay, string loggerLabel, string loggerDescription, - CancellationToken cancellationToken = default); - - bool MaintainerShouldKeepRunning(); -} - -/// -/// Dictates if executor workers should continue running. -/// Extends the functionality of the base IExecutionEndArbiter by accessing the job repository. -/// Written as a test-friendly alternative to `while(true){}` -/// -internal interface IAppliedExecutorExecutionEndArbiter -{ - bool ExecutorsShouldKeepRunning(); -} - -internal sealed class AppliedExecutionEndArbiter : IAppliedMaintainerExecutionEndArbiter, - IAppliedExecutorExecutionEndArbiter, IDisposable -{ - private readonly IExecutionEndArbiter _executionEndArbiter; - private readonly CancellationTokenSource _interruptCts = new(); - private readonly Lock _lock = new(); - private readonly ILogger _logger; - private readonly ISleepService _sleepService; - private readonly AsyncManualResetEvent _watchedJobsToMaintainEvent = new(); - - private bool _disposed; - private int _inactiveJobsCount; - private int _watchedJobsCount; - - /// - /// Centralize decision on whether to send interrupt signal. - /// Unsafe on its own, assumed to be running within a lock statement by the method that invokes it. - /// - private bool ShouldSendMaintainerInterruptSignalUnsafe() - { - return !_executionEndArbiter.ShouldKeepRunning() - && _inactiveJobsCount == 0 - && _watchedJobsCount == 0; - } - - private void TryCancelInterrupt() - { - try - { - _interruptCts.Cancel(); - } - catch (ObjectDisposedException) - { - // Dispose may have already run (e.g. host shutdown); interrupt signalling is best-effort. - } - } - - private void OnInactiveJobChange(int inactiveJobCount) - { - bool shouldInterrupt; - lock (_lock) - { - if (_disposed) - { - return; - } - - _inactiveJobsCount = inactiveJobCount; - shouldInterrupt = ShouldSendMaintainerInterruptSignalUnsafe(); - } - - if (shouldInterrupt) - { - TryCancelInterrupt(); - } - } - - private void OnWatchedJobChange(int watchedJobCount) - { - bool shouldInterrupt; - int previousValue; - - lock (_lock) - { - if (_disposed) - { - return; - } - - previousValue = _watchedJobsCount; - _watchedJobsCount = watchedJobCount; - shouldInterrupt = ShouldSendMaintainerInterruptSignalUnsafe(); - } - - if (previousValue != watchedJobCount) - { - // Confirmed a change - - if (watchedJobCount == 0) - { - _watchedJobsToMaintainEvent.Reset(); - } - else - { - _watchedJobsToMaintainEvent.Set(); - } - } - - if (shouldInterrupt) - { - TryCancelInterrupt(); - } - } - - public AppliedExecutionEndArbiter( - IExecutionEndArbiter executionEndArbiter, - IJobRepository jobRepository, - ISleepService sleepService, - ILogger logger) - { - _executionEndArbiter = executionEndArbiter; - _sleepService = sleepService; - _logger = logger; - jobRepository.SubscribeToInactiveCountUpdate(OnInactiveJobChange); - jobRepository.SubscribeToWatchedJobsUpdate(OnWatchedJobChange); - } - - public bool ExecutorsShouldKeepRunning() - { - // The executor doesn't care about other executors currently processing jobs, so ignoring the watched jobs count. - lock (_lock) - { - return _executionEndArbiter.ShouldKeepRunning() - || _inactiveJobsCount > 0; - } - } - - public async Task MaintainerDelayWaitAsync(TimeSpan delay, string loggerLabel, string loggerDescription, - CancellationToken cancellationToken = default) - { - CancellationToken interruptToken; - lock (_lock) - { - if (_disposed) - { - return; - } - - interruptToken = _interruptCts.Token; - } - - using var linkedCts = - CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, interruptToken); - - try - { - // Immediately check to see if the event is set. - if (!await _watchedJobsToMaintainEvent.WaitAsync(TimeSpan.Zero, linkedCts.Token)) - { - // Event is not set, so wait until it is - // This wait prevents the maintainer from creating noise (trace-level though it may be) - // when there are no watched jobs to maintain. - _logger.LogTrace("{Label}: Waiting for watchable events", loggerLabel); - await _watchedJobsToMaintainEvent.WaitAsync(linkedCts.Token); - return; - } - - _logger.LogTrace("{Label}: {Time} until next {Description}", loggerLabel, delay, - loggerDescription); - await _sleepService.DelayAsync(delay, linkedCts.Token); - } - catch (OperationCanceledException) when (interruptToken.IsCancellationRequested - && !cancellationToken.IsCancellationRequested) - { - // Interrupt-driven cancellation: treat the delay as having elapsed. - } - } - - public bool MaintainerShouldKeepRunning() - { - lock (_lock) - { - return _executionEndArbiter.ShouldKeepRunning() - || _inactiveJobsCount > 0 - || _watchedJobsCount > 0; - } - } - - public void Dispose() - { - lock (_lock) - { - if (_disposed) - { - return; - } - - _disposed = true; - } - - _interruptCts.Dispose(); - } -} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutionEndArbiter.cs b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutionEndArbiter.cs index 50fc0cd4..6af37e6b 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutionEndArbiter.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutionEndArbiter.cs @@ -5,7 +5,7 @@ namespace RedShirt.Example.JobWorker.Core.Services.ExecutionState; /// /// Dictates if the app should continue running. -/// Originally written as a test-friendly alternative to `while(true){}` +/// Originally written as a test-friendly alternative to while(true){} /// public interface IExecutionEndArbiter : IDisposable { diff --git a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutorExecutionEndArbiter.cs b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutorExecutionEndArbiter.cs new file mode 100644 index 00000000..828b328b --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/ExecutorExecutionEndArbiter.cs @@ -0,0 +1,73 @@ +using RedShirt.Example.JobWorker.Core.Services.Jobs; + +namespace RedShirt.Example.JobWorker.Core.Services.ExecutionState; + +/// +/// Dictates if executor workers should continue running. +/// Extends the functionality of the base IExecutionEndArbiter by accessing the job repository. +/// Originally written as a test-friendly alternative to while(true){} +/// +internal interface IExecutorExecutionEndArbiter +{ + /// + /// Determine whether executor workers should keep running. + /// + /// true if executors should keep running, otherwise false + bool ExecutorsShouldKeepRunning(); +} + +internal sealed class ExecutorExecutionEndArbiter : IExecutorExecutionEndArbiter +{ + private readonly IExecutionEndArbiter _executionEndArbiter; + private readonly Lock _lock = new(); + private int _idempotencyBlockedJobsCount; + + private int _inactiveJobsCount; + + private void OnInactiveJobCountChange(int inactiveJobCount) + { + lock (_lock) + { + _inactiveJobsCount = inactiveJobCount; + } + } + + private void OnIdempotencyBlockedJobsCountChange(int idempotencyBlockedJobsCount) + { + lock (_lock) + { + _idempotencyBlockedJobsCount = idempotencyBlockedJobsCount; + } + } + + /// + /// Determine whether the monitor should keep running. + /// Assumed to be running in a lock statement. + /// + /// true if the monitor should keep running, otherwise false + private bool ShouldKeepRunningUnsafe() + { + return _executionEndArbiter.ShouldKeepRunning() + // Tracking inactive jobs + || _inactiveJobsCount > 0 + // Tracking jobs that may become inactive again + || _idempotencyBlockedJobsCount > 0; + } + + public ExecutorExecutionEndArbiter(IJobRepository jobRepository, IExecutionEndArbiter executionEndArbiter) + { + _executionEndArbiter = executionEndArbiter; + // Track inactive jobs + jobRepository.SubscribeToInactiveCountUpdate(OnInactiveJobCountChange); + // Track jobs that may become inactive again + jobRepository.SubscribeToIdempotencyBlockedCountUpdate(OnIdempotencyBlockedJobsCountChange); + } + + public bool ExecutorsShouldKeepRunning() + { + lock (_lock) + { + return ShouldKeepRunningUnsafe(); + } + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiter.cs b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiter.cs new file mode 100644 index 00000000..6e912ef2 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiter.cs @@ -0,0 +1,189 @@ +using Microsoft.Extensions.Logging; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Core.Services.Jobs; +using RedShirt.Example.JobWorker.Core.Utility; + +namespace RedShirt.Example.JobWorker.Core.Services.ExecutionState; + +/// +/// Dictates if the heartbeat monitor should continue running. +/// Extends the functionality of the base IExecutionEndArbiter by accessing the job repository. +/// Originally written as a test-friendly alternative to while(true){} +/// +internal interface IHeartbeatMonitorExecutionEndArbiter : IDisposable +{ + /// + /// Delays for , honouring both and + /// an internal interrupt signal that triggers when the worker is stopping. + /// If there are no watched jobs to monitor, then also delays until there are watched jobs to monitor before + /// delaying for . + /// Cancellation caused by the internal interrupt signal is ignored and treated as a completed delay. + /// + /// How long to wait when the skip-wait event is not set. + /// Caller cancellation. + Task HeartbeatMonitorDelayWaitAsync(TimeSpan delay, CancellationToken cancellationToken = default); + + /// + /// Determine whether the heartbeat monitor should keep running. + /// + /// true if the monitor should keep running, otherwise false + bool MonitorShouldKeepRunning(); +} + +internal sealed class HeartbeatMonitorExecutionEndArbiter : IHeartbeatMonitorExecutionEndArbiter +{ + private const string LogLabel = "Heartbeat Monitor"; + private readonly IExecutionEndArbiter _executionEndArbiter; + private readonly CancellationTokenSource _interruptCts = new(); + private readonly Lock _lock = new(); + private readonly ILogger _logger; + private readonly AsyncManualResetEvent _relevantJobsToObserveEvent = new(); + private readonly ISleepService _sleepService; + private bool _disposed; + private bool _relevantJobsToObserveEventIsActive; + + private int _watchedJobsCount; + + private void OnWatchedJobsCountChange(int watchedJobCount) + { + bool shouldInterrupt; + lock (_lock) + { + if (_disposed) + { + return; + } + + _watchedJobsCount = watchedJobCount; + shouldInterrupt = !ShouldKeepRunningUnsafe(); + ConsiderUpdatingEventUnsafe(); + } + + if (shouldInterrupt) + { + TryCancelInterrupt(); + } + } + + private void ConsiderUpdatingEventUnsafe() + { + if (_watchedJobsCount == 0) + { + // Set to zero from non-zero + _relevantJobsToObserveEvent.Reset(); + _relevantJobsToObserveEventIsActive = false; + } + else if (!_relevantJobsToObserveEventIsActive) + { + // Set to non-zero, and was not previously active (suggesting from zero) + _relevantJobsToObserveEvent.Set(); + _relevantJobsToObserveEventIsActive = true; + } + } + + /// + /// Determine whether the monitor should keep running. + /// Assumed to be running in a lock statement. + /// + /// true if the monitor should keep running, otherwise false + private bool ShouldKeepRunningUnsafe() + { + return _executionEndArbiter.ShouldKeepRunning() + // All watched jobs need to be under observation for heartbeats + || _watchedJobsCount > 0; + } + + private void TryCancelInterrupt() + { + try + { + _interruptCts.Cancel(); + } + catch (ObjectDisposedException) + { + // Dispose may have already run (e.g. host shutdown); interrupt signalling is best-effort. + } + } + + private void Dispose(bool disposing) + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + } + + if (disposing) + { + _interruptCts.Dispose(); + } + } + + public HeartbeatMonitorExecutionEndArbiter(IJobRepository jobRepository, IExecutionEndArbiter executionEndArbiter, + ISleepService sleepService, ILogger logger) + { + _executionEndArbiter = executionEndArbiter; + _logger = logger; + _sleepService = sleepService; + // All watched jobs are candidates for heartbeats + jobRepository.SubscribeToWatchedJobsUpdate(OnWatchedJobsCountChange); + } + + public void Dispose() + { + Dispose(true); + // ReSharper disable once GCSuppressFinalizeForTypeWithoutDestructor + GC.SuppressFinalize(this); + } + + public async Task HeartbeatMonitorDelayWaitAsync(TimeSpan delay, CancellationToken cancellationToken = default) + { + CancellationToken interruptToken; + lock (_lock) + { + if (_disposed) + { + return; + } + + interruptToken = _interruptCts.Token; + } + + using var linkedCts = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, interruptToken); + + try + { + // Immediately check to see if the event is set. + if (!await _relevantJobsToObserveEvent.WaitAsync(TimeSpan.Zero, linkedCts.Token)) + { + // Event is not set, so wait until it is + // This wait prevents the monitor from creating noise (trace-level though it may be) + // when there are no watched jobs to maintain. + _logger.LogTrace("{LogLabel}: Waiting for watchable events", LogLabel); + await _relevantJobsToObserveEvent.WaitAsync(linkedCts.Token); + return; + } + + _logger.LogTrace("{LogLabel}: {Time} until next heartbeat check", LogLabel, delay); + await _sleepService.DelayAsync(delay, linkedCts.Token); + } + catch (OperationCanceledException) when (interruptToken.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + // Interrupt-driven cancellation: treat the delay as having elapsed. + } + } + + public bool MonitorShouldKeepRunning() + { + lock (_lock) + { + return ShouldKeepRunningUnsafe(); + } + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiter.cs b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiter.cs new file mode 100644 index 00000000..3de426e4 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Core/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiter.cs @@ -0,0 +1,210 @@ +using Microsoft.Extensions.Logging; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Core.Services.Jobs; +using RedShirt.Example.JobWorker.Core.Utility; + +namespace RedShirt.Example.JobWorker.Core.Services.ExecutionState; + +/// +/// Dictates if the idempotency monitor should continue running. +/// Extends the functionality of the base IExecutionEndArbiter by accessing the job repository. +/// Originally written as a test-friendly alternative to while(true){} +/// +internal interface IIdempotencyMonitorExecutionEndArbiter : IDisposable +{ + /// + /// Delays for , honouring both and + /// an internal interrupt signal that triggers when the worker is stopping. + /// If there are no blocked jobs for the idempotency monitor to observe, then also delays until there are + /// before delaying for . + /// Cancellation caused by the internal interrupt signal is ignored and treated as a completed delay. + /// + /// How long to wait when the skip-wait event is not set. + /// Caller cancellation. + Task IdempotencyMonitorDelayWaitAsync(TimeSpan delay, CancellationToken cancellationToken = default); + + /// + /// Determine whether the idempotency monitor should keep running. + /// + /// true if the monitor should keep running, otherwise false + bool MonitorShouldKeepRunning(); +} + +internal sealed class IdempotencyMonitorExecutionEndArbiter : IIdempotencyMonitorExecutionEndArbiter +{ + private const string LogLabel = "Idempotency Monitor"; + private readonly IExecutionEndArbiter _executionEndArbiter; + private readonly CancellationTokenSource _interruptCts = new(); + private readonly Lock _lock = new(); + private readonly ILogger _logger; + private readonly AsyncManualResetEvent _relevantJobsToObserveEvent = new(); + private readonly ISleepService _sleepService; + private bool _disposed; + + private int _idempotencyBlockedJobs; + private bool _relevantJobsToObserveEventIsActive; + private int _watchedJobsCount; + + private void OnWatchedJobsCountChange(int watchedJobCount) + { + bool shouldInterrupt; + lock (_lock) + { + if (_disposed) + { + return; + } + + _watchedJobsCount = watchedJobCount; + shouldInterrupt = !ShouldKeepRunningUnsafe(); + ConsiderUpdatingEventUnsafe(); + } + + if (shouldInterrupt) + { + TryCancelInterrupt(); + } + } + + private void OnIdempotencyBlockedJobsCountChange(int idempotencyBlockedJobsCount) + { + bool shouldInterrupt; + lock (_lock) + { + if (_disposed) + { + return; + } + + _idempotencyBlockedJobs = idempotencyBlockedJobsCount; + shouldInterrupt = !ShouldKeepRunningUnsafe(); + ConsiderUpdatingEventUnsafe(); + } + + if (shouldInterrupt) + { + TryCancelInterrupt(); + } + } + + private void ConsiderUpdatingEventUnsafe() + { + if (_idempotencyBlockedJobs == 0) + { + // Set to zero from non-zero + _relevantJobsToObserveEvent.Reset(); + _relevantJobsToObserveEventIsActive = false; + } + else if (!_relevantJobsToObserveEventIsActive) + { + // Set to non-zero, and was not previously active (suggesting from zero) + _relevantJobsToObserveEvent.Set(); + _relevantJobsToObserveEventIsActive = true; + } + } + + /// + /// Determine whether the monitor should keep running. + /// Assumed to be running in a lock statement. + /// + /// true if the monitor should keep running, otherwise false + private bool ShouldKeepRunningUnsafe() + { + return _executionEndArbiter.ShouldKeepRunning() + || _watchedJobsCount > 0; + } + + private void TryCancelInterrupt() + { + try + { + _interruptCts.Cancel(); + } + catch (ObjectDisposedException) + { + // Dispose may have already run (e.g. host shutdown); interrupt signalling is best-effort. + } + } + + private void Dispose(bool disposing) + { + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + } + + if (disposing) + { + _interruptCts.Dispose(); + } + } + + public IdempotencyMonitorExecutionEndArbiter(IJobRepository jobRepository, IExecutionEndArbiter executionEndArbiter, + ISleepService sleepService, ILogger logger) + { + _executionEndArbiter = executionEndArbiter; + _logger = logger; + _sleepService = sleepService; + jobRepository.SubscribeToWatchedJobsUpdate(OnWatchedJobsCountChange); + jobRepository.SubscribeToIdempotencyBlockedCountUpdate(OnIdempotencyBlockedJobsCountChange); + } + + public bool MonitorShouldKeepRunning() + { + lock (_lock) + { + return ShouldKeepRunningUnsafe(); + } + } + + public async Task IdempotencyMonitorDelayWaitAsync(TimeSpan delay, CancellationToken cancellationToken = default) + { + CancellationToken interruptToken; + lock (_lock) + { + if (_disposed) + { + return; + } + + interruptToken = _interruptCts.Token; + } + + using var linkedCts = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, interruptToken); + + try + { + // Immediately check to see if the event is set. + if (!await _relevantJobsToObserveEvent.WaitAsync(TimeSpan.Zero, linkedCts.Token)) + { + // Event is not set, so wait until it is + // This wait prevents the monitor from creating noise (trace-level though it may be) + // when there are no watched jobs to maintain. + _logger.LogTrace("{LogLabel}: Waiting for watchable events", LogLabel); + await _relevantJobsToObserveEvent.WaitAsync(linkedCts.Token); + return; + } + + _logger.LogTrace("{LogLabel}: {Time} until next follow-up check", LogLabel, delay); + await _sleepService.DelayAsync(delay, linkedCts.Token); + } + catch (OperationCanceledException) when (interruptToken.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + // Interrupt-driven cancellation: treat the delay as having elapsed. + } + } + + public void Dispose() + { + Dispose(true); + // ReSharper disable once GCSuppressFinalizeForTypeWithoutDestructor + GC.SuppressFinalize(this); + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Handler.cs b/src/RedShirt.Example.JobWorker.Core/Services/Handler.cs index 6129e030..4d744860 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Handler.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Handler.cs @@ -31,7 +31,7 @@ public interface IHandler /// /// /// -/// +/// /// /// /// @@ -39,7 +39,7 @@ public interface IHandler internal sealed class Handler( IExecutionEndArbiter executionEndArbiter, IJobLoaderLoop jobLoaderLoop, - IHeartbeatMaintainer heartbeatMaintainer, + IHeartbeatMonitor heartbeatMonitor, IJobExecutor jobExecutor, IIdempotencyMonitor idempotencyMonitor, IJobSubscriberManager jobSubscriberManager, @@ -162,13 +162,13 @@ await addToTaskFuncAsync(WorkerThreadType.JobSubscriberManager, } /* - * Note: The Maintainer and Idempotency Monitor tasks are intended to abort immediately if configuration or choice of job source doesn't require them. + * Note: The Heartbeat Monitor and Idempotency Monitor tasks are intended to abort immediately if configuration or choice of job source doesn't require them. * It made for simpler execution in Handler to just run them and add them to the list. */ - // Maintainer thread - await addToTaskFuncAsync(WorkerThreadType.HeartbeatMaintainer, - () => heartbeatMaintainer.RunAsync(cancellationToken)); + // Heartbeat monitor thread + await addToTaskFuncAsync(WorkerThreadType.HeartbeatMonitor, + () => heartbeatMonitor.RunAsync(cancellationToken)); // Idempotency monitor thread await addToTaskFuncAsync(WorkerThreadType.IdempotencyMonitor, @@ -205,7 +205,7 @@ await addToTaskFuncAsync(WorkerThreadType.IdempotencyMonitor, private enum WorkerThreadType { JobExecutor, - HeartbeatMaintainer, + HeartbeatMonitor, IdempotencyMonitor, JobSubscriberManager, MessagePoller diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatCalculator.cs b/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatCalculator.cs index 33903f19..f27443a8 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatCalculator.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatCalculator.cs @@ -4,7 +4,7 @@ namespace RedShirt.Example.JobWorker.Core.Services.Heartbeats; /// -/// The abstracted heartbeat checks exist to make reading/testing the code of the Maintainer implementation simpler. +/// The abstracted heartbeat checks exist to make reading/testing the code of the Monitor implementation simpler. /// internal interface IHeartbeatCalculator { diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMaintainer.cs b/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMonitor.cs similarity index 88% rename from src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMaintainer.cs rename to src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMonitor.cs index f06cb14f..04dcef1d 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMaintainer.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Heartbeats/HeartbeatMonitor.cs @@ -15,28 +15,28 @@ namespace RedShirt.Example.JobWorker.Core.Services.Heartbeats; /// -/// The maintainer is responsible for making sure that messages checked out from the job source remain 'in flight'. +/// The monitor is responsible for making sure that messages checked out from the job source remain 'in flight'. /// -internal interface IHeartbeatMaintainer : IHandlerSubComponent; +internal interface IHeartbeatMonitor : IHandlerSubComponent; #pragma warning disable S107 -internal sealed class HeartbeatMaintainer( +internal sealed class HeartbeatMonitor( IHeartbeatCalculator heartbeatCalculator, - IAppliedMaintainerExecutionEndArbiter appliedExecutionEndArbiter, + IHeartbeatMonitorExecutionEndArbiter heartbeatExecutionEndArbiter, IJobRepository jobRepository, IJobSource jobSource, ICoreHealthStateUpdateService healthStateUpdateService, ISleepService sleepService, IOptions coreOptions, - ILogger logger) : IHeartbeatMaintainer + ILogger logger) : IHeartbeatMonitor #pragma warning restore S107 { /// /// Set the minimum amount of time to sleep for between loops. - /// The heartbeat maintainer loop uses this to round values up to 500ms to avoid possible inching + /// The heartbeat monitor loop uses this to round values up to 500ms to avoid possible inching /// to the next heartbeat check because of what I think is date comparison imprecision, /// Task.Delay imprecision, or something similar. - /// In local testing, had a situation where the HeartbeatMaintainer slept for 00:00:00.0013576, + /// In local testing, had a situation where the HeartbeatMonitor slept for 00:00:00.0013576, /// then for 00:00:00.0002317, and so on for ~15 more times until it finally reached /// the actual heartbeat threshold. /// Mitigating that issue by setting a minimum. The recommended @@ -56,8 +56,7 @@ internal sealed class HeartbeatMaintainer( /// private async Task LogAndWaitAsync(TimeSpan timeToWait, CancellationToken cancellationToken = default) { - await appliedExecutionEndArbiter.MaintainerDelayWaitAsync(timeToWait, "Heartbeat Monitor", "heartbeat check", - cancellationToken); + await heartbeatExecutionEndArbiter.HeartbeatMonitorDelayWaitAsync(timeToWait, cancellationToken); } /// @@ -114,7 +113,7 @@ await sleepService.DelayAsync(TimeSpan.FromSeconds(Math.Pow(2, args.AttemptNumbe await GetRetryPipeline().ExecuteAsync( async token => await jobSource.HeartbeatAsync(jobRepositoryEntry.RawJobModel, token), cancellationToken); - await jobRepositoryEntry.SetLastHeartbeatTimeAsync(DateTime.UtcNow, cancellationToken); + jobRepositoryEntry.LastHeartbeatTime = DateTime.UtcNow; } catch (WorkerJobSourceException e) { @@ -124,7 +123,7 @@ await GetRetryPipeline().ExecuteAsync( // by the time the next loop iteration comes around. // // The documented recommendation for a heartbeat interval is ~75% of the time until message expiry - await jobRepositoryEntry.SetAsCannotHeartbeatAsync(cancellationToken); + jobRepositoryEntry.CanHeartbeat = false; } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -136,7 +135,7 @@ await GetRetryPipeline().ExecuteAsync( throw; } - await jobRepositoryEntry.SetAsCannotHeartbeatAsync(cancellationToken); + jobRepositoryEntry.CanHeartbeat = false; } return heartbeatCalculator.TimeUntilNextHeartbeat(jobRepositoryEntry); @@ -177,7 +176,7 @@ public async Task RunAsync(CancellationToken cancellat return HandlerComponentResponse.NotEnabled; } - while (appliedExecutionEndArbiter.MaintainerShouldKeepRunning()) + while (heartbeatExecutionEndArbiter.MonitorShouldKeepRunning()) { var jobs = await jobRepository.GetAllInFlightJobsAsync(cancellationToken); diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Idempotency/IdempotencyMonitor.cs b/src/RedShirt.Example.JobWorker.Core/Services/Idempotency/IdempotencyMonitor.cs index 78d39c8d..c459d56e 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Idempotency/IdempotencyMonitor.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Idempotency/IdempotencyMonitor.cs @@ -19,7 +19,7 @@ namespace RedShirt.Example.JobWorker.Core.Services.Idempotency; internal interface IIdempotencyMonitor : IHandlerSubComponent; internal sealed class IdempotencyMonitor( - IAppliedMaintainerExecutionEndArbiter executionEndArbiter, + IIdempotencyMonitorExecutionEndArbiter idempotencyMonitorExecutionEndArbiter, IJobRepository jobRepository, IIdempotencyExecutionService idempotencyExecutionService, ISafeJobAcknowledgementService safeJobAcknowledgementService, @@ -97,7 +97,7 @@ await idempotencyExecutionService.SetResultInCacheAsync(blockedJob.RawJobModel, if (unblockedJob is { } jobToUnblock) { /* - * I'm invoking the reload operation in this point in the loop out of fear of an infinite cycle of idempotency monitoring. + * I'm changing the job's state at this point in the loop out of fear of an infinite cycle of idempotency monitoring. * * If the job were reloaded within the idempotency lock, then there would be the potential of a race condition. * If the JobExecutor thread receives job and attempted to acquire an idempotency lock before @@ -107,18 +107,17 @@ await idempotencyExecutionService.SetResultInCacheAsync(blockedJob.RawJobModel, * Instead, we are very deliberately doing this outside of the idempotency lock. */ - await jobRepository.ReloadUnblockedJobAsync(jobToUnblock, cancellationToken); + jobToUnblock.State = JobState.Inactive; } } } /// - /// Minor centralization of a log message, mirroring HeartbeatMaintainer. + /// Minor centralization of a log message, mirroring HeartbeatMonitor. /// private async Task LogAndWaitAsync(TimeSpan timeToWait, CancellationToken cancellationToken = default) { - await executionEndArbiter.MaintainerDelayWaitAsync(timeToWait, "Idempotency Monitor", "follow-up check", - cancellationToken); + await idempotencyMonitorExecutionEndArbiter.IdempotencyMonitorDelayWaitAsync(timeToWait, cancellationToken); } public async Task RunAsync(CancellationToken cancellationToken = default) @@ -131,7 +130,7 @@ public async Task RunAsync(CancellationToken cancellat var intervalTimeSpan = TimeSpan.FromSeconds(options.Value.EffectiveMonitorIntervalSeconds); - while (executionEndArbiter.MaintainerShouldKeepRunning()) + while (idempotencyMonitorExecutionEndArbiter.MonitorShouldKeepRunning()) { await CheckBlockedJobsAsync(cancellationToken); await LogAndWaitAsync(intervalTimeSpan, cancellationToken); diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobExecutor.cs b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobExecutor.cs index 066cb38b..978ba02a 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobExecutor.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobExecutor.cs @@ -25,7 +25,7 @@ internal interface IJobExecutor } internal sealed class JobExecutor( - IAppliedExecutorExecutionEndArbiter appliedExecutionEndArbiter, + IExecutorExecutionEndArbiter appliedExecutionEndArbiter, IJobRepository jobRepository, IIdempotencyExecutionService idempotencyExecutionService, ISafeJobRunner safeJobRunner, @@ -127,7 +127,7 @@ public async Task RunAsync(int executorId, Cancellatio logger.LogTrace( "Executor {Id} was unable to obtain a lock on message {MessageId} , deferring to Idempotency Monitor", executorId, repositoryEntry.JobModel.MessageId); - await repositoryEntry.SetStateAsync(JobState.BlockedByIdempotency, cancellationToken); + repositoryEntry.State = JobState.BlockedByIdempotency; continue; } @@ -136,7 +136,7 @@ public async Task RunAsync(int executorId, Cancellatio // Mark as complete for all branches of ActOnJobAsync by doing it afterwards // Reminder that JobState does not imply anything about success or acknowledgement success. // It only means that the JobWorker is done with the job. - await repositoryEntry.SetStateAsync(JobState.Complete, cancellationToken); + repositoryEntry.State = JobState.Complete; await jobRepository.RemoveJobAsync(repositoryEntry, cancellationToken); } finally diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobRepository.cs b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobRepository.cs index 825c3f39..c5208655 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobRepository.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/JobRepository.cs @@ -36,10 +36,14 @@ internal interface IJobRepository Task LoadAsync(IReadOnlyList intakeItems, CancellationToken cancellationToken = default); - Task ReloadUnblockedJobAsync(IJobRepositoryEntry job, CancellationToken cancellationToken = default); - Task RemoveJobAsync(IJobRepositoryEntry job, CancellationToken cancellationToken = default); + /// + /// Register a callback invoked with the current count of jobs blocked by idempotency whenever + /// that count changes via repository operations. Invoked immediately with the current count on subscribe. + /// + void SubscribeToIdempotencyBlockedCountUpdate(Action callback); + /// /// Register a callback invoked with the current inactive-job count whenever that count changes /// via repository operations. Invoked immediately with the current count on subscribe. @@ -84,6 +88,8 @@ internal sealed class JobRepository( /// private readonly AsyncManualResetEvent _repositoryEmptyEvent = new(true); + private readonly Lock _tallyLock = new(); + /// /// Jobs that have recently been unblocked due to an idempotency lock. /// This queue intended as a shortlist that will jump the normal sorted line of the inactive jobs list. @@ -92,17 +98,24 @@ internal sealed class JobRepository( private readonly SemaphoreSlim _watchedJobsListSemaphore = new(1, 1); + private Action? _idempotencyBlockedJobsCallbacks; + + private int _idempotencyBlockedTally; + private Action? _inactiveCountCallbacks; /// - /// Inactive potential jobs + /// Inactive potential jobs. /// Reminder: This is currently a list instead of a queue because it needs to be sorted in a manner that is consistent - /// with the Batch approach + /// with the Batch approach. /// Similarly, confirming that it is intentional that this list not be marked as readonly. /// private List _inactiveJobsList = []; + private int _inactiveJobsTally; + private Action? _watchedJobsCallbacks; + private int _watchedJobsTally; private void NotifyInactiveCountUpdate(int count) { @@ -115,6 +128,17 @@ private void NotifyInactiveCountUpdate(int count) callbacks?.Invoke(count); } + private void NotifyIdempotencyBlockedCountUpdate(int count) + { + Action? callbacks; + lock (_callbackLock) + { + callbacks = _idempotencyBlockedJobsCallbacks; + } + + callbacks?.Invoke(count); + } + private void NotifyWatchedJobsUpdate(int count) { Action? callbacks; @@ -128,27 +152,49 @@ private void NotifyWatchedJobsUpdate(int count) private async Task TryGetUnblockedJobAsync(CancellationToken cancellationToken) { - if (!_unblockedJobsQueue.TryDequeue(out var result)) + IJobRepositoryEntry? result; + var iterated = false; + + while (_unblockedJobsQueue.TryDequeue(out result)) { - return new TryGetJobResponse + iterated = true; + // Handle potential edge case of something disposing an item that was recently unblocked + // This absolutely should not happen, but at least it won't mess things up further if it does. + + if (!result.IsDisposed) { - Success = false, - Result = null - }; + break; + } } - await _inactiveJobsListSemaphore.WaitAsync(cancellationToken); - try + if (iterated) { - if (_inactiveJobsList.Count == 0 && _unblockedJobsQueue.IsEmpty) + // Check to see if we emptied the queue, but only if we actually dequeued something + // Assume that the event is up to date and doesn't need a redundant reset. + await _inactiveJobsListSemaphore.WaitAsync(cancellationToken); + try { - // Jobs are no longer available - _jobsAvailableEvent.Reset(); + if (_inactiveJobsList.Count == 0 && _unblockedJobsQueue.IsEmpty) + { + // Jobs are no longer available + _jobsAvailableEvent.Reset(); + } + } + finally + { + _inactiveJobsListSemaphore.Release(); } } - finally + + if (result is null + // Account for technical race condition, will never happen in practice + || result.IsDisposed) { - _inactiveJobsListSemaphore.Release(); + return new TryGetJobResponse + { + Success = false, + Result = null + }; } return new TryGetJobResponse @@ -189,6 +235,121 @@ private async Task TryGetInactiveJobAsync(CancellationToken c }; } + /// + /// Specifically handle transition from idempotency-blocked back to inactive. + /// + /// + /// + /// + private void OnEntryStateUpdateUnblocked(IJobRepositoryEntry job, JobState? oldState, JobState newState) + { + if (oldState != JobState.BlockedByIdempotency || newState != JobState.Inactive) + { + return; + } + + // Identified as a newly-unblocked job. + // Shortlist the job for re-execution in memory. + _unblockedJobsQueue.Enqueue(job); + // Tell any active invocations of GetNextJobAsync that there is something available. + _jobsAvailableEvent.Set(); + } + + /// + /// Handle tally management when an entry changes state. + /// + /// + /// + /// + private void OnEntryStateUpdateTallies(IJobRepositoryEntry job, JobState? oldState, JobState newState) + { + _ = job; + + var updatedWatched = false; + var updatedInactive = false; + var updatedIdempotencyBlocked = false; + + var localTallyWatched = 0; + var localTallyInactive = 0; + var localTallyIdempotencyBlocked = 0; + + lock (_tallyLock) + { + /* Track watched tally */ + + if (oldState is not null && newState == JobState.Complete) + { + // Moving from watched to unwatched (as opposed to directly to Complete, which would skip watching altogether) + updatedWatched = true; + _watchedJobsTally--; + } + else if (oldState is null && newState != JobState.Complete) + { + // Moving from unwatched to watched (as opposed to directly to Complete) + updatedWatched = true; + _watchedJobsTally++; + } + + /* Track individual tallies */ + + // ReSharper disable once SwitchStatementMissingSomeEnumCasesNoDefault + switch (oldState) + { + case JobState.Inactive: + _inactiveJobsTally--; + updatedInactive = true; + break; + case JobState.BlockedByIdempotency: + _idempotencyBlockedTally--; + updatedIdempotencyBlocked = true; + break; + } + + // ReSharper disable once SwitchStatementMissingSomeEnumCasesNoDefault + switch (newState) + { + case JobState.Inactive: + _inactiveJobsTally++; + updatedInactive = true; + break; + case JobState.BlockedByIdempotency: + _idempotencyBlockedTally++; + updatedIdempotencyBlocked = true; + break; + } + + if (updatedInactive) + { + localTallyInactive = _inactiveJobsTally; + } + + if (updatedIdempotencyBlocked) + { + localTallyIdempotencyBlocked = _idempotencyBlockedTally; + } + + if (updatedWatched) + { + localTallyWatched = _watchedJobsTally; + } + } + + if (updatedInactive) + { + NotifyInactiveCountUpdate(localTallyInactive); + } + + if (updatedIdempotencyBlocked) + { + NotifyIdempotencyBlockedCountUpdate(localTallyIdempotencyBlocked); + } + + if (updatedWatched) + { + NotifyWatchedJobsUpdate(localTallyWatched); + } + } + internal List WatchedJobs { get; } = []; public async Task> GetAllInFlightJobsAsync(CancellationToken cancellationToken = default) @@ -232,9 +393,15 @@ public async Task GetWatchedJobsCountAsync(CancellationToken cancellationTo { await _watchedJobsListSemaphore.WaitAsync(cancellationToken); - var count = WatchedJobs.Count; - - _watchedJobsListSemaphore.Release(); + int count; + try + { + count = WatchedJobs.Count; + } + finally + { + _watchedJobsListSemaphore.Release(); + } return count; } @@ -288,8 +455,7 @@ public async Task GetWatchedJobsCountAsync(CancellationToken cancellationTo await _jobsAvailableEvent.WaitAsync(TimeSpan.FromMilliseconds(250), cancellationToken); } while (result is null); - await result.SetStateAsync(JobState.Active, cancellationToken); - NotifyInactiveCountUpdate(await GetInactiveJobCountAsync(cancellationToken)); + result.State = JobState.Active; return result; } @@ -317,9 +483,12 @@ public async Task LoadAsync(IReadOnlyList intakeItems, var job = new JobRepositoryEntry { JobModel = envelope.JobModel, - RawJobModel = envelope.RawJobModel + RawJobModel = envelope.RawJobModel, + LastHeartbeatTime = DateTime.UtcNow, + State = JobState.Inactive }; - await job.SetLastHeartbeatTimeAsync(DateTime.UtcNow, cancellationToken); + job.SubscribeToState(OnEntryStateUpdateTallies); + job.SubscribeToState(OnEntryStateUpdateUnblocked); _inactiveJobsList.Add(job); // Worry about sorting later, see below @@ -352,34 +521,40 @@ public async Task LoadAsync(IReadOnlyList intakeItems, _jobsAvailableEvent.Set(); NotifyWatchedJobsUpdate(await GetWatchedJobsCountAsync(cancellationToken)); - NotifyInactiveCountUpdate(await GetInactiveJobCountAsync(cancellationToken)); - } - - public async Task ReloadUnblockedJobAsync(IJobRepositoryEntry job, CancellationToken cancellationToken = default) - { - await job.SetStateAsync(JobState.Inactive, cancellationToken); - // Shortlist the job for re-execution in memory - _unblockedJobsQueue.Enqueue(job); - // Tell any active invocations of GetNextJobAsync that there is something available. - _jobsAvailableEvent.Set(); - NotifyInactiveCountUpdate(await GetInactiveJobCountAsync(cancellationToken)); } public async Task RemoveJobAsync(IJobRepositoryEntry job, CancellationToken cancellationToken = default) { - int watchedCount; - int inactiveCount; + ArgumentNullException.ThrowIfNull(job); + + await _inactiveJobsListSemaphore.WaitAsync(cancellationToken); + try + { + if (_inactiveJobsList.Remove(job) + && _inactiveJobsList.Count == 0 + && _unblockedJobsQueue.IsEmpty) + { + // Jobs are no longer available + _jobsAvailableEvent.Reset(); + } + } + finally + { + _inactiveJobsListSemaphore.Release(); + } + await _watchedJobsListSemaphore.WaitAsync(cancellationToken); try { WatchedJobs.Remove(job); - watchedCount = WatchedJobs.Count; - inactiveCount = WatchedJobs.Count(watchedJob => watchedJob.State == JobState.Inactive); - if (watchedCount == 0) + if (WatchedJobs.Count == 0) { // Avoid possible race condition in JobLoader + + // If there's nothing to grab, then there must be an executor about to demand something. _jobsDemandEvent.Set(); + // No watched jobs is the very definition of an empty repository _repositoryEmptyEvent.Set(); } } @@ -388,8 +563,7 @@ public async Task RemoveJobAsync(IJobRepositoryEntry job, CancellationToken canc _watchedJobsListSemaphore.Release(); } - NotifyWatchedJobsUpdate(watchedCount); - NotifyInactiveCountUpdate(inactiveCount); + job.Dispose(); } public void SubscribeToInactiveCountUpdate(Action callback) @@ -401,29 +575,24 @@ public void SubscribeToInactiveCountUpdate(Action callback) _inactiveCountCallbacks += callback; } - /* - * Putting it on the record that I don't particularly like the below implementation on principle. - * It uses a blocking semaphore call, and it duplicates tally logic (especially true for this particular method). - * - * That said, I think the cure would be worse than the disease: - * * Implementing a check specifically for inactive jobs in this method's sibling - * SubscribeToInactiveCountUpdate would need some sort of tracker on the individual - * items changing state that reports in when an item is inactive/non-inactive. - * * Current subscribers do so at instantiation, before the worker - * threads even have a chance to start adding jobs to the repository. - * This suggests that the blocking will be a tiny one-off. - * While this is also a compelling argument for removing the callback - * call at subscription altogether, I think that running it is more intuitive - * (my issues with it aside). - */ - _watchedJobsListSemaphore.Wait(); - try + lock (_tallyLock) { - callback(WatchedJobs.Count(job => job.State == JobState.Inactive)); + callback(_inactiveJobsTally); } - finally + } + + public void SubscribeToIdempotencyBlockedCountUpdate(Action callback) + { + ArgumentNullException.ThrowIfNull(callback); + + lock (_callbackLock) { - _watchedJobsListSemaphore.Release(); + _idempotencyBlockedJobsCallbacks += callback; + } + + lock (_tallyLock) + { + callback(_idempotencyBlockedTally); } } @@ -438,18 +607,7 @@ public void SubscribeToWatchedJobsUpdate(Action callback) /* * Putting it on the record that I don't particularly like the below implementation on principle. - * It uses a blocking semaphore call, and it duplicates tally logic (especially true for sibling method SubscribeToInactiveCountUpdate). - * - * That said, I think the cure would be worse than the disease: - * * Implementing a check specifically for inactive jobs in this method's sibling - * SubscribeToInactiveCountUpdate would need some sort of tracker on the individual - * items changing state that reports in when an item is inactive/non-inactive. - * * Current subscribers do so at instantiation, before the worker - * threads even have a chance to start adding jobs to the repository. - * This suggests that the blocking will be a tiny one-off. - * While this is also a compelling argument for removing the callback - * call at subscription altogether, I think that running it is more intuitive - * (my issues with it aside). + * No matter how brief, I'm always twitchy about using a blocking call to a semaphore wait. Probably for no good reason, though. */ _watchedJobsListSemaphore.Wait(); try @@ -469,11 +627,16 @@ public Task WaitForJobDemandAsync(TimeSpan waitDuration, CancellationToken public async Task WaitForEmptyRepositoryAsync(CancellationToken cancellationToken = default) { - while (await GetWatchedJobsCountAsync(cancellationToken) > 0) + int count; + do { // Short timeout mirrors GetNextJobAsync: avoids missing a Set/Reset edge under concurrency await _repositoryEmptyEvent.WaitAsync(TimeSpan.FromMilliseconds(250), cancellationToken); - } + lock (_tallyLock) + { + count = _watchedJobsTally; + } + } while (count > 0); } public int GetBacklogMaxCount() diff --git a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/Subscriptions/JobSubscriberIntakeQueue.cs b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/Subscriptions/JobSubscriberIntakeQueue.cs index 456a7651..316d0fa2 100644 --- a/src/RedShirt.Example.JobWorker.Core/Services/Jobs/Subscriptions/JobSubscriberIntakeQueue.cs +++ b/src/RedShirt.Example.JobWorker.Core/Services/Jobs/Subscriptions/JobSubscriberIntakeQueue.cs @@ -57,7 +57,6 @@ public JobSubscriberIntakeQueue(IExecutionEndArbiter executionEndArbiter) { executionEndArbiter.AddOnStopCallback(_ => Cancel()); } - #pragma warning disable S2325 public void Load(IJobSourceResponse jobSourceResponse) #pragma warning disable S2325 diff --git a/test/RedShirt.Example.JobWorker.Core.IntegrationTests/Tests/JobResultTranslationTests.cs b/test/RedShirt.Example.JobWorker.Core.IntegrationTests/Tests/JobResultTranslationTests.cs index ef89f651..9cb85959 100644 --- a/test/RedShirt.Example.JobWorker.Core.IntegrationTests/Tests/JobResultTranslationTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.IntegrationTests/Tests/JobResultTranslationTests.cs @@ -64,9 +64,7 @@ public async Task JobResult_FromLogicRunner_IsTranslated_ToJobSource_AndFailureH var repositoryEntry = new Mock(MockBehavior.Strict); repositoryEntry.Setup(e => e.JobModel).Returns(job.Object); repositoryEntry.Setup(e => e.RawJobModel).Returns(rawJob.Object); - repositoryEntry - .Setup(e => e.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + repositoryEntry.SetupSet(e => e.State = JobState.Complete); // Application logic var logicRunner = new Mock(MockBehavior.Strict); @@ -97,7 +95,7 @@ public async Task JobResult_FromLogicRunner_IsTranslated_ToJobSource_AndFailureH // Executor loop control: process one job, then stop var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Models/JobRepositoryEntryTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Models/JobRepositoryEntryTests.cs index 584bc419..7f9608f7 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Models/JobRepositoryEntryTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Models/JobRepositoryEntryTests.cs @@ -6,28 +6,44 @@ namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Models; public class JobRepositoryEntryTests { - [Fact] - public async Task ConcurrentReadsAndWrites_DoNotThrow() + private static JobRepositoryEntry CreateEntry() { - var jre = new JobRepositoryEntry + return new JobRepositoryEntry { JobModel = new Mock(MockBehavior.Strict).Object, - RawJobModel = new Mock(MockBehavior.Strict).Object + RawJobModel = new Mock(MockBehavior.Strict).Object, + LastHeartbeatTime = default, + State = JobState.Inactive }; + } - var writers = Enumerable.Range(0, 8).Select(async i => + [Fact] + public void CanHeartbeat_WhenSetTrue_ThrowsArgumentException() + { + var jre = CreateEntry(); + + var ex = Assert.Throws(() => jre.CanHeartbeat = true); + Assert.Equal("value", ex.ParamName); + Assert.True(jre.CanHeartbeat); + } + + [Fact] + public async Task ConcurrentReadsAndWrites_DoNotThrow() + { + var jre = CreateEntry(); + + var writers = Enumerable.Range(0, 8).Select(i => Task.Run(() => { for (var n = 0; n < 100; n++) { - await jre.SetStateAsync((JobState) (n % 4), TestContext.Current.CancellationToken); - await jre.SetLastHeartbeatTimeAsync(DateTime.UtcNow.AddSeconds(-n), - TestContext.Current.CancellationToken); + jre.State = (JobState) (n % 4); + jre.LastHeartbeatTime = DateTime.UtcNow.AddSeconds(-n); if (i == 0 && n == 50) { - await jre.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken); + jre.CanHeartbeat = false; } } - }); + }, TestContext.Current.CancellationToken)); var readers = Enumerable.Range(0, 8).Select(_ => Task.Run(() => { @@ -44,7 +60,78 @@ await jre.SetLastHeartbeatTimeAsync(DateTime.UtcNow.AddSeconds(-n), } [Fact] - public async Task TestGettersSetters() + public void Dispose_SetsStateToCompleteAndClearsFurtherSubscriptions() + { + var jre = CreateEntry(); + var transitions = new List<(JobState? Original, JobState Current)>(); + jre.SubscribeToState((_, original, current) => transitions.Add((original, current))); + transitions.Clear(); + + jre.Dispose(); + jre.Dispose(); + + Assert.True(jre.IsDisposed); + Assert.Equal(JobState.Complete, jre.State); + Assert.Equal([(JobState.Inactive, JobState.Complete)], transitions); + + var ex = Assert.Throws(() => jre.SubscribeToState((_, _, _) => { })); + Assert.Equal(nameof(JobRepositoryEntry), ex.ObjectName); + } + + [Fact] + public void State_WhenSetNull_ThrowsArgumentNullException() + { + var jre = CreateEntry(); + + var ex = Assert.Throws(() => jre.State = null); + Assert.Equal("value", ex.ParamName); + Assert.Equal(JobState.Inactive, jre.State); + } + + [Fact] + public void SubscribeToState_InvokesWithOriginalAndCurrentValues() + { + var jre = CreateEntry(); + var first = new List<(IJobRepositoryEntry Entry, JobState? Original, JobState Current)>(); + var second = new List<(IJobRepositoryEntry Entry, JobState? Original, JobState Current)>(); + + jre.SubscribeToState((entry, original, current) => first.Add((entry, original, current))); + Assert.Equal([(jre, null, JobState.Inactive)], first); + + jre.SubscribeToState((entry, original, current) => second.Add((entry, original, current))); + Assert.Equal([(jre, null, JobState.Inactive)], second); + + jre.State = JobState.Active; + jre.State = JobState.Active; + jre.State = JobState.Complete; + + Assert.Equal( + [ + (jre, null, JobState.Inactive), (jre, JobState.Inactive, JobState.Active), + (jre, JobState.Active, JobState.Complete) + ], + first); + Assert.Equal( + [ + (jre, null, JobState.Inactive), (jre, JobState.Inactive, JobState.Active), + (jre, JobState.Active, JobState.Complete) + ], + second); + Assert.All(first, item => Assert.Same(jre, item.Entry)); + Assert.All(second, item => Assert.Same(jre, item.Entry)); + Assert.Equal(JobState.Complete, jre.State); + } + + [Fact] + public void SubscribeToState_WhenNull_ThrowsArgumentNullException() + { + var jre = CreateEntry(); + + Assert.Throws(() => jre.SubscribeToState(null!)); + } + + [Fact] + public void TestGettersSetters() { var jobModel = new Mock(MockBehavior.Strict).Object; var rawJobModel = new Mock(MockBehavior.Strict).Object; @@ -52,7 +139,9 @@ public async Task TestGettersSetters() var jre = new JobRepositoryEntry { JobModel = jobModel, - RawJobModel = rawJobModel + RawJobModel = rawJobModel, + LastHeartbeatTime = default, + State = JobState.Inactive }; Assert.True(jre.CanHeartbeat); @@ -61,15 +150,16 @@ public async Task TestGettersSetters() // Set/Get Heartbeat Time Assert.Equal(default, jre.LastHeartbeatTime); var newDate = DateTime.UtcNow - TimeSpan.FromMinutes(2); - await jre.SetLastHeartbeatTimeAsync(newDate, TestContext.Current.CancellationToken); + jre.LastHeartbeatTime = newDate; Assert.Equal(newDate, jre.LastHeartbeatTime); // Set/Get State - await jre.SetStateAsync(JobState.Active, TestContext.Current.CancellationToken); + jre.State = JobState.Active; Assert.Equal(JobState.Active, jre.State); - // Set/Get FlightTimeCanBeExtended - await jre.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken); + // Set/Get CanHeartbeat (false only) + jre.CanHeartbeat = false; Assert.False(jre.CanHeartbeat); + Assert.False(jre.IsDisposed); } } \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/AppliedExecutionEndArbiterTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/AppliedExecutionEndArbiterTests.cs deleted file mode 100644 index 23b6ef8d..00000000 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/AppliedExecutionEndArbiterTests.cs +++ /dev/null @@ -1,622 +0,0 @@ -using Microsoft.Extensions.Logging.Abstractions; -using RedShirt.Example.JobWorker.Common.Services.Utility; -using RedShirt.Example.JobWorker.Core.Services.ExecutionState; -using RedShirt.Example.JobWorker.Core.Services.Jobs; -using System.Reflection; - -namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Services.ExecutionState; - -public class AppliedExecutionEndArbiterTests -{ - private static Mock CreateSleepService() - { - return new Mock(MockBehavior.Strict); - } - - private static Mock CreateJobRepository(int inactiveCount = 0, int watchedCount = 0) - { - return CreateJobRepository(out _, inactiveCount, watchedCount); - } - - private static Mock CreateJobRepository( - out JobCountNotifier notifier, - int inactiveCount = 0, - int watchedCount = 0) - { - var captured = new JobCountNotifier(); - notifier = captured; - - var jobRepository = new Mock(MockBehavior.Strict); - jobRepository - .Setup(r => r.SubscribeToInactiveCountUpdate(It.IsAny>())) - .Callback>(callback => - { - captured.NotifyInactive = callback; - callback(inactiveCount); - }); - jobRepository - .Setup(r => r.SubscribeToWatchedJobsUpdate(It.IsAny>())) - .Callback>(callback => - { - captured.NotifyWatched = callback; - callback(watchedCount); - }); - return jobRepository; - } - - [Fact] - public void CountCallbacks_AfterDispose_AreIgnored() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 1, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - - arbiter.Dispose(); - - notifier.NotifyInactive(0); - notifier.NotifyWatched(0); - - // Counts must not change after dispose; keep-running still reflects pre-dispose state. - Assert.True(arbiter.MaintainerShouldKeepRunning()); - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact] - public void CountCallbacks_UpdateKeepRunningDecisions() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 1, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - Assert.True(arbiter.MaintainerShouldKeepRunning()); - - notifier.NotifyInactive(0); - Assert.False(arbiter.ExecutorsShouldKeepRunning()); - Assert.True(arbiter.MaintainerShouldKeepRunning()); - - notifier.NotifyWatched(0); - Assert.False(arbiter.ExecutorsShouldKeepRunning()); - Assert.False(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void Dispose_IsIdempotent() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - arbiter.Dispose(); - arbiter.Dispose(); - } - - [Fact] - public void ExecutorsShouldKeepRunning_WhenInnerTrueAndNoInactive_ReturnsTrue() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(0, 5).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_CompletesNormallyWhenNeitherTokenCancels() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - // Keep jobs present so the interrupt signal is not sent and the wait event is set. - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - sleepService - .Setup(s => s.DelayAsync(delay, It.IsAny())) - .Returns(Task.CompletedTask); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - sleepService.Object, NullLogger.Instance); - - await arbiter.MaintainerDelayWaitAsync(delay, "test", "test", TestContext.Current.CancellationToken); - - sleepService.Verify(s => s.DelayAsync(delay, It.IsAny()), Times.Once); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenCallerCancelsDuringSleep_PropagatesCancellation() - { - var delay = TimeSpan.FromSeconds(5); - using var callerCts = new CancellationTokenSource(); - - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var delayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var sleepService = CreateSleepService(); - sleepService - .Setup(s => s.DelayAsync(delay, It.IsAny())) - .Returns((TimeSpan _, CancellationToken token) => - { - delayStarted.SetResult(); - return Task.Delay(Timeout.Infinite, token); - }); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(1, 1).Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", callerCts.Token); - await delayStarted.Task; - - await callerCts.CancelAsync(); - - await Assert.ThrowsAnyAsync(() => delayTask); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenCallerCancels_PropagatesCancellation() - { - var delay = TimeSpan.FromSeconds(5); - using var callerCts = new CancellationTokenSource(); - await callerCts.CancelAsync(); - - var innerArbiter = new Mock(MockBehavior.Strict); - // Keep jobs present so only the caller token drives cancellation. - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - sleepService - .Setup(s => s.DelayAsync(delay, It.IsAny())) - .Returns((TimeSpan _, CancellationToken token) => Task.FromCanceled(token)); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - sleepService.Object, NullLogger.Instance); - - await Assert.ThrowsAnyAsync(() => - arbiter.MaintainerDelayWaitAsync(delay, "test", "test", callerCts.Token)); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenCountsDropToEmptyWhileStopping_InterruptsAndCompletes() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - // Stopping, but jobs are still present so the interrupt is not sent yet. - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - var delayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - CancellationToken linkedToken = default; - - var sleepService = CreateSleepService(); - sleepService - .Setup(s => s.DelayAsync(delay, It.IsAny())) - .Returns((TimeSpan _, CancellationToken token) => - { - linkedToken = token; - delayStarted.SetResult(); - return Task.Delay(Timeout.Infinite, token); - }); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 1, 1).Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - await delayStarted.Task; - - // Both counts must be empty before the interrupt fires. - notifier.NotifyInactive(0); - Assert.False(linkedToken.IsCancellationRequested); - - notifier.NotifyWatched(0); - Assert.True(linkedToken.IsCancellationRequested); - - await delayTask; - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenDisposed_ReturnsWithoutSleeping() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - sleepService.Object, NullLogger.Instance); - - arbiter.Dispose(); - - await arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenInterrupted_IgnoresCancellation() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - // Empty job counts while stopping cancels the internal interrupt token on subscribe. - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - var sleepService = CreateSleepService(); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository().Object, - sleepService.Object, NullLogger.Instance); - - await arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenNoWatchedJobsAndKeepRunning_DoesNotInterrupt() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier).Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - await Task.Delay(50, TestContext.Current.CancellationToken); - Assert.False(delayTask.IsCompleted); - - // Empty counts while keep-running must not fire the interrupt; only watched jobs unblock. - notifier.NotifyInactive(0); - await Task.Delay(50, TestContext.Current.CancellationToken); - Assert.False(delayTask.IsCompleted); - - notifier.NotifyWatched(1); - await delayTask; - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenNoWatchedJobs_WaitsUntilWatchedThenReturnsWithoutSleeping() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier).Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - await Task.Delay(50, TestContext.Current.CancellationToken); - Assert.False(delayTask.IsCompleted); - - notifier.NotifyWatched(1); - await delayTask; - - sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenWaitingForWatched_CallerCancelPropagates() - { - var delay = TimeSpan.FromSeconds(5); - using var callerCts = new CancellationTokenSource(); - - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository().Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", callerCts.Token); - - await Task.Delay(50, TestContext.Current.CancellationToken); - Assert.False(delayTask.IsCompleted); - - await callerCts.CancelAsync(); - - await Assert.ThrowsAnyAsync(() => delayTask); - sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenWaitingForWatched_InterruptCompletesWithoutThrowing() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - // Stopping with inactive work present: interrupt is deferred until counts clear. - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - var sleepService = CreateSleepService(); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 1).Object, - sleepService.Object, NullLogger.Instance); - - var delayTask = arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - await Task.Delay(50, TestContext.Current.CancellationToken); - Assert.False(delayTask.IsCompleted); - - notifier.NotifyInactive(0); - await delayTask; - - sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact(Timeout = 5000)] - public async Task MaintainerDelayWaitAsync_WhenWatchedCountUnchanged_StillTakesSleepPath() - { - var delay = TimeSpan.FromSeconds(5); - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - var sleepService = CreateSleepService(); - sleepService - .Setup(s => s.DelayAsync(delay, It.IsAny())) - .Returns(Task.CompletedTask); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 0, 1).Object, - sleepService.Object, NullLogger.Instance); - - // Same count must not Reset the wait event; sleep path should remain available. - notifier.NotifyWatched(1); - - await arbiter.MaintainerDelayWaitAsync(delay, "test", "test", CancellationToken.None); - - sleepService.Verify(s => s.DelayAsync(delay, It.IsAny()), Times.Once); - } - - [Fact] - public void MaintainerShouldKeepRunning_WhenInnerTrueAndNoJobs_ReturnsTrue() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository().Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - } - - /// - /// Test with impossible IJobRepository output - /// - [Fact] - public void TestExecutorStopRunningWeird() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(-1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.False(arbiter.ExecutorsShouldKeepRunning()); - } - - /// - /// All checks return true - /// - [Fact] - public void TestExecutorsKeepRunningA() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(true); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact] - public void TestExecutorsKeepRunningBecauseInactive() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact] - public void TestExecutorsKeepRunningDespiteInner() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact] - public void TestExecutorsKeepRunningDespiteWatched() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(0, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - // Confirming that we're ignoring watched jobs - Assert.False(arbiter.ExecutorsShouldKeepRunning()); - } - - [Fact] - public void TestExecutorsStopRunning() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository().Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.False(arbiter.ExecutorsShouldKeepRunning()); - } - - /// - /// All checks return true - /// - [Fact] - public void TestMaintainerKeepRunningA() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(true); - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void TestMaintainerKeepRunningBecauseInactive() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void TestMaintainerKeepRunningBecauseWatched() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(0, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void TestMaintainerKeepRunningDespiteInner() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(1, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.True(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void TestMaintainerStopRunning() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository().Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.False(arbiter.MaintainerShouldKeepRunning()); - } - - /// - /// Test with impossible IJobRepository output - /// - [Fact] - public void TestMaintainerStopRunningWeird() - { - var innerArbiter = new Mock(); - innerArbiter - .Setup(a => a.ShouldKeepRunning()) - .Returns(false); // Inner arbiter says no - - using var arbiter = new AppliedExecutionEndArbiter(innerArbiter.Object, CreateJobRepository(-1, -1).Object, - CreateSleepService().Object, NullLogger.Instance); - - Assert.False(arbiter.MaintainerShouldKeepRunning()); - } - - [Fact] - public void TryCancelInterrupt_WhenCtsAlreadyDisposed_SwallowsObjectDisposedException() - { - var innerArbiter = new Mock(MockBehavior.Strict); - innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); - - using var arbiter = new AppliedExecutionEndArbiter( - innerArbiter.Object, - CreateJobRepository(out var notifier, 1).Object, - CreateSleepService().Object, NullLogger.Instance); - - var field = typeof(AppliedExecutionEndArbiter).GetField("_interruptCts", - BindingFlags.Instance | BindingFlags.NonPublic); - Assert.NotNull(field); - var cts = (CancellationTokenSource) field.GetValue(arbiter)!; - cts.Dispose(); - - // Clearing the last active count would cancel the interrupt CTS; it is already disposed. - notifier.NotifyInactive(0); - - Assert.False(arbiter.MaintainerShouldKeepRunning()); - } - - private sealed class JobCountNotifier - { - public Action NotifyInactive { get; set; } = _ => { }; - public Action NotifyWatched { get; set; } = _ => { }; - } -} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/ExecutorExecutionEndArbiterTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/ExecutorExecutionEndArbiterTests.cs new file mode 100644 index 00000000..e8df09cf --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/ExecutorExecutionEndArbiterTests.cs @@ -0,0 +1,112 @@ +using RedShirt.Example.JobWorker.Core.Services.ExecutionState; +using RedShirt.Example.JobWorker.Core.Services.Jobs; + +namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Services.ExecutionState; + +public class ExecutorExecutionEndArbiterTests +{ + private static Mock CreateJobRepository( + out JobCountNotifier notifier, + int inactiveCount = 0, + int blockedCount = 0) + { + var captured = new JobCountNotifier(); + notifier = captured; + var jobRepository = new Mock(MockBehavior.Strict); + jobRepository + .Setup(r => r.SubscribeToInactiveCountUpdate(It.IsAny>())) + .Callback>(callback => + { + captured.NotifyInactive = callback; + callback(inactiveCount); + }); + jobRepository + .Setup(r => r.SubscribeToIdempotencyBlockedCountUpdate(It.IsAny>())) + .Callback>(callback => + { + captured.NotifyBlocked = callback; + callback(blockedCount); + }); + return jobRepository; + } + + private static ExecutorExecutionEndArbiter CreateArbiter(IExecutionEndArbiter inner, IJobRepository jobRepository) + { + return new ExecutorExecutionEndArbiter(jobRepository, inner); + } + + [Fact] + public void CountCallbacks_UpdateKeepRunningDecisions() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(false); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out var notifier, 1, 1).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + notifier.NotifyInactive(0); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + notifier.NotifyBlocked(0); + Assert.False(arbiter.ExecutorsShouldKeepRunning()); + notifier.NotifyInactive(2); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerFalseAndJobsPresent_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(false); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1, 1).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerFalseAndNoJobs_ReturnsFalse() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(false); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _).Object); + Assert.False(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerTrueAndBothCountsPositive_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1, 1).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerTrueAndNoBlockedJobs_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerTrueAndNoInactive_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 0, 1).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + [Fact] + public void ExecutorsShouldKeepRunning_WhenInnerTrueAndNoJobs_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _).Object); + Assert.True(arbiter.ExecutorsShouldKeepRunning()); + } + + private sealed class JobCountNotifier + { + public Action NotifyBlocked { get; set; } = _ => { }; + public Action NotifyInactive { get; set; } = _ => { }; + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiterTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiterTests.cs new file mode 100644 index 00000000..d438ac0b --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/HeartbeatMonitorExecutionEndArbiterTests.cs @@ -0,0 +1,283 @@ +using Microsoft.Extensions.Logging.Abstractions; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Core.Services.ExecutionState; +using RedShirt.Example.JobWorker.Core.Services.Jobs; +using System.Reflection; + +namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Services.ExecutionState; + +public class HeartbeatMonitorExecutionEndArbiterTests +{ + private static Mock CreateSleepService() + { + return new Mock(MockBehavior.Strict); + } + + private static Mock CreateJobRepository(out JobCountNotifier notifier, int watchedCount = 0) + { + var captured = new JobCountNotifier(); + notifier = captured; + + var jobRepository = new Mock(MockBehavior.Strict); + jobRepository + .Setup(r => r.SubscribeToWatchedJobsUpdate(It.IsAny>())) + .Callback>(callback => + { + captured.NotifyWatched = callback; + callback(watchedCount); + }); + return jobRepository; + } + + private static Mock CreateJobRepository(int watchedCount = 0) + { + return CreateJobRepository(out _, watchedCount); + } + + private static HeartbeatMonitorExecutionEndArbiter CreateArbiter( + IExecutionEndArbiter inner, + IJobRepository jobRepository, + ISleepService sleepService) + { + return new HeartbeatMonitorExecutionEndArbiter(jobRepository, inner, sleepService, + NullLogger.Instance); + } + + [Fact] + public void CountCallbacks_AfterDispose_AreIgnored() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(out var notifier, 1).Object, + CreateSleepService().Object); + + Assert.True(arbiter.MonitorShouldKeepRunning()); + arbiter.Dispose(); + notifier.NotifyWatched(0); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void CountCallbacks_UpdateKeepRunningDecisions() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(out var notifier, 1).Object, + CreateSleepService().Object); + + Assert.True(arbiter.MonitorShouldKeepRunning()); + notifier.NotifyWatched(0); + Assert.False(arbiter.MonitorShouldKeepRunning()); + notifier.NotifyWatched(2); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void Dispose_IsIdempotent() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + using var arbiter = + CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, CreateSleepService().Object); + arbiter.Dispose(); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_CompletesNormallyWhenNeitherTokenCancels() + { + var delay = TimeSpan.FromSeconds(5); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + var sleepService = CreateSleepService(); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns(Task.CompletedTask); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, sleepService.Object); + await arbiter.HeartbeatMonitorDelayWaitAsync(delay, TestContext.Current.CancellationToken); + sleepService.Verify(s => s.DelayAsync(delay, It.IsAny()), Times.Once); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenCallerCancelsDuringSleep_PropagatesCancellation() + { + var delay = TimeSpan.FromSeconds(5); + using var callerCts = new CancellationTokenSource(); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + var delayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sleepService = CreateSleepService(); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns((TimeSpan _, CancellationToken token) => + { + delayStarted.SetResult(); + return Task.Delay(Timeout.Infinite, token); + }); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, sleepService.Object); + var delayTask = arbiter.HeartbeatMonitorDelayWaitAsync(delay, callerCts.Token); + await delayStarted.Task; + await callerCts.CancelAsync(); + await Assert.ThrowsAnyAsync(() => delayTask); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenCallerCancels_PropagatesCancellation() + { + var delay = TimeSpan.FromSeconds(5); + using var callerCts = new CancellationTokenSource(); + await callerCts.CancelAsync(); + + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + var sleepService = CreateSleepService(); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns((TimeSpan _, CancellationToken token) => Task.FromCanceled(token)); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, sleepService.Object); + await Assert.ThrowsAnyAsync(() => + arbiter.HeartbeatMonitorDelayWaitAsync(delay, callerCts.Token)); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenDisposed_ReturnsWithoutSleeping() + { + var delay = TimeSpan.FromSeconds(5); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + var sleepService = CreateSleepService(); + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, sleepService.Object); + arbiter.Dispose(); + await arbiter.HeartbeatMonitorDelayWaitAsync(delay, CancellationToken.None); + sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenInterrupted_IgnoresCancellation() + { + var delay = TimeSpan.FromSeconds(5); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + var sleepService = CreateSleepService(); + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository().Object, sleepService.Object); + await arbiter.HeartbeatMonitorDelayWaitAsync(delay, CancellationToken.None); + sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenWatchedCountDropsToZero_InterruptsAndCompletes() + { + var delay = TimeSpan.FromSeconds(5); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + + var delayStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken linkedToken = default; + var sleepService = CreateSleepService(); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns((TimeSpan _, CancellationToken token) => + { + linkedToken = token; + delayStarted.SetResult(); + return Task.Delay(Timeout.Infinite, token); + }); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(out var notifier, 1).Object, + sleepService.Object); + var delayTask = arbiter.HeartbeatMonitorDelayWaitAsync(delay, CancellationToken.None); + await delayStarted.Task; + Assert.False(linkedToken.IsCancellationRequested); + notifier.NotifyWatched(0); + Assert.True(linkedToken.IsCancellationRequested); + await delayTask; + } + + [Fact(Timeout = 5000)] + public async Task HeartbeatMonitorDelayWaitAsync_WhenWatchedCountUnchanged_StillTakesSleepPath() + { + var delay = TimeSpan.FromSeconds(5); + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + var sleepService = CreateSleepService(); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns(Task.CompletedTask); + + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(out var notifier, 1).Object, + sleepService.Object); + notifier.NotifyWatched(1); + await arbiter.HeartbeatMonitorDelayWaitAsync(delay, CancellationToken.None); + sleepService.Verify(s => s.DelayAsync(delay, It.IsAny()), Times.Once); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerFalseAndNoWatchedJobs_ReturnsFalse() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + using var arbiter = + CreateArbiter(innerArbiter.Object, CreateJobRepository().Object, CreateSleepService().Object); + Assert.False(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerFalseAndWatchedJobs_ReturnsTrue() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + using var arbiter = + CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, CreateSleepService().Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerTrueAndNoWatchedJobs_ReturnsTrue() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + using var arbiter = + CreateArbiter(innerArbiter.Object, CreateJobRepository().Object, CreateSleepService().Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerTrueAndWatchedJobs_ReturnsTrue() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + using var arbiter = + CreateArbiter(innerArbiter.Object, CreateJobRepository(1).Object, CreateSleepService().Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void TryCancelInterrupt_WhenCtsAlreadyDisposed_SwallowsObjectDisposedException() + { + var innerArbiter = new Mock(MockBehavior.Strict); + innerArbiter.Setup(a => a.ShouldKeepRunning()).Returns(false); + using var arbiter = CreateArbiter(innerArbiter.Object, CreateJobRepository(out var notifier, 1).Object, + CreateSleepService().Object); + + var field = typeof(HeartbeatMonitorExecutionEndArbiter).GetField("_interruptCts", + BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + var cts = (CancellationTokenSource) field.GetValue(arbiter)!; + cts.Dispose(); + notifier.NotifyWatched(0); + Assert.False(arbiter.MonitorShouldKeepRunning()); + } + + private sealed class JobCountNotifier + { + public Action NotifyWatched { get; set; } = _ => { }; + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiterTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiterTests.cs new file mode 100644 index 00000000..756b5b43 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/ExecutionState/IdempotencyMonitorExecutionEndArbiterTests.cs @@ -0,0 +1,126 @@ +using Microsoft.Extensions.Logging.Abstractions; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Core.Services.ExecutionState; +using RedShirt.Example.JobWorker.Core.Services.Jobs; + +namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Services.ExecutionState; + +public class IdempotencyMonitorExecutionEndArbiterTests +{ + private static Mock CreateJobRepository(out JobCountNotifier notifier, int watchedCount = 0, + int blockedCount = 0) + { + var captured = new JobCountNotifier(); + notifier = captured; + var jobRepository = new Mock(MockBehavior.Strict); + jobRepository + .Setup(r => r.SubscribeToWatchedJobsUpdate(It.IsAny>())) + .Callback>(callback => + { + captured.NotifyWatched = callback; + callback(watchedCount); + }); + jobRepository + .Setup(r => r.SubscribeToIdempotencyBlockedCountUpdate(It.IsAny>())) + .Callback>(callback => + { + captured.NotifyBlocked = callback; + callback(blockedCount); + }); + return jobRepository; + } + + private static IdempotencyMonitorExecutionEndArbiter CreateArbiter(IExecutionEndArbiter inner, + IJobRepository jobRepository, ISleepService? sleepService = null) + { + return new IdempotencyMonitorExecutionEndArbiter(jobRepository, inner, + sleepService ?? new Mock(MockBehavior.Strict).Object, + NullLogger.Instance); + } + + [Fact] + public void CountCallbacks_AfterDispose_AreIgnored() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out var notifier, 1, 1).Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + arbiter.Dispose(); + notifier.NotifyWatched(0); + notifier.NotifyBlocked(0); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void Dispose_IsIdempotent() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1).Object); + arbiter.Dispose(); + } + + [Fact(Timeout = 5000)] + public async Task IdempotencyMonitorDelayWaitAsync_CompletesNormallyWhenWatchedJobsExist() + { + var delay = TimeSpan.FromSeconds(5); + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var sleepService = new Mock(MockBehavior.Strict); + sleepService + .Setup(s => s.DelayAsync(delay, It.IsAny())) + .Returns(Task.CompletedTask); + + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1, 1).Object, sleepService.Object); + await arbiter.IdempotencyMonitorDelayWaitAsync(delay, CancellationToken.None); + sleepService.Verify(s => s.DelayAsync(delay, It.IsAny()), Times.Once); + } + + [Fact(Timeout = 5000)] + public async Task IdempotencyMonitorDelayWaitAsync_WhenDisposed_ReturnsWithoutSleeping() + { + var delay = TimeSpan.FromSeconds(5); + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + var sleepService = new Mock(MockBehavior.Strict); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _, 1).Object, sleepService.Object); + arbiter.Dispose(); + await arbiter.IdempotencyMonitorDelayWaitAsync(delay, CancellationToken.None); + sleepService.Verify(s => s.DelayAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerFalseAndNoWatchedJobs_ReturnsFalse() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(false); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _).Object); + Assert.False(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerFalseAndWatchedJobs_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(false); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out var notifier, 1).Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + notifier.NotifyWatched(0); + Assert.False(arbiter.MonitorShouldKeepRunning()); + } + + [Fact] + public void MonitorShouldKeepRunning_WhenInnerTrueAndNoWatchedJobs_ReturnsTrue() + { + var inner = new Mock(MockBehavior.Strict); + inner.Setup(a => a.ShouldKeepRunning()).Returns(true); + using var arbiter = CreateArbiter(inner.Object, CreateJobRepository(out _).Object); + Assert.True(arbiter.MonitorShouldKeepRunning()); + } + + private sealed class JobCountNotifier + { + public Action NotifyBlocked { get; set; } = _ => { }; + public Action NotifyWatched { get; set; } = _ => { }; + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/HandlerTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/HandlerTests.cs index 2fd24d1d..44bce04a 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/HandlerTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/HandlerTests.cs @@ -30,13 +30,13 @@ private static Mock> CreateLogger() private static void SetupNotEnabledWorkers( Mock executor, - Mock maintainer, + Mock monitor, Mock idempotencyMonitor, Mock jobSubscriberManager) { executor.Setup(e => e.RunAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(HandlerComponentResponse.NotEnabled); - maintainer.Setup(m => m.RunAsync(It.IsAny())) + monitor.Setup(m => m.RunAsync(It.IsAny())) .ReturnsAsync(HandlerComponentResponse.NotEnabled); idempotencyMonitor.Setup(m => m.RunAsync(It.IsAny())) .ReturnsAsync(HandlerComponentResponse.NotEnabled); @@ -94,12 +94,12 @@ public async Task HandleAsync_DoesNotCompleteWhenOnlyNotEnabledWorkersFinish() }); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), new NullLogger()); @@ -142,12 +142,12 @@ public async Task HandleAsync_LogsFinishedAndNotEnabledResponses() .ReturnsAsync(HandlerComponentResponse.Finished); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), logger.Object); @@ -157,7 +157,7 @@ public async Task HandleAsync_LogsFinishedAndNotEnabledResponses() Assert.True(result); VerifyWorkerDoneLogged(logger, "MessagePoller", HandlerComponentResponse.Finished, Times.Once()); VerifyWorkerDoneLogged(logger, "JobExecutor", HandlerComponentResponse.NotEnabled, Times.Once()); - VerifyWorkerDoneLogged(logger, "HeartbeatMaintainer", HandlerComponentResponse.NotEnabled, Times.Once()); + VerifyWorkerDoneLogged(logger, "HeartbeatMonitor", HandlerComponentResponse.NotEnabled, Times.Once()); VerifyWorkerDoneLogged(logger, "IdempotencyMonitor", HandlerComponentResponse.NotEnabled, Times.Once()); VerifyWorkerDoneLogged(logger, "JobSubscriberManager", HandlerComponentResponse.NotEnabled, Times.Once()); VerifyWorkerResponseLogged(logger, HandlerComponentResponse.Cancelled, Times.Never()); @@ -179,12 +179,12 @@ public async Task HandleAsync_ReturnsFalseOnUnhandledWorkerException() .ThrowsAsync(expected); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), new NullLogger()); @@ -205,12 +205,12 @@ public async Task HandleAsync_WhenInvokedTwice_ThrowsInvalidOperationException() .ReturnsAsync(HandlerComponentResponse.Finished); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), new NullLogger()); @@ -240,12 +240,12 @@ public async Task HandleAsync_WhenWorkerThrowsOperationCanceledWhileTokenCancele }); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), new NullLogger()); @@ -289,12 +289,12 @@ public async Task HandleAsync_WhenWorkerThrowsOperationCanceledWhileTokenCancele }); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), logger.Object); @@ -331,12 +331,12 @@ public async Task HandleAsync_WhenWorkerThrowsOperationCanceledWhileTokenNotCanc .ThrowsAsync(expected); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), new NullLogger()); @@ -358,12 +358,12 @@ public async Task HandleAsync_WhenWorkerThrowsUnexpectedOperationCanceled_LogsCa .ThrowsAsync(new OperationCanceledException("unexpected cancel")); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), logger.Object); @@ -387,12 +387,12 @@ public async Task HandleAsync_WhenWorkerThrows_LogsExceptionResponse() .ThrowsAsync(new InvalidOperationException("worker blew up")); var executor = new Mock(MockBehavior.Strict); - var maintainer = new Mock(MockBehavior.Strict); + var monitor = new Mock(MockBehavior.Strict); var idempotencyMonitor = new Mock(MockBehavior.Strict); var jobSubscriberManager = new Mock(MockBehavior.Strict); - SetupNotEnabledWorkers(executor, maintainer, idempotencyMonitor, jobSubscriberManager); + SetupNotEnabledWorkers(executor, monitor, idempotencyMonitor, jobSubscriberManager); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = 1}), logger.Object); @@ -420,8 +420,8 @@ public async Task TestRunAsync(int numberOfExecutorThreads, int expectedNumberOf executor.Setup(e => e.RunAsync(It.IsAny(), TestContext.Current.CancellationToken)) .ReturnsAsync(HandlerComponentResponse.Finished); - var maintainer = new Mock(MockBehavior.Strict); - maintainer.Setup(m => m.RunAsync(TestContext.Current.CancellationToken)) + var monitor = new Mock(MockBehavior.Strict); + monitor.Setup(m => m.RunAsync(TestContext.Current.CancellationToken)) .ReturnsAsync(HandlerComponentResponse.NotEnabled); var idempotencyMonitor = new Mock(MockBehavior.Strict); @@ -438,7 +438,7 @@ public async Task TestRunAsync(int numberOfExecutorThreads, int expectedNumberOf }; Assert.Equal(expectedNumberOfThreads, options.EffectiveWorkerThreadCount); - var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, maintainer.Object, executor.Object, + var handler = new Handler(executionEndArbiter, jobLoaderLoop.Object, monitor.Object, executor.Object, idempotencyMonitor.Object, jobSubscriberManager.Object, Options.Create(new ThreadConfigurationModel {WorkerThreadCount = numberOfExecutorThreads}), new NullLogger()); @@ -454,7 +454,7 @@ public async Task TestRunAsync(int numberOfExecutorThreads, int expectedNumberOf executor.Verify(e => e.RunAsync(i1, TestContext.Current.CancellationToken)); } - Assert.Single(maintainer.Invocations); + Assert.Single(monitor.Invocations); Assert.Single(idempotencyMonitor.Invocations); Assert.Single(jobSubscriberManager.Invocations); } diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/MaintainerTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/HeartbeatMonitorTests.cs similarity index 76% rename from test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/MaintainerTests.cs rename to test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/HeartbeatMonitorTests.cs index bd2d3fcf..0fb89612 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/MaintainerTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Heartbeats/HeartbeatMonitorTests.cs @@ -14,7 +14,7 @@ namespace RedShirt.Example.JobWorker.Core.UnitTests.Tests.Services.Heartbeats; -public class HeartbeatMaintainerTests +public class HeartbeatMonitorTests { private static ICoreHealthStateUpdateService CreateHealthStateUpdateService() { @@ -32,11 +32,10 @@ private static ISleepService CreateSleepService() return sleepService.Object; } - private static void SetupMaintainerDelay(Mock arbiter) + private static void SetupMonitorDelay(Mock arbiter) { arbiter - .Setup(a => a.MaintainerDelayWaitAsync(It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + .Setup(a => a.HeartbeatMonitorDelayWaitAsync(It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); } @@ -46,18 +45,18 @@ private static void SetupMaintainerDelay(Mock(MockBehavior.Strict); - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); var jobRepository = new Mock(MockBehavior.Strict); var jobSource = new Mock(MockBehavior.Strict); jobSource.Setup(s => s.RecommendedHeartbeatIntervalSeconds).Returns(intervalSeconds); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Empty(executionEndArbiter.Invocations); Assert.Empty(jobRepository.Invocations); @@ -75,8 +74,7 @@ public async Task RunAsync_WhenUnexpectedHeartbeatException_AndHaltOnFailureFals entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.CanHeartbeat = false); var heartbeatCalculator = new Mock(); heartbeatCalculator.Setup(c => c.IsReadyForHeartbeat(entry.Object)).Returns(true); @@ -85,10 +83,10 @@ public async Task RunAsync_WhenUnexpectedHeartbeatException_AndHaltOnFailureFals .Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -114,17 +112,17 @@ public async Task RunAsync_WhenUnexpectedHeartbeatException_AndHaltOnFailureFals var health = new Mock(MockBehavior.Strict); health.Setup(h => h.NoteIncident()); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, health.Object, CreateSleepService(), Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), - new NullLogger()); + new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); health.Verify(h => h.NoteIncident(), Times.Once); - entry.Verify(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken), Times.Once); + entry.VerifySet(e => e.CanHeartbeat = false, Times.Once); } [Fact(Timeout = 1500)] @@ -144,10 +142,10 @@ public async Task RunAsync_WhenUnexpectedHeartbeatException_AndHaltOnFailure_Pro var heartbeatCalculator = new Mock(); heartbeatCalculator.Setup(c => c.IsReadyForHeartbeat(entry.Object)).Returns(true); - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(true); var jobRepository = new Mock(MockBehavior.Strict); @@ -164,19 +162,19 @@ public async Task RunAsync_WhenUnexpectedHeartbeatException_AndHaltOnFailure_Pro var health = new Mock(MockBehavior.Strict); health.Setup(h => h.NoteIncident()); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, health.Object, CreateSleepService(), Options.Create(new CoreConfigurationModel {HaltOnFailure = true}), - new NullLogger()); + new NullLogger()); var thrown = await Assert.ThrowsAsync(() => - maintainer.RunAsync(TestContext.Current.CancellationToken)); + monitor.RunAsync(TestContext.Current.CancellationToken)); Assert.Same(unexpected, thrown); health.Verify(h => h.NoteIncident(), Times.Once); - entry.Verify(e => e.SetAsCannotHeartbeatAsync(It.IsAny()), Times.Never); + entry.VerifySet(e => e.CanHeartbeat = false, Times.Never); } /// @@ -194,10 +192,9 @@ public async Task TestFilterOutCannotHeartbeatJobs() entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.CanHeartbeat).Returns(true); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); var redHerringEntry = new Mock(MockBehavior.Strict); redHerringEntry.Setup(e => e.State).Returns(JobState.Active); redHerringEntry.Setup(e => e.CanHeartbeat).Returns(false); @@ -211,10 +208,10 @@ public async Task TestFilterOutCannotHeartbeatJobs() .Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -242,13 +239,13 @@ public async Task TestFilterOutCannotHeartbeatJobs() .Setup(s => s.HeartbeatAsync(rawJobModel.Object, TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -264,10 +261,10 @@ public async Task TestHeartbeatNoJobs() var heartbeatCalculator = new Mock(); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -289,13 +286,13 @@ public async Task TestHeartbeatNoJobs() .Setup(s => s.RecommendedHeartbeatIntervalSeconds) .Returns(1); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -317,10 +314,9 @@ public async Task TestHeartbeatSingleJob() entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); var heartbeatCalculator = new Mock(); heartbeatCalculator @@ -331,10 +327,10 @@ public async Task TestHeartbeatSingleJob() .Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -361,13 +357,13 @@ public async Task TestHeartbeatSingleJob() .Setup(s => s.HeartbeatAsync(rawJobModel.Object, TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -388,13 +384,11 @@ public async Task TestHeartbeatSingleJobButGotHeartbeatException() var rawJobModel = new Mock(MockBehavior.Strict); entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); - entry.Setup(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.CanHeartbeat = false); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); var heartbeatCalculator = new Mock(); heartbeatCalculator @@ -405,10 +399,10 @@ public async Task TestHeartbeatSingleJobButGotHeartbeatException() .Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -436,20 +430,19 @@ public async Task TestHeartbeatSingleJobButGotHeartbeatException() .Returns(() => throw new WorkerJobSourceException("Test") {CouldBeTransient = false, IsHandled = false, CouldBeExternallySolvable = false}); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); jobSource.Verify(s => s.HeartbeatAsync(rawJobModel.Object, TestContext.Current.CancellationToken), Times.Once); - entry.Verify(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken), - Times.Once); + entry.VerifySet(e => e.CanHeartbeat = false, Times.Once); } /// @@ -477,10 +470,10 @@ public async Task TestHeartbeatSingleJob_Complete() .Returns(TimeSpan.FromMilliseconds(100)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -504,13 +497,13 @@ public async Task TestHeartbeatSingleJob_Complete() .Setup(s => s.RecommendedHeartbeatIntervalSeconds) .Returns(1); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -528,8 +521,7 @@ public async Task TestHeartbeatSingleJob_ExhaustsTransientRetriesThenDisablesExt var rawJobModel = new Mock(MockBehavior.Strict); entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); - entry.Setup(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.CanHeartbeat = false); entry.Setup(e => e.State).Returns(JobState.Active); var heartbeatCalculator = new Mock(MockBehavior.Strict); @@ -537,10 +529,10 @@ public async Task TestHeartbeatSingleJob_ExhaustsTransientRetriesThenDisablesExt heartbeatCalculator.Setup(c => c.TimeUntilNextHeartbeat(entry.Object)).Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -569,18 +561,17 @@ public async Task TestHeartbeatSingleJob_ExhaustsTransientRetriesThenDisablesExt .Setup(s => s.DelayAsync(It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), sleepService.Object, - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); jobSource.Verify(s => s.HeartbeatAsync(rawJobModel.Object, TestContext.Current.CancellationToken), Times.Exactly(Globals.HeartbeatRetryCount + 1)); - entry.Verify(e => e.SetAsCannotHeartbeatAsync(TestContext.Current.CancellationToken), - Times.Once); + entry.VerifySet(e => e.CanHeartbeat = false, Times.Once); } /// @@ -597,10 +588,9 @@ public async Task TestHeartbeatSingleJob_NotReadyYet() entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); var heartbeatCalculator = new Mock(); heartbeatCalculator @@ -611,10 +601,10 @@ public async Task TestHeartbeatSingleJob_NotReadyYet() .Returns(TimeSpan.FromMilliseconds(100)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -638,13 +628,13 @@ public async Task TestHeartbeatSingleJob_NotReadyYet() .Setup(s => s.RecommendedHeartbeatIntervalSeconds) .Returns(1); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -666,10 +656,9 @@ public async Task TestHeartbeatSingleJob_PreciseTiming() entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); var heartbeatCalculator = new Mock(); heartbeatCalculator @@ -680,10 +669,10 @@ public async Task TestHeartbeatSingleJob_PreciseTiming() .Returns(TimeSpan.FromMilliseconds(100)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -710,13 +699,13 @@ public async Task TestHeartbeatSingleJob_PreciseTiming() .Setup(s => s.HeartbeatAsync(rawJobModel.Object, TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); @@ -734,18 +723,17 @@ public async Task TestHeartbeatSingleJob_RetriesTransientFailuresThenSucceeds() entry.Setup(e => e.JobModel).Returns(subject.Object); entry.Setup(e => e.RawJobModel).Returns(rawJobModel.Object); entry.Setup(e => e.State).Returns(JobState.Active); - entry.Setup(e => e.SetLastHeartbeatTimeAsync(It.IsAny(), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.LastHeartbeatTime = It.IsAny()); var heartbeatCalculator = new Mock(MockBehavior.Strict); heartbeatCalculator.Setup(c => c.IsReadyForHeartbeat(entry.Object)).Returns(true); heartbeatCalculator.Setup(c => c.TimeUntilNextHeartbeat(entry.Object)).Returns(TimeSpan.FromSeconds(1)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -786,18 +774,17 @@ public async Task TestHeartbeatSingleJob_RetriesTransientFailuresThenSucceeds() .Setup(s => s.DelayAsync(It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), sleepService.Object, - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Equal(3, attempts); - entry.Verify(e => e.SetAsCannotHeartbeatAsync(It.IsAny()), - Times.Never); - entry.Verify(e => e.SetLastHeartbeatTimeAsync(It.IsAny(), It.IsAny()), Times.Once); + entry.VerifySet(e => e.CanHeartbeat = false, Times.Never); + entry.VerifySet(e => e.LastHeartbeatTime = It.IsAny(), Times.Once); } /// @@ -816,10 +803,9 @@ public async Task TestHeartbeatTwoJob() entry1.Setup(e => e.RawJobModel).Returns(rawJobModel1.Object); entry1.Setup(e => e.CanHeartbeat).Returns(true); entry1.Setup(e => e.State).Returns(JobState.Active); - entry1.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry1.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); // Second job var subject2 = new Mock(MockBehavior.Strict); @@ -830,10 +816,9 @@ public async Task TestHeartbeatTwoJob() entry2.Setup(e => e.RawJobModel).Returns(rawJobModel2.Object); entry2.Setup(e => e.State).Returns(JobState.Active); entry2.Setup(e => e.CanHeartbeat).Returns(true); - entry2.Setup(e => e.SetLastHeartbeatTimeAsync(It.Is(dt => - dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && - dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250)), TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry2.SetupSet(e => e.LastHeartbeatTime = It.Is(dt => + dt > DateTime.UtcNow - TimeSpan.FromMilliseconds(250) && + dt < DateTime.UtcNow + TimeSpan.FromMilliseconds(250))); // Heartbeat var heartbeatCalculator = new Mock(); @@ -851,10 +836,10 @@ public async Task TestHeartbeatTwoJob() .Returns(TimeSpan.FromMilliseconds(100)); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -885,13 +870,13 @@ public async Task TestHeartbeatTwoJob() .Setup(s => s.HeartbeatAsync(rawJobModel2.Object, TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); - var maintainer = new HeartbeatMaintainer(heartbeatCalculator.Object, executionEndArbiter.Object, + var monitor = new HeartbeatMonitor(heartbeatCalculator.Object, executionEndArbiter.Object, jobRepository.Object, jobSource.Object, CreateHealthStateUpdateService(), CreateSleepService(), - Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); + Options.Create(new CoreConfigurationModel {HaltOnFailure = false}), new NullLogger()); - await maintainer.RunAsync(TestContext.Current.CancellationToken); + await monitor.RunAsync(TestContext.Current.CancellationToken); Assert.Single(jobRepository.Invocations); diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Idempotency/IdempotencyMonitorTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Idempotency/IdempotencyMonitorTests.cs index 70ba3c8f..3f25c6cf 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Idempotency/IdempotencyMonitorTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Idempotency/IdempotencyMonitorTests.cs @@ -55,25 +55,24 @@ private static (Mock Entry, Mock JobModel, Mock< return (entry, jobModel, rawJobModel); } - private static void SetupMaintainerDelay(Mock arbiter) + private static void SetupMonitorDelay(Mock arbiter) { arbiter - .Setup(a => a.MaintainerDelayWaitAsync(It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + .Setup(a => a.IdempotencyMonitorDelayWaitAsync(It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); } [Fact(Timeout = 1000)] - public async Task RunAsync_PassesWaitLabelAndDescriptionToDelay() + public async Task RunAsync_DelaysUsingEffectiveMonitorInterval() { var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter - .Setup(a => a.MaintainerDelayWaitAsync(TimeSpan.FromSeconds(3), "Idempotency Monitor", "follow-up check", + .Setup(a => a.IdempotencyMonitorDelayWaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -99,7 +98,7 @@ public async Task RunAsync_PassesWaitLabelAndDescriptionToDelay() await monitor.RunAsync(TestContext.Current.CancellationToken); executionEndArbiter.Verify( - a => a.MaintainerDelayWaitAsync(TimeSpan.FromSeconds(3), "Idempotency Monitor", "follow-up check", + a => a.IdempotencyMonitorDelayWaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken), Times.Once); } @@ -107,13 +106,13 @@ public async Task RunAsync_PassesWaitLabelAndDescriptionToDelay() public async Task RunAsync_SleepsUsingEffectiveMonitorIntervalBetweenLoops() { var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter - .Setup(a => a.MaintainerDelayWaitAsync(TimeSpan.FromSeconds(3), It.IsAny(), It.IsAny(), + .Setup(a => a.IdempotencyMonitorDelayWaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -139,7 +138,7 @@ public async Task RunAsync_SleepsUsingEffectiveMonitorIntervalBetweenLoops() await monitor.RunAsync(TestContext.Current.CancellationToken); executionEndArbiter.Verify( - a => a.MaintainerDelayWaitAsync(TimeSpan.FromSeconds(3), It.IsAny(), It.IsAny(), + a => a.IdempotencyMonitorDelayWaitAsync(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken), Times.Once); } @@ -165,10 +164,10 @@ public async Task RunAsync_WhenCachedResultIsNullOrUnsuccessful_ReloadsUnblocked }; var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -184,9 +183,7 @@ public async Task RunAsync_WhenCachedResultIsNullOrUnsuccessful_ReloadsUnblocked jobRepository .Setup(r => r.GetAllIdempotencyBlockedJobsAsync(TestContext.Current.CancellationToken)) .ReturnsAsync([entry.Object]); - jobRepository - .Setup(r => r.ReloadUnblockedJobAsync(entry.Object, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + entry.SetupSet(e => e.State = JobState.Inactive); var idempotencyExecutionService = new Mock(MockBehavior.Strict); idempotencyExecutionService @@ -203,8 +200,7 @@ public async Task RunAsync_WhenCachedResultIsNullOrUnsuccessful_ReloadsUnblocked await monitor.RunAsync(TestContext.Current.CancellationToken); - jobRepository.Verify(r => r.ReloadUnblockedJobAsync(entry.Object, TestContext.Current.CancellationToken), - Times.Once); + entry.VerifySet(e => e.State = JobState.Inactive, Times.Once); jobRepository.Verify(r => r.RemoveJobAsync(It.IsAny(), It.IsAny()), Times.Never); idempotencyLock.Verify(l => l.UnlockAsync(It.IsAny()), Times.Once); @@ -231,10 +227,10 @@ public async Task RunAsync_WhenCachedResultIsSuccessAndAcknowledgeFails_RemovesJ }; var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -307,10 +303,10 @@ public async Task RunAsync_WhenCachedResultIsSuccessAndAcknowledgeSucceeds_Remov }; var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -367,7 +363,7 @@ public async Task RunAsync_WhenCachedResultIsSuccessAndAcknowledgeSucceeds_Remov public async Task RunAsync_WhenDisabled_ReturnsImmediately() { var monitor = new IdempotencyMonitor( - new Mock(MockBehavior.Strict).Object, + new Mock(MockBehavior.Strict).Object, new Mock(MockBehavior.Strict).Object, new Mock(MockBehavior.Strict).Object, new Mock(MockBehavior.Strict).Object, @@ -383,10 +379,10 @@ public async Task RunAsync_WhenLockNotAcquired_LeavesJobBlocked() var idempotencyLock = CreateLock(false); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); - SetupMaintainerDelay(executionEndArbiter); + var executionEndArbiter = new Mock(MockBehavior.Strict); + SetupMonitorDelay(executionEndArbiter); executionEndArbiter - .Setup(a => a.MaintainerShouldKeepRunning()) + .Setup(a => a.MonitorShouldKeepRunning()) .Returns(() => { if (doQuit) @@ -416,9 +412,7 @@ public async Task RunAsync_WhenLockNotAcquired_LeavesJobBlocked() await monitor.RunAsync(TestContext.Current.CancellationToken); idempotencyLock.Verify(l => l.UnlockAsync(It.IsAny()), Times.Once); - jobRepository.Verify( - r => r.ReloadUnblockedJobAsync(It.IsAny(), It.IsAny()), - Times.Never); + entry.VerifySet(e => e.State = JobState.Inactive, Times.Never); jobRepository.Verify(r => r.RemoveJobAsync(It.IsAny(), It.IsAny()), Times.Never); } diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobExecutorTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobExecutorTests.cs index 4a831375..1c3c7e73 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobExecutorTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobExecutorTests.cs @@ -52,8 +52,7 @@ public async Task ExecuteSingleJob(CoreJobResult safeRunnerResult) AcknowledgedSuccessfully = true, LoggedFailureSuccessfully = null }; - jobRepositoryEntry.Setup(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + jobRepositoryEntry.SetupSet(j => j.State = JobState.Complete); var runException = safeRunnerResult == CoreJobResult.Success ? null @@ -71,7 +70,7 @@ public async Task ExecuteSingleJob(CoreJobResult safeRunnerResult) .ReturnsAsync(ackResult); var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => @@ -121,8 +120,7 @@ public async Task ExecuteSingleJob(CoreJobResult safeRunnerResult) safeAcknowledgementService.Verify( s => s.AcknowledgeSafelyAsync(rawJobModel.Object, safeRunnerResult, runException, null, TestContext.Current.CancellationToken), Times.Once); - jobRepositoryEntry.Verify(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken), - Times.Once); + jobRepositoryEntry.VerifySet(j => j.State = JobState.Complete, Times.Once); idempotencyExecutionService.Verify( s => s.SetResultInCacheAsync(rawJobModel.Object, safeRunnerResult, ackResult, TestContext.Current.CancellationToken), Times.Once); @@ -133,7 +131,7 @@ public async Task ExecuteSingleJob(CoreJobResult safeRunnerResult) public async Task PrepareToExitOnNull() { var doQuit = false; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => @@ -168,8 +166,7 @@ public async Task PrepareToExitOnNull() public async Task WhenCachedResultIsSuccessAndAcknowledgeFails_SkipsExecution() { var (jobRepositoryEntry, jobModel, rawJobModel) = CreateRepositoryEntry(); - jobRepositoryEntry.Setup(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + jobRepositoryEntry.SetupSet(j => j.State = JobState.Complete); var cachedResult = new IdempotencyCacheResult { JobResult = CoreJobResult.Success, @@ -186,7 +183,7 @@ public async Task WhenCachedResultIsSuccessAndAcknowledgeFails_SkipsExecution() }; var calls = 0; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => ++calls <= 1); @@ -230,8 +227,7 @@ public async Task WhenCachedResultIsSuccessAndAcknowledgeFails_SkipsExecution() s => s.SetResultInCacheAsync(rawJobModel.Object, CoreJobResult.Success, failedAck, TestContext.Current.CancellationToken), Times.Once); - jobRepositoryEntry.Verify(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken), - Times.Once); + jobRepositoryEntry.VerifySet(j => j.State = JobState.Complete, Times.Once); jobRepository.Verify(r => r.RemoveJobAsync(jobRepositoryEntry.Object, TestContext.Current.CancellationToken), Times.Once); } @@ -240,8 +236,7 @@ public async Task WhenCachedResultIsSuccessAndAcknowledgeFails_SkipsExecution() public async Task WhenCachedResultIsSuccessAndAcknowledgeSucceeds_SkipsExecution() { var (jobRepositoryEntry, jobModel, rawJobModel) = CreateRepositoryEntry(); - jobRepositoryEntry.Setup(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + jobRepositoryEntry.SetupSet(j => j.State = JobState.Complete); var cachedResult = new IdempotencyCacheResult { JobResult = CoreJobResult.Success, @@ -258,7 +253,7 @@ public async Task WhenCachedResultIsSuccessAndAcknowledgeSucceeds_SkipsExecution }; var calls = 0; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => ++calls <= 1); @@ -303,8 +298,7 @@ public async Task WhenCachedResultIsSuccessAndAcknowledgeSucceeds_SkipsExecution s => s.SetResultInCacheAsync(rawJobModel.Object, CoreJobResult.Success, successAck, TestContext.Current.CancellationToken), Times.Once); - jobRepositoryEntry.Verify(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken), - Times.Once); + jobRepositoryEntry.VerifySet(j => j.State = JobState.Complete, Times.Once); jobRepository.Verify(r => r.RemoveJobAsync(jobRepositoryEntry.Object, TestContext.Current.CancellationToken), Times.Once); } @@ -320,8 +314,7 @@ public async Task WhenCachedResultIsUnsuccessful_RunsJobAsRetry(CoreJobResult sa AcknowledgedSuccessfully = true, LoggedFailureSuccessfully = null }; - jobRepositoryEntry.Setup(j => j.SetStateAsync(JobState.Complete, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + jobRepositoryEntry.SetupSet(j => j.State = JobState.Complete); var runException = safeRunnerResult == CoreJobResult.Success ? null @@ -339,7 +332,7 @@ public async Task WhenCachedResultIsUnsuccessful_RunsJobAsRetry(CoreJobResult sa .ReturnsAsync(ackResult); var calls = 0; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => ++calls <= 1); @@ -394,16 +387,14 @@ public async Task WhenCachedResultIsUnsuccessful_RunsJobAsRetry(CoreJobResult sa public async Task WhenIdempotencyLockNotAcquired_MarksJobBlockedAndContinues() { var (jobRepositoryEntry, jobModel, _) = CreateRepositoryEntry(); - jobRepositoryEntry - .Setup(j => j.SetStateAsync(JobState.BlockedByIdempotency, TestContext.Current.CancellationToken)) - .Returns(Task.CompletedTask); + jobRepositoryEntry.SetupSet(j => j.State = JobState.BlockedByIdempotency); var idempotencyLock = new Mock(MockBehavior.Strict); idempotencyLock.SetupGet(l => l.IsAcquired).Returns(false); idempotencyLock.Setup(l => l.UnlockAsync(It.IsAny())).Returns(Task.CompletedTask); var calls = 0; - var executionEndArbiter = new Mock(MockBehavior.Strict); + var executionEndArbiter = new Mock(MockBehavior.Strict); executionEndArbiter .Setup(a => a.ExecutorsShouldKeepRunning()) .Returns(() => ++calls <= 1); @@ -425,8 +416,7 @@ public async Task WhenIdempotencyLockNotAcquired_MarksJobBlockedAndContinues() await executor.RunAsync(0, TestContext.Current.CancellationToken); - jobRepositoryEntry.Verify( - j => j.SetStateAsync(JobState.BlockedByIdempotency, TestContext.Current.CancellationToken), Times.Once); + jobRepositoryEntry.VerifySet(j => j.State = JobState.BlockedByIdempotency, Times.Once); idempotencyExecutionService.Verify( s => s.GetCachedResultAsync(It.IsAny(), It.IsAny()), Times.Never); } diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobRepositoryTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobRepositoryTests.cs index 3d0cfcfb..e9098ca6 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobRepositoryTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/Jobs/JobRepositoryTests.cs @@ -32,39 +32,103 @@ private static JobRepository CreateRepository( Options.Create(new JobRepository.ConfigurationModel {BacklogSize = backlogSize})); } - [Fact(Timeout = 500)] - public async Task LoadAsync_WhenResponseHasNoItems_DoesNotTouchWatchedJobs() + private static Mock CreateJobModel(string messageId) { - var executionEndArbiter = new Mock(MockBehavior.Strict); - var jobLoaderStateService = new Mock(MockBehavior.Strict); - var sorter = new Mock(MockBehavior.Strict); + var jobModel = new Mock(MockBehavior.Strict); + jobModel.Setup(m => m.MessageId).Returns(messageId); + return jobModel; + } - var jobRepository = new JobRepository( - executionEndArbiter.Object, - jobLoaderStateService.Object, - sorter.Object, - Options.Create(new JobRepository.ConfigurationModel - { - BacklogSize = 0 - })); + [Fact(Timeout = 2000)] + public async Task GetNextJobAsync_WhenOnlyUnblockedJobIsDisposed_FallsBackToInactiveQueue() + { + var jobRepository = CreateRepository(); - await jobRepository.LoadAsync([], TestContext.Current.CancellationToken); + await jobRepository.LoadAsync( + [ + new JobEnvelope + { + JobModel = CreateJobModel("unblocked-then-disposed").Object, + RawJobModel = new Mock(MockBehavior.Strict).Object + }, + new JobEnvelope + { + JobModel = CreateJobModel("still-inactive").Object, + RawJobModel = new Mock(MockBehavior.Strict).Object + } + ], + TestContext.Current.CancellationToken); - Assert.Empty(jobRepository.WatchedJobs); - Assert.Empty(sorter.Invocations); + var unblockedEntry = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + Assert.NotNull(unblockedEntry); + unblockedEntry.State = JobState.BlockedByIdempotency; + unblockedEntry.State = JobState.Inactive; + + await jobRepository.RemoveJobAsync(unblockedEntry, TestContext.Current.CancellationToken); + Assert.True(unblockedEntry.IsDisposed); + + var nextJob = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + + Assert.NotSame(unblockedEntry, nextJob); + Assert.NotNull(nextJob); + Assert.False(nextJob.IsDisposed); + Assert.Equal("still-inactive", nextJob.JobModel.MessageId); + Assert.Equal(JobState.Active, nextJob.State); } [Fact(Timeout = 2000)] - public async Task ReloadUnblockedJobAsync_ShortlistsJobAheadOfInactiveQueue() + public async Task GetNextJobAsync_WhenUnblockedJobIsDisposed_SkipsToNextUnblockedJob() { - var executionEndArbiter = new Mock(MockBehavior.Strict); - executionEndArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + var jobRepository = CreateRepository(); + + await jobRepository.LoadAsync( + [ + new JobEnvelope + { + JobModel = CreateJobModel("first-unblocked").Object, + RawJobModel = new Mock(MockBehavior.Strict).Object + }, + new JobEnvelope + { + JobModel = CreateJobModel("second-unblocked").Object, + RawJobModel = new Mock(MockBehavior.Strict).Object + }, + new JobEnvelope + { + JobModel = CreateJobModel("inactive").Object, + RawJobModel = new Mock(MockBehavior.Strict).Object + } + ], + TestContext.Current.CancellationToken); + + var firstUnblocked = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + var secondUnblocked = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + Assert.NotNull(firstUnblocked); + Assert.NotNull(secondUnblocked); + firstUnblocked.State = JobState.BlockedByIdempotency; + firstUnblocked.State = JobState.Inactive; + secondUnblocked.State = JobState.BlockedByIdempotency; + secondUnblocked.State = JobState.Inactive; + + await jobRepository.RemoveJobAsync(firstUnblocked, TestContext.Current.CancellationToken); + Assert.True(firstUnblocked.IsDisposed); + + var nextJob = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + + Assert.NotNull(nextJob); + Assert.Same(secondUnblocked, nextJob); + Assert.False(nextJob.IsDisposed); + Assert.Equal(JobState.Active, nextJob.State); + Assert.Equal("second-unblocked", nextJob.JobModel.MessageId); + } + + [Fact(Timeout = 500)] + public async Task LoadAsync_WhenResponseHasNoItems_DoesNotTouchWatchedJobs() + { + var executionEndArbiter = new Mock(MockBehavior.Strict); var jobLoaderStateService = new Mock(MockBehavior.Strict); - var sorter = new Mock(); - sorter - .Setup(s => s.GetSortedListOfJobs(It.IsAny>())) - .Returns((List input) => input); + var sorter = new Mock(MockBehavior.Strict); var jobRepository = new JobRepository( executionEndArbiter.Object, @@ -75,48 +139,10 @@ public async Task ReloadUnblockedJobAsync_ShortlistsJobAheadOfInactiveQueue() BacklogSize = 0 })); - var queuedModel = new Mock(MockBehavior.Strict); - queuedModel.Setup(m => m.MessageId).Returns("queued"); - var queuedRaw = new Mock(MockBehavior.Strict).Object; - var unblockedModel = new Mock(MockBehavior.Strict); - unblockedModel.Setup(m => m.MessageId).Returns("unblocked"); - - await jobRepository.LoadAsync( - [ - new JobEnvelope - { - JobModel = queuedModel.Object, - RawJobModel = queuedRaw - } - ], - TestContext.Current.CancellationToken); - - var queuedEntry = Assert.Single(jobRepository.WatchedJobs, - j => j.JobModel.MessageId == "queued"); - Assert.Same(queuedRaw, queuedEntry.RawJobModel); - - var blockedEntry = new Mock(MockBehavior.Strict); - blockedEntry.Setup(e => e.JobModel).Returns(unblockedModel.Object); - var blockedState = JobState.BlockedByIdempotency; - blockedEntry.Setup(e => e.State).Returns(() => blockedState); - blockedEntry - .Setup(e => e.SetStateAsync(JobState.Inactive, TestContext.Current.CancellationToken)) - .Callback(() => blockedState = JobState.Inactive) - .Returns(Task.CompletedTask); - blockedEntry - .Setup(e => e.SetStateAsync(JobState.Active, TestContext.Current.CancellationToken)) - .Callback(() => blockedState = JobState.Active) - .Returns(Task.CompletedTask); - jobRepository.WatchedJobs.Add(blockedEntry.Object); - - await jobRepository.ReloadUnblockedJobAsync(blockedEntry.Object, TestContext.Current.CancellationToken); - - var nextJob = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + await jobRepository.LoadAsync([], TestContext.Current.CancellationToken); - Assert.Same(blockedEntry.Object, nextJob); - blockedEntry.Verify(e => e.SetStateAsync(JobState.Inactive, TestContext.Current.CancellationToken), - Times.Once); - blockedEntry.Verify(e => e.SetStateAsync(JobState.Active, TestContext.Current.CancellationToken), Times.Once); + Assert.Empty(jobRepository.WatchedJobs); + Assert.Empty(sorter.Invocations); } [Fact(Timeout = 2000)] @@ -142,13 +168,16 @@ await jobRepository.LoadAsync( ], TestContext.Current.CancellationToken); Assert.Equal([0, 1], inactiveCounts); - Assert.Equal([0, 1], watchedCounts); + Assert.Equal([0, 1, 1], watchedCounts); var job = Assert.Single(jobRepository.WatchedJobs); + Assert.False(job.IsDisposed); await jobRepository.RemoveJobAsync(job, TestContext.Current.CancellationToken); + Assert.Equal(JobState.Complete, job.State); + Assert.True(job.IsDisposed); Assert.Equal([0, 1, 0], inactiveCounts); - Assert.Equal([0, 1, 0], watchedCounts); + Assert.Equal([0, 1, 1, 0], watchedCounts); } [Fact] @@ -646,9 +675,9 @@ public async Task TestLoadJobsAndWaitForJob_RequeuedBacklog() Assert.True(gottenJob.CanHeartbeat); // Imitate JobExecutor by marking the task as blocked by idempotency. // Not strictly necessary, but it does imitate the logic of JobExecutor to put the job on the radar of the Idempotency Monitor. - await gottenJob.SetStateAsync(JobState.BlockedByIdempotency, TestContext.Current.CancellationToken); + gottenJob.State = JobState.BlockedByIdempotency; // Reload the unblocked job, imitating the Idempotency Monitor (this puts it into the queue) - await jobRepository.ReloadUnblockedJobAsync(gottenJob, TestContext.Current.CancellationToken); + gottenJob.State = JobState.Inactive; Assert.Equal(responseSize, await jobRepository.GetWatchedJobsCountAsync(TestContext.Current.CancellationToken)); @@ -770,9 +799,9 @@ public async Task TestLoadJobsAndWaitForJob_RequeuedBacklogThenEmpty() Assert.True(gottenJob.CanHeartbeat); // Imitate JobExecutor by marking the task as blocked by idempotency. // Not strictly necessary, but it does imitate the logic of JobExecutor to put the job on the radar of the Idempotency Monitor. - await gottenJob.SetStateAsync(JobState.BlockedByIdempotency, TestContext.Current.CancellationToken); + gottenJob.State = JobState.BlockedByIdempotency; // Reload the unblocked job, imitating the Idempotency Monitor (this puts it into the queue) - await jobRepository.ReloadUnblockedJobAsync(gottenJob, TestContext.Current.CancellationToken); + gottenJob.State = JobState.Inactive; // Wait for another job to finish await manualResetEvent.WaitAsync(TestContext.Current.CancellationToken); @@ -1101,7 +1130,8 @@ public async Task TestRemoveJobsAsync() Options.Create(options)); var job = new Mock(); - job.Setup(j => j.State).Returns(JobState.Complete); + job.SetupProperty(j => j.State, JobState.Active); + job.Setup(j => j.Dispose()).Callback(() => job.Object.State = JobState.Complete); jobRepository.WatchedJobs.Add(job.Object); Assert.Equal(0, await jobRepository.GetInactiveJobCountAsync(TestContext.Current.CancellationToken)); @@ -1109,6 +1139,8 @@ public async Task TestRemoveJobsAsync() await jobRepository.RemoveJobAsync(job.Object, TestContext.Current.CancellationToken); + job.Verify(j => j.Dispose(), Times.Once); + Assert.Equal(JobState.Complete, job.Object.State); Assert.Equal(0, await jobRepository.GetWatchedJobsCountAsync(TestContext.Current.CancellationToken)); } @@ -1136,11 +1168,13 @@ public async Task TestRemoveJobsAsyncB() Options.Create(options)); var job = new Mock(); - job.Setup(j => j.State).Returns(JobState.Complete); + job.SetupProperty(j => j.State, JobState.Active); + job.Setup(j => j.Dispose()).Callback(() => job.Object.State = JobState.Complete); jobRepository.WatchedJobs.Add(job.Object); var job2 = new Mock(); - job2.Setup(j => j.State).Returns(JobState.Complete); + job2.SetupProperty(j => j.State, JobState.Active); + job2.Setup(j => j.Dispose()).Callback(() => job2.Object.State = JobState.Complete); jobRepository.WatchedJobs.Add(job2.Object); Assert.Equal(0, await jobRepository.GetInactiveJobCountAsync(TestContext.Current.CancellationToken)); @@ -1148,6 +1182,10 @@ public async Task TestRemoveJobsAsyncB() await jobRepository.RemoveJobAsync(job.Object, TestContext.Current.CancellationToken); + job.Verify(j => j.Dispose(), Times.Once); + job2.Verify(j => j.Dispose(), Times.Never); + Assert.Equal(JobState.Complete, job.Object.State); + Assert.Equal(JobState.Active, job2.Object.State); Assert.Equal(1, await jobRepository.GetWatchedJobsCountAsync(TestContext.Current.CancellationToken)); } @@ -1230,6 +1268,67 @@ public async Task TestWaitForDemand_True() Assert.Empty(sorter.Invocations); } + [Fact(Timeout = 2000)] + public async Task UnblockingJob_ShortlistsJobAheadOfInactiveQueue() + { + var executionEndArbiter = new Mock(MockBehavior.Strict); + executionEndArbiter.Setup(a => a.ShouldKeepRunning()).Returns(true); + + var jobLoaderStateService = new Mock(MockBehavior.Strict); + var sorter = new Mock(); + sorter + .Setup(s => s.GetSortedListOfJobs(It.IsAny>())) + .Returns((List input) => input); + + var jobRepository = new JobRepository( + executionEndArbiter.Object, + jobLoaderStateService.Object, + sorter.Object, + Options.Create(new JobRepository.ConfigurationModel + { + BacklogSize = 0 + })); + + var unblockedModel = new Mock(MockBehavior.Strict); + unblockedModel.Setup(m => m.MessageId).Returns("unblocked"); + var unblockedRaw = new Mock(MockBehavior.Strict).Object; + var queuedModel = new Mock(MockBehavior.Strict); + queuedModel.Setup(m => m.MessageId).Returns("queued"); + var queuedRaw = new Mock(MockBehavior.Strict).Object; + + await jobRepository.LoadAsync( + [ + new JobEnvelope + { + JobModel = unblockedModel.Object, + RawJobModel = unblockedRaw + } + ], + TestContext.Current.CancellationToken); + + var unblockedEntry = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + Assert.NotNull(unblockedEntry); + Assert.Equal("unblocked", unblockedEntry.JobModel.MessageId); + + await jobRepository.LoadAsync( + [ + new JobEnvelope + { + JobModel = queuedModel.Object, + RawJobModel = queuedRaw + } + ], + TestContext.Current.CancellationToken); + + unblockedEntry.State = JobState.BlockedByIdempotency; + unblockedEntry.State = JobState.Inactive; + + var nextJob = await jobRepository.GetNextJobAsync(TestContext.Current.CancellationToken); + + Assert.Same(unblockedEntry, nextJob); + Assert.Equal(JobState.Active, nextJob.State); + } + [Fact(Timeout = 2000)] public async Task WaitForEmptyRepositoryAsync_CompletesWhenLastJobRemoved() { diff --git a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/SourceMessages/SourceMessageSorterTests.cs b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/SourceMessages/SourceMessageSorterTests.cs index 5440a537..4e926b8b 100644 --- a/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/SourceMessages/SourceMessageSorterTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.UnitTests/Tests/Services/SourceMessages/SourceMessageSorterTests.cs @@ -1,4 +1,5 @@ using RedShirt.Example.JobWorker.Common.Models; +using RedShirt.Example.JobWorker.Core.Enums; using RedShirt.Example.JobWorker.Core.Models; using RedShirt.Example.JobWorker.Core.Services.SourceMessages; @@ -29,7 +30,9 @@ public void Test_Message_Retention(int numberOfMessages) var output = sorter.GetSortedListOfJobs(items.Select(i => new JobRepositoryEntry { JobModel = i, - RawJobModel = new Mock(MockBehavior.Strict).Object + RawJobModel = new Mock(MockBehavior.Strict).Object, + LastHeartbeatTime = DateTime.UtcNow, + State = JobState.Inactive }).ToList()); Assert.Equal(numberOfMessages, output.Count);