From 480633162cae079df67682809cf01b97ad31055c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 7 Sep 2026 18:57:09 -0500 Subject: [PATCH 1/4] Add native broker execution tracking and managed node delivery --- .agents/skills/foundatio/SKILL.md | 8 + docs/guide/locks.md | 4 + docs/guide/messaging.md | 22 ++ .../AwsFoundatioBuilderExtensions.cs | 2 +- .../AwsMessageTransport.Administration.cs | 60 ++++ .../AwsMessageTransport.Batching.cs | 4 +- .../AwsMessageTransport.NodeSubscriptions.cs | 141 ++++++++ src/Foundatio.Aws/AwsMessageTransport.cs | 47 ++- .../AwsMessageTransportOptions.cs | 13 +- src/Foundatio.Aws/AwsRequestBatcher.cs | 8 +- .../Messaging/RedisMessageExecutionScripts.cs | 120 +++++++ .../Messaging/RedisMessageExecutionStore.cs | 338 ++++++++++++++++++ .../RedisMessageExecutionStoreOptions.cs | 46 +++ .../RedisFoundatioBuilderExtensions.cs | 24 ++ src/Foundatio.Redis/RedisLockProvider.cs | 72 ++++ .../MessageExecutionStoreConformanceTests.cs | 90 +++++ .../RecordingMessageTransport.cs | 11 +- src/Foundatio/FoundatioServicesExtensions.cs | 7 + .../Lock/LockOwnershipLostException.cs | 6 + src/Foundatio/Messaging/IMessageContext.cs | 6 + .../Messaging/IMessageProcessingObserver.cs | 11 + .../Messaging/InMemoryMessageTransport.cs | 8 +- .../Messaging/MessageAdministration.cs | 139 +++++++ src/Foundatio/Messaging/MessageBus.cs | 91 ++++- src/Foundatio/Messaging/MessageClientCore.cs | 249 +++++++------ .../Messaging/MessageDeliveryLease.cs | 118 ++++++ .../Messaging/MessageHandlerRegistration.cs | 3 +- .../Messaging/MessageNodeSubscription.cs | 59 +++ src/Foundatio/Messaging/MessageOutcome.cs | 55 +++ src/Foundatio/Messaging/MessageTransport.cs | 9 +- src/Foundatio/Messaging/ReceivedMessage.cs | 11 +- .../Messaging/Tracking/ExecutionHeaders.cs | 11 + .../Tracking/IMessageExecutionStore.cs | 62 ++++ .../Tracking/InMemoryMessageExecutionStore.cs | 264 ++++++++++++++ .../Tracking/MessageExecutionCounters.cs | 40 +++ .../Tracking/MessageExecutionOptions.cs | 32 ++ .../Tracking/MessageExecutionPipeline.cs | 257 +++++++++++++ .../Tracking/MessageExecutionState.cs | 52 +++ .../MessageExecutionStoreExtensions.cs | 24 ++ .../Tracking/MessageProcessingContext.cs | 197 ++++++++++ tests/Foundatio.Aws.Tests/AwsEnvelopeTests.cs | 4 +- .../AwsMessageAdministrationTests.cs | 59 +++ .../AwsNodeSubscriptionTests.cs | 40 +++ .../RedisMessageExecutionStoreTests.cs | 12 + .../RedisResourceLockTests.cs | 42 +++ .../Messaging/MessageEndpointPolicyTests.cs | 171 +++++++++ .../Messaging/MessageExecutionStoreTests.cs | 8 + 47 files changed, 2918 insertions(+), 139 deletions(-) create mode 100644 src/Foundatio.Aws/AwsMessageTransport.Administration.cs create mode 100644 src/Foundatio.Aws/AwsMessageTransport.NodeSubscriptions.cs create mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs create mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs create mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs create mode 100644 src/Foundatio.Redis/RedisLockProvider.cs create mode 100644 src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs create mode 100644 src/Foundatio/Lock/LockOwnershipLostException.cs create mode 100644 src/Foundatio/Messaging/IMessageProcessingObserver.cs create mode 100644 src/Foundatio/Messaging/MessageAdministration.cs create mode 100644 src/Foundatio/Messaging/MessageDeliveryLease.cs create mode 100644 src/Foundatio/Messaging/MessageNodeSubscription.cs create mode 100644 src/Foundatio/Messaging/MessageOutcome.cs create mode 100644 src/Foundatio/Messaging/Tracking/ExecutionHeaders.cs create mode 100644 src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs create mode 100644 src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionState.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs create mode 100644 src/Foundatio/Messaging/Tracking/MessageProcessingContext.cs create mode 100644 tests/Foundatio.Aws.Tests/AwsMessageAdministrationTests.cs create mode 100644 tests/Foundatio.Aws.Tests/AwsNodeSubscriptionTests.cs create mode 100644 tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs create mode 100644 tests/Foundatio.Redis.Tests/RedisResourceLockTests.cs create mode 100644 tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs create mode 100644 tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index 69bfa40ac..9ab76d8c8 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -358,3 +358,11 @@ Validate a custom transport or job store against the shared conformance suites i | `Foundatio.Xunit` | xUnit v2 test logging, retry attributes | | `Foundatio.Xunit.v3` | xUnit v3 test logging, retry attributes | | `Foundatio.DataProtection` | ASP.NET Core Data Protection key storage via `IFileStorage` | + +## Broker execution integration + +- Optional broker execution history uses `IMessageExecutionStore` with `.Messaging.UseInMemoryExecutionTracking()` or `.UseRedisExecutionTracking()`. `MessageExecutionPipeline` and native `MessageProcessingContext` support progress, cancellation, and attempt-fenced state. Do not enqueue the same delivery through `IJobRuntimeStore`; the bus owns receiving and settlement. +- `ConsumeWithOutcomeAsync` handles returned Success/Retry/DeadLetter/Unsettled outcomes. Endpoint receive capacity, visibility, automatic renewal, and graceful drain are configured with MessageHandlerOptions. +- `SubscribeNodeAsync` supplies independent best-effort node broadcasts. AWS uses managed tagged resources and startup stale cleanup, not native TTL. Acknowledge-before-callback intentionally allows lost notifications. +- Native `MessageAdministration` owns bounded dead-letter inspection/replay. Send-before-delete is at least once, not atomic. Optional replay preparation can create fresh tracked execution IDs. +- `.Locking.UseRedis()` uses native RedisLockProvider with ownership-checked renewal/release. It coordinates live resource ownership, not persistent duplicate detection. diff --git a/docs/guide/locks.md b/docs/guide/locks.md index 9cce8cc9c..b5cef18c2 100644 --- a/docs/guide/locks.md +++ b/docs/guide/locks.md @@ -676,3 +676,7 @@ if (lck is not null) - [Jobs](./jobs) - Background jobs with distributed locking - [Resilience](./resilience) - Retry policies for lock acquisition - [Serialization](./serialization) - Serializer configuration and performance + +## Native Redis resource locks + +`foundatio.Locking.UseRedis()` registers `RedisLockProvider` using the application's shared `IConnectionMultiplexer`. Acquisition uses Redis `SET NX` with expiration; renewal and release compare the ownership token atomically. `LockOwnershipLostException` signals that a holder may no longer extend its lease. Expired holders cannot release a successor's lock. These are leased resource locks, not deduplication or fencing of external writes. Configure distinct key prefixes for unrelated applications. diff --git a/docs/guide/messaging.md b/docs/guide/messaging.md index af3ec9713..a2cd7b776 100644 --- a/docs/guide/messaging.md +++ b/docs/guide/messaging.md @@ -153,3 +153,25 @@ SQS/SNS support varies by destination role. Do not infer topic capabilities from ## Migration The former publish-only interfaces live under `Foundatio.Messaging.Legacy`; `Messaging.AddLegacyAdapter()` adapts them to this bus. Legacy `IQueue` workers migrate to an explicit consumer and `SendAsync`. There is no `Deliveries.Both`, handler-name-derived subscriber identity, or `PerInstance` flag. Choose `AddConsumer` or `AddSubscriber`, and use a stable service identity or explicit names for durable subscribers. + +## Broker-driven execution tracking + +For queues that need progress, cancellation, and operational history, configure `.Messaging.UseInMemoryExecutionTracking()` or `.Messaging.UseRedisExecutionTracking()`. This registers `IMessageExecutionStore`; it does not create runnable jobs, a job worker, or a second scheduler. Delivery remains owned by the message bus. + +`MessageExecutionPipeline` runs a raw delivery callback with `MessageProcessingContext`, applies returned `MessageOutcome` values, and persists only confirmed settlement. Pass it to a manual-ack raw `ConsumeAsync` endpoint with `WaitForManualSettlement = false`. The producer creates a unique execution state before sending and puts its ID in `ExecutionHeaders.ExecutionId`. Integrations such as Foundatio.Mediator perform this composition automatically. + +Processing claims and subsequent progress/status writes are fenced by the broker attempt number. A terminal execution is not started again. Expired history does not block valid broker work; progress/history updates then become no-ops. Tracking is not persistent deduplication or exactly-once delivery. Broker acceptance and persistence are separate operations; uncertain sends and acknowledgments remain nonterminal. Application side effects must tolerate retries. + +Endpoint options own `MaxConcurrency`, `PrefetchCount`, `VisibilityTimeout`, `AutoRenewLock`, and `ShutdownTimeout`. Manual and automatic renewal share one conservative delivery deadline. Lost ownership cancels processing. Stopping receiving allows admitted handlers to drain within the configured shutdown timeout. + +`ConsumeWithOutcomeAsync` accepts `MessageOutcome.Success`, `Retry`, `DeadLetter`, or `Unsettled` without requiring expected application failures to throw exceptions. + +## Per-node broadcasts + +`SubscribeNodeAsync` receives an independent best-effort copy on each running node, acknowledging before callbacks. Memory and Redis use native expiring subscriptions. AWS creates a tagged SQS queue and SNS subscription, heartbeats ownership, cleans up on disposal, and reaps stale resources when another node starts. AWS managed resources do not advertise native TTL expiration. Configure IAM for the required resource lifecycle, tagging, and discovery operations, including when durable topology is externally provisioned. + +Use this for invalidation and UI refresh signals. A disconnected, paused, or restarting node may miss events and should refresh authoritative state. Use durable service subscriptions for work that needs replay and retries. + +## Operational recovery + +`MessageAdministration` supplies statistics, bounded dead-letter inspection, deletion, and replay through native capabilities. Providers without native inspection use bounded receive/hold/release. Scans stop at 1,000 messages or ten seconds; their cleanup has an independent timeout. A successful replacement send precedes deleting the original. That fallback is not transactional and an uncertain send can produce a duplicate. Callers may prepare fresh execution metadata for an operator replay while retaining the failed history. diff --git a/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs b/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs index 219255ee8..871a4d02c 100644 --- a/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs +++ b/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs @@ -19,7 +19,7 @@ public static FoundatioBuilder.MessagingBuilder UseAws(this FoundatioBuilder.Mes { return builder.UseTransport(sp => { - var options = new AwsMessageTransportOptions(); + var options = new AwsMessageTransportOptions { LoggerFactory = sp.GetService() }; BindFromConfiguration(options, sp.GetService()?.GetSection("Aws")); configure?.Invoke(options); return new AwsMessageTransport(options); diff --git a/src/Foundatio.Aws/AwsMessageTransport.Administration.cs b/src/Foundatio.Aws/AwsMessageTransport.Administration.cs new file mode 100644 index 000000000..84d67c473 --- /dev/null +++ b/src/Foundatio.Aws/AwsMessageTransport.Administration.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; + +namespace Foundatio.Messaging; + +public sealed partial class AwsMessageTransport +{ + /// Validates existence and declared SQS attributes, reporting all mismatches together. + public async Task ValidateAsync(IReadOnlyList declarations, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(declarations); + var problems = new List(); + foreach (var declaration in declarations) + { + if (!await ExistsAsync(declaration.Address, cancellationToken).ConfigureAwait(false)) + { + problems.Add($"Destination '{declaration.Address.Key}' does not exist."); + continue; + } + if (declaration.Address.Role != DestinationRole.Queue) continue; + ValidateQueueArguments(declaration.ProviderArguments); + string url = await ResolveQueueUrlAsync(declaration.Address, cancellationToken).ConfigureAwait(false); + var response = await _sqs.Value.GetQueueAttributesAsync(new GetQueueAttributesRequest + { + QueueUrl = url, + AttributeNames = ["All"] + }, cancellationToken).ConfigureAwait(false); + foreach (var expected in declaration.ProviderArguments ?? new Dictionary()) + { + response.Attributes.TryGetValue(expected.Key, out var actual); + if (!String.Equals(expected.Value, actual, StringComparison.Ordinal)) + problems.Add($"Queue '{declaration.Address.Name}' attribute {expected.Key} is '{actual ?? "(unset)"}', expected '{expected.Value}'."); + } + if (response.Attributes.TryGetValue("RedrivePolicy", out string? redrive) && !String.IsNullOrEmpty(redrive)) + problems.Add($"Queue '{declaration.Address.Name}' has a broker RedrivePolicy. Foundatio owns delivery attempts; remove the broker policy."); + } + if (problems.Count > 0) throw new InvalidOperationException(String.Join(Environment.NewLine, problems)); + } + + private static void ValidateQueueArguments(IReadOnlyDictionary? arguments) + { + if (arguments is null) return; + if (arguments.TryGetValue("RedrivePolicy", out var policy) && !String.IsNullOrEmpty(policy)) + throw new ArgumentException("Foundatio owns retry and dead-letter policy; do not configure a broker RedrivePolicy.", nameof(arguments)); + foreach (var argument in arguments) + { + if (argument.Key is not ("VisibilityTimeout" or "MessageRetentionPeriod")) continue; + int min = argument.Key == "VisibilityTimeout" ? 1 : 60; + int max = argument.Key == "VisibilityTimeout" ? 43200 : 1209600; + if (!Int32.TryParse(argument.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int value) || value < min || value > max) + throw new ArgumentException($"SQS {argument.Key} must be between {min} and {max} seconds.", nameof(arguments)); + } + } + +} diff --git a/src/Foundatio.Aws/AwsMessageTransport.Batching.cs b/src/Foundatio.Aws/AwsMessageTransport.Batching.cs index 242e9737a..98bdd9719 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.Batching.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.Batching.cs @@ -48,9 +48,9 @@ public async Task SendAsync(DestinationAddress destination, IReadOnl string address = topic ? await ResolveTopicArnAsync(destination.Name, ct).ConfigureAwait(false) : await ResolveQueueUrlAsync(destination, ct).ConfigureAwait(false); for (int offset = 0; offset < prepared.Count;) { - var batch = new List(10); + var batch = new List(_options.MaxBatchSize); int bytes = 0; - while (offset < prepared.Count && batch.Count < 10 && bytes + prepared[offset].Bytes <= maximumBytes) + while (offset < prepared.Count && batch.Count < _options.MaxBatchSize && bytes + prepared[offset].Bytes <= maximumBytes) { var entry = prepared[offset++]; batch.Add(entry); diff --git a/src/Foundatio.Aws/AwsMessageTransport.NodeSubscriptions.cs b/src/Foundatio.Aws/AwsMessageTransport.NodeSubscriptions.cs new file mode 100644 index 000000000..5bbd477f6 --- /dev/null +++ b/src/Foundatio.Aws/AwsMessageTransport.NodeSubscriptions.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Messaging; + +public sealed partial class AwsMessageTransport +{ + private const string NodeRoleTag = "fnd:role"; + private const string NodeTopicTag = "fnd:topic"; + private const string NodeHeartbeatTag = "fnd:heartbeat"; + + /// Owns a tagged SQS subscription for one node; active nodes renew it and startup reaps stale peers. + public async Task OpenNodeSubscriptionAsync(MessageNodeSubscriptionOptions options, CancellationToken cancellationToken = default) + { + options.Validate(); + ArgumentOutOfRangeException.ThrowIfLessThan(options.MessageRetention, TimeSpan.FromMinutes(1)); + ArgumentOutOfRangeException.ThrowIfGreaterThan(options.MessageRetention, TimeSpan.FromDays(14)); + // A fresh identity avoids queue-deleted-recently failures and prevents two processes adopting one receipt namespace. + var source = DestinationAddress.ForSubscription(options.Topic, $"node-{options.NodeId}-{Guid.NewGuid():N}"); + await EnsureSubscriptionAsync(source, cancellationToken).ConfigureAwait(false); + string url = await ResolveQueueUrlAsync(source, cancellationToken).ConfigureAwait(false); + try + { + await _sqs.Value.SetQueueAttributesAsync(new SetQueueAttributesRequest + { + QueueUrl = url, + Attributes = new Dictionary + { + ["MessageRetentionPeriod"] = Math.Ceiling(options.MessageRetention.TotalSeconds).ToString(CultureInfo.InvariantCulture) + } + }, cancellationToken).ConfigureAwait(false); + await TagNodeAsync(url, options.Topic, cancellationToken).ConfigureAwait(false); + await SweepNodesAsync(options.Topic, options.StaleAfter, cancellationToken).ConfigureAwait(false); + return new NodeOwner(this, source, url, options); + } + catch + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try { await DeleteAsync(source, cleanup.Token).ConfigureAwait(false); } catch { } + throw; + } + } + + private Task TagNodeAsync(string url, string topic, CancellationToken cancellationToken) + => _sqs.Value.TagQueueAsync(new TagQueueRequest + { + QueueUrl = url, + Tags = new Dictionary + { + [NodeRoleTag] = "node-subscription", + [NodeTopicTag] = topic, + [NodeHeartbeatTag] = DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture) + } + }, cancellationToken); + + private async Task SweepNodesAsync(string topic, TimeSpan staleAfter, CancellationToken cancellationToken) + { + string physicalPrefix = SanitizeResourceName(_options.ResourcePrefix + topic + "/node-"); + physicalPrefix = physicalPrefix[..Math.Min(32, physicalPrefix.Length)]; + string? cursor = null; + do + { + var page = await _sqs.Value.ListQueuesAsync(new ListQueuesRequest + { + QueueNamePrefix = physicalPrefix, + MaxResults = 100, + NextToken = cursor + }, cancellationToken).ConfigureAwait(false); + foreach (string url in page.QueueUrls ?? []) + { + try + { + var tags = (await _sqs.Value.ListQueueTagsAsync(new ListQueueTagsRequest { QueueUrl = url }, cancellationToken).ConfigureAwait(false)).Tags; + if (tags is null || !tags.TryGetValue(NodeRoleTag, out var role) || role != "node-subscription" + || !tags.TryGetValue(NodeTopicTag, out var ownedTopic) || ownedTopic != topic + || !tags.TryGetValue(NodeHeartbeatTag, out var heartbeat) + || !DateTimeOffset.TryParse(heartbeat, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var lastSeen) + || lastSeen >= DateTimeOffset.UtcNow - staleAfter) continue; + // Re-read ownership immediately before deletion; a peer may have resumed during the scan. + var current = (await _sqs.Value.ListQueueTagsAsync(new ListQueueTagsRequest { QueueUrl = url }, cancellationToken).ConfigureAwait(false)).Tags; + if (current is null || !current.TryGetValue(NodeHeartbeatTag, out var value) || value != heartbeat) continue; + var attributes = await _sqs.Value.GetQueueAttributesAsync(new GetQueueAttributesRequest { QueueUrl = url, AttributeNames = ["QueueArn"] }, cancellationToken).ConfigureAwait(false); + string? topicArn = await FindTopicArnAsync(topic, cancellationToken).ConfigureAwait(false); + if (topicArn is not null && await FindSubscriptionArnAsync(topicArn, attributes.QueueARN, cancellationToken).ConfigureAwait(false) is { } arn) + await _sns.Value.UnsubscribeAsync(arn, cancellationToken).ConfigureAwait(false); + await _sqs.Value.DeleteQueueAsync(url, cancellationToken).ConfigureAwait(false); + } + catch (QueueDoesNotExistException) { } + } + cursor = page.NextToken; + } while (!String.IsNullOrEmpty(cursor)); + } + + private sealed class NodeOwner : IManagedNodeSubscription + { + private readonly AwsMessageTransport _owner; + private readonly CancellationTokenSource _stop = new(); + private readonly Task _heartbeat; + private readonly ILogger _logger; + private int _disposed; + public DestinationAddress Source { get; } + + public NodeOwner(AwsMessageTransport owner, DestinationAddress source, string url, MessageNodeSubscriptionOptions options) + { + _owner = owner; + _logger = owner._options.LoggerFactory?.CreateLogger() ?? NullLogger.Instance; + Source = source; + _heartbeat = HeartbeatAsync(url, options); + } + + private async Task HeartbeatAsync(string url, MessageNodeSubscriptionOptions options) + { + while (!_stop.IsCancellationRequested) + { + try + { + await Task.Delay(options.HeartbeatInterval, _stop.Token).ConfigureAwait(false); + await _owner.TagNodeAsync(url, options.Topic, _stop.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) { return; } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to renew node subscription {Source}; retrying on the next heartbeat", Source); } + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; + await _stop.CancelAsync().ConfigureAwait(false); + await _heartbeat.ConfigureAwait(false); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try { await _owner.DeleteAsync(Source, cleanup.Token).ConfigureAwait(false); } + finally { _stop.Dispose(); } + } + } +} diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index 829b0445d..054b30702 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -30,7 +30,7 @@ namespace Foundatio.Messaging; /// of, so those capabilities are intentionally not implemented (the core owns retry/dead-lettering). /// public sealed partial class AwsMessageTransport : IMessageTransport, ISupportsPull, ISupportsVisibilityTimeout, - ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsProvisioning, ISupportsStats, ITransportInfo + ISupportsManagedNodeSubscriptions, ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsProvisioning, ISupportsStats, ITransportInfo { private const string EnvelopeAttributeName = "fnd.envelope"; private const string HeadersAttributeName = "fnd.headers"; @@ -48,7 +48,8 @@ public sealed partial class AwsMessageTransport : IMessageTransport, ISupportsPu private readonly ConcurrentDictionary _queueUrls = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _topicArns = new(StringComparer.Ordinal); private int _isDisposed; - private readonly bool _ownsClients = true; + private readonly bool _ownsSqs = true; + private readonly bool _ownsSns = true; public AwsMessageTransport(AwsMessageTransportOptions options) { @@ -66,7 +67,16 @@ public AwsMessageTransport(AwsMessageTransportOptions options, IAmazonSQS sqs, I ArgumentNullException.ThrowIfNull(sns); _sqs = new Lazy(() => sqs); _sns = new Lazy(() => sns); - _ownsClients = false; + _ownsSqs = false; + _ownsSns = false; + } + + /// Uses a caller-owned SQS client; SNS is created lazily only when needed. + public AwsMessageTransport(AwsMessageTransportOptions options, IAmazonSQS sqs) : this(options) + { + ArgumentNullException.ThrowIfNull(sqs); + _sqs = new Lazy(() => sqs); + _ownsSqs = false; } public AwsMessageTransport(string connectionString) : this(AwsMessageTransportOptions.FromConnectionString(connectionString)) { } @@ -119,7 +129,7 @@ public async Task> ReceiveAsync(DestinationAddress MaxNumberOfMessages = Math.Clamp(request.MaxMessages <= 0 ? 1 : request.MaxMessages, 1, 10), VisibilityTimeout = (int)Math.Clamp(visibility.TotalSeconds, 0, 43200), MessageAttributeNames = ["All"], - MessageSystemAttributeNames = ["ApproximateReceiveCount"] + MessageSystemAttributeNames = ["ApproximateReceiveCount", "SentTimestamp"] }; if (request.MaxWaitTime is { } wait) sqsRequest.WaitTimeSeconds = (int)Math.Clamp(wait.TotalSeconds, 0, 20); @@ -181,6 +191,8 @@ public async Task> ReceiveAsync(DestinationAddress ContentType = contentType, Destination = source, LockExpiresUtc = receiveStarted.AddSeconds(sqsRequest.VisibilityTimeout.GetValueOrDefault()), + EnqueuedUtc = message.Attributes is not null && message.Attributes.TryGetValue("SentTimestamp", out var timestamp) && Int64.TryParse(timestamp, out long milliseconds) + ? DateTimeOffset.FromUnixTimeMilliseconds(milliseconds) : null, Body = body, Headers = headers, EnvelopeError = envelopeError, @@ -247,7 +259,14 @@ public async Task EnsureAsync(IReadOnlyList declarations await EnsureSubscriptionAsync(declaration.Address, ct).ConfigureAwait(false); break; default: - await ResolveQueueUrlAsync(declaration.Address, allowCreate: true, ct).ConfigureAwait(false); + ValidateQueueArguments(declaration.ProviderArguments); + string url = await ResolveQueueUrlAsync(declaration.Address, allowCreate: true, ct).ConfigureAwait(false); + if (declaration.ProviderArguments is { Count: > 0 } arguments) + await _sqs.Value.SetQueueAttributesAsync(new SetQueueAttributesRequest + { + QueueUrl = url, + Attributes = new Dictionary(arguments) + }, ct).ConfigureAwait(false); break; } } @@ -357,7 +376,8 @@ public async Task GetStatsAsync(DestinationAddress dest return new MessageDestinationStats { Queued = response.ApproximateNumberOfMessages, - Working = response.ApproximateNumberOfMessagesNotVisible + Working = response.ApproximateNumberOfMessagesNotVisible, + Delayed = response.ApproximateNumberOfMessagesDelayed }; } @@ -367,9 +387,9 @@ public async ValueTask DisposeAsync() return; await DisposeBatchersAsync().ConfigureAwait(false); - if (_ownsClients && _sqs.IsValueCreated) + if (_ownsSqs && _sqs.IsValueCreated) _sqs.Value.Dispose(); - if (_ownsClients && _sns.IsValueCreated) + if (_ownsSns && _sns.IsValueCreated) _sns.Value.Dispose(); } @@ -591,7 +611,7 @@ private PreparedMessage PrepareMessage(int index, TransportMessage message, Date var headers = message.Headers; string envelope = JsonSerializer.Serialize(new AwsEnvelope(1, encoding, message.MessageId, message.ContentType, headers)); int bytes = checked(Encoding.UTF8.GetByteCount(body) + AttributeBytes(EnvelopeAttributeName, envelope)); - foreach (string name in _nativeMessageHeaders) + foreach (string name in NativeHeaders(headers)) { string? value = headers.GetValueOrDefault(name); if (!String.IsNullOrEmpty(value)) @@ -602,13 +622,20 @@ private PreparedMessage PrepareMessage(int index, TransportMessage message, Date static int AttributeBytes(string name, string value) => checked(Encoding.UTF8.GetByteCount(name) + Encoding.UTF8.GetByteCount(value) + 6); } + private IEnumerable NativeHeaders(MessageHeaders headers) + { + if (!_options.ExposeAllNativeHeaders) return _nativeMessageHeaders; + var names = headers.Keys.Where(AwsMessageTransportOptions.IsValidNativeHeader).Union(_nativeMessageHeaders, StringComparer.Ordinal).ToArray(); + return names.Length <= 9 ? names : _nativeMessageHeaders; + } + private Dictionary BuildAttributes(PreparedMessage message, Func createAttribute) { var attributes = new Dictionary(_nativeMessageHeaders.Length + 1, StringComparer.Ordinal) { [EnvelopeAttributeName] = createAttribute(message.Envelope) }; - foreach (string name in _nativeMessageHeaders) + foreach (string name in NativeHeaders(message.Headers)) { string? value = message.Headers.GetValueOrDefault(name); if (!String.IsNullOrEmpty(value)) diff --git a/src/Foundatio.Aws/AwsMessageTransportOptions.cs b/src/Foundatio.Aws/AwsMessageTransportOptions.cs index 76ef974ab..f383c1fe8 100644 --- a/src/Foundatio.Aws/AwsMessageTransportOptions.cs +++ b/src/Foundatio.Aws/AwsMessageTransportOptions.cs @@ -7,6 +7,9 @@ namespace Foundatio.Messaging; public class AwsMessageTransportOptions { + /// Optional logging for managed node lifecycle operations. + public Microsoft.Extensions.Logging.ILoggerFactory? LoggerFactory { get; set; } + /// AWS credentials. When null, the SDK's default credential chain is used. public AWSCredentials? Credentials { get; set; } @@ -29,6 +32,12 @@ public class AwsMessageTransportOptions /// public IReadOnlyCollection NativeMessageHeaders { get; set; } = []; + /// Expose all valid headers when they fit AWS's attribute limit; otherwise retain only explicitly selected native headers. + public bool ExposeAllNativeHeaders { get; set; } + + /// Maximum entries in a native batch, between one and ten. + public int MaxBatchSize { get; set; } = 10; + /// Default receive visibility timeout when none is supplied. Maps to the SQS visibility window. public TimeSpan DefaultVisibilityTimeout { get; set; } = TimeSpan.FromSeconds(30); @@ -50,6 +59,8 @@ public class AwsMessageTransportOptions internal void Validate() { ArgumentNullException.ThrowIfNull(NativeMessageHeaders); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxBatchSize, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(MaxBatchSize, 10); ArgumentOutOfRangeException.ThrowIfGreaterThan(NativeMessageHeaders.Count, 9, nameof(NativeMessageHeaders)); var names = new HashSet(StringComparer.Ordinal); foreach (string name in NativeMessageHeaders) @@ -66,7 +77,7 @@ internal void Validate() ArgumentOutOfRangeException.ThrowIfGreaterThan(BatchTimeout, TimeSpan.FromMinutes(5)); } - private static bool IsValidNativeHeader(string name) + internal static bool IsValidNativeHeader(string name) { if (String.IsNullOrEmpty(name) || name.Length > 256 || name[0] == '.' || name[^1] == '.' || name.Contains("..", StringComparison.Ordinal) || name.StartsWith("AWS.", StringComparison.OrdinalIgnoreCase) || name.StartsWith("Amazon.", StringComparison.OrdinalIgnoreCase) diff --git a/src/Foundatio.Aws/AwsRequestBatcher.cs b/src/Foundatio.Aws/AwsRequestBatcher.cs index aeb113a20..c88803af2 100644 --- a/src/Foundatio.Aws/AwsRequestBatcher.cs +++ b/src/Foundatio.Aws/AwsRequestBatcher.cs @@ -13,6 +13,7 @@ internal sealed class AwsRequestBatcher : IAsyncDisposable private readonly Func _size; private readonly int _maximumBytes; private readonly int _concurrency; + private readonly int _maxBatchSize; private readonly TimeSpan _delay; private readonly TimeSpan _timeout; private readonly bool _delayWhenIdle; @@ -30,6 +31,7 @@ public AwsRequestBatcher(AwsMessageTransportOptions options, int maximumBytes, F _execute = execute; _delayWhenIdle = delayWhenIdle; _concurrency = options.MaxConcurrentBatches; + _maxBatchSize = options.MaxBatchSize; _delay = options.BatchDelay; _timeout = options.BatchTimeout; _channel = Channel.CreateBounded(new BoundedChannelOptions(options.MaxPendingBatchMessages) @@ -49,7 +51,7 @@ public AwsRequestBatcher(AwsMessageTransportOptions options, int maximumBytes, F public void ObserveBatchSize(int count) { - count = Math.Clamp(count, 1, 10); + count = Math.Clamp(count, 1, _maxBatchSize); int previous = Volatile.Read(ref _observedBatchSize); while (previous < count) { @@ -113,12 +115,12 @@ private async Task RunAsync() private async Task> ReadBatchAsync(bool waitForMore) { - var batch = new List(10); + var batch = new List(_maxBatchSize); int bytes = 0; Task? deadline = null; try { - while (batch.Count < 10) + while (batch.Count < _maxBatchSize) { if (_channel.Reader.TryPeek(out var pending)) { diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs new file mode 100644 index 000000000..ba72bef4d --- /dev/null +++ b/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +// A job hash, its cancellation flag, and its indexes change in one server operation. Index scores +// remain creation times for pagination; a separate index records actual server expiration deadlines. +internal static class RedisMessageExecutionScripts +{ + public const string IncrementCounter = """ + local value = redis.call('HINCRBY', KEYS[1], ARGV[1], ARGV[2]) + redis.call('PEXPIRE', KEYS[1], ARGV[3]) + return value + """; + + public const string Mutate = """ + local op, prefix, id = ARGV[1], ARGV[2], ARGV[3] + local key, ttl, expected, now = KEYS[1], tonumber(ARGV[4]), ARGV[5], ARGV[6] + local clock = redis.call('TIME') + local serverNow = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) + local function queueKey(queue) return prefix .. ':queues:' .. queue end + local function statusKey(queue, status) return queueKey(queue) .. ':status:' .. status end + local function terminal(status) return status == '2' or status == '3' or status == '4' end + local function removeIndexes(queue, job) + redis.call('ZREM', queueKey(queue), job) + redis.call('ZREM', queueKey(queue) .. ':expires', job) + for status = 0, 6 do redis.call('ZREM', statusKey(queue, status), job) end + end + local function cleanup(queue) + local expires = queueKey(queue) .. ':expires' + local members = redis.call('ZRANGEBYSCORE', expires, '-inf', serverNow, 'LIMIT', 0, 128) + for _, job in ipairs(members) do + local remaining = redis.call('PTTL', prefix .. ':' .. job) + if remaining == -2 then + removeIndexes(queue, job) + else + redis.call('ZADD', expires, remaining == -1 and 9007199254740991 or serverNow + math.max(1, remaining), job) + end + end + return #members + end + local function touchIndex(index, score, retention) + local existed = redis.call('EXISTS', index) == 1 + redis.call('ZADD', index, score, id) + if retention < 0 then + redis.call('PERSIST', index) + else + local remaining = redis.call('PTTL', index) + if not existed or (remaining >= 0 and remaining < retention) then + redis.call('PEXPIRE', index, retention) + end + end + end + local function refresh(queue, status) + local retention = ttl + if retention == -2 then retention = tonumber(redis.call('HGET', key, 'RetentionMs') or '-1') end + redis.call('HSET', key, 'RetentionMs', retention) + if retention < 0 then + redis.call('PERSIST', key) + redis.call('PERSIST', key .. ':cancel') + else + redis.call('PEXPIRE', key, retention) + redis.call('PEXPIRE', key .. ':cancel', retention) + end + local created = tonumber(redis.call('HGET', key, 'CreatedUtc') or '0') + touchIndex(queueKey(queue), created, retention) + touchIndex(statusKey(queue, status), created, retention) + touchIndex(queueKey(queue) .. ':expires', retention < 0 and 9007199254740991 or serverNow + retention, retention) + cleanup(queue) + end + if op == 'clean' then return cleanup(id) end + if op == 'prune' then + if redis.call('EXISTS', key) == 0 then removeIndexes(ARGV[7], id) end + return 1 + end + local oldQueue = redis.call('HGET', key, 'QueueName') + local oldStatus = redis.call('HGET', key, 'Status') + if op == 'remove' then + if oldQueue then removeIndexes(oldQueue, id) end + redis.call('DEL', key, key .. ':cancel') + return 1 + end + if op == 'set' then + if oldQueue then removeIndexes(oldQueue, id) end + redis.call('DEL', key, key .. ':cancel') + for i = 7, #ARGV, 2 do redis.call('HSET', key, ARGV[i], ARGV[i + 1]) end + refresh(redis.call('HGET', key, 'QueueName'), redis.call('HGET', key, 'Status')) + return 1 + end + if not oldQueue or terminal(oldStatus) then return 0 end + local attempt = tonumber(redis.call('HGET', key, 'Attempt') or '0') + if expected ~= '' and tonumber(expected) ~= attempt then return 0 end + if op == 'cancel' then + local remaining = redis.call('PTTL', key) + redis.call('SET', key .. ':cancel', '1') + if remaining >= 0 then redis.call('PEXPIRE', key .. ':cancel', math.max(1, remaining)) end + return 1 + end + if (op == 'progress' or op == 'heartbeat') and oldStatus ~= '1' then return 0 end + if op == 'status' then + local status, nextAttempt = oldStatus, nil + for i = 7, #ARGV, 2 do + if ARGV[i] == 'Status' then status = ARGV[i + 1] end + if ARGV[i] == 'Attempt' then nextAttempt = tonumber(ARGV[i + 1]) end + end + if nextAttempt and nextAttempt < attempt then return 0 end + if status == '1' and nextAttempt == attempt and (oldStatus == '1' or oldStatus == '5') then return 0 end + if status == '6' and oldStatus ~= '0' then return 0 end + if status ~= oldStatus then redis.call('ZREM', statusKey(oldQueue, oldStatus), id) end + if not terminal(status) then redis.call('HDEL', key, 'CompletedUtc') end + if status == '1' then redis.call('HDEL', key, 'ErrorMessage') end + end + for i = 7, #ARGV, 2 do redis.call('HSET', key, ARGV[i], ARGV[i + 1]) end + redis.call('HSET', key, 'LastUpdatedUtc', now) + refresh(oldQueue, redis.call('HGET', key, 'Status')) + return 1 + """; +} diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs new file mode 100644 index 000000000..9980cd7e7 --- /dev/null +++ b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs @@ -0,0 +1,338 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Globalization; +using StackExchange.Redis; + +namespace Foundatio.Messaging; + +/// +/// Redis-backed implementation of . +/// Each job is stored as a Redis Hash. Per-queue and per-status sorted sets scored by creation time +/// index jobs for pagination. Cancellation uses a separate key that shares the job's TTL. +/// +/// +/// Key layout ({prefix} is , optionally preceded by +/// ): +/// +/// {prefix}:{jobId} — hash of job fields; metadata is stored as meta:{name} fields +/// {prefix}:{jobId}:cancel — cancellation flag +/// {prefix}:queues:{queueName} — sorted set of every job in the queue +/// {prefix}:queues:{queueName}:status:{status} — sorted set per value +/// {prefix}:counters:{queueName}:{yyyy-MM-ddTHH} — hourly counter hash +/// +/// Writes update hashes, cancellation flags, and indexes atomically using server-side scripts. +/// A separate expiration index uses Redis server deadlines; creation time never implies expiration. +/// For Redis Cluster, configure a common hash tag in KeyPrefix so a store's keys share a slot. +/// +public sealed class RedisMessageExecutionStore : IMessageExecutionStore +{ + /// + public bool IsShared => true; + + private const string MetadataFieldPrefix = "meta:"; + private static readonly TimeSpan MessageExecutionCounterBucketRetention = TimeSpan.FromHours(48); + + private readonly IConnectionMultiplexer _redis; + private readonly RedisMessageExecutionStoreOptions _options; + private readonly TimeProvider _timeProvider; + private readonly string _keyPrefix; + + public RedisMessageExecutionStore(IConnectionMultiplexer redis, RedisMessageExecutionStoreOptions? options = null, TimeProvider? timeProvider = null) + { + _redis = redis; + _options = options ?? new RedisMessageExecutionStoreOptions(); + _timeProvider = timeProvider ?? TimeProvider.System; + _keyPrefix = string.IsNullOrEmpty(_options.ResourcePrefix) + ? _options.KeyPrefix + : $"{_options.ResourcePrefix}:{_options.KeyPrefix}"; + } + + public Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default) + => MutateAsync("set", state.JobId, ResolveTtl(expiry, state.Status), + BuildEntries(state, state.CreatedUtc.ToUnixTimeMilliseconds()), cancellationToken); + + public async Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default) + { + var entries = await _redis.GetDatabase().HashGetAllAsync(JobKey(jobId)).WaitAsync(cancellationToken).ConfigureAwait(false); + return entries.Length == 0 ? null : ParseJobState(entries); + } + + public Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null) + { + var updates = new List { new("Status", FormatStatus(status)) }; + if (startedUtc.HasValue) updates.Add(new("StartedUtc", FormatTimestamp(startedUtc.Value))); + if (IsTerminal(status)) updates.Add(new("CompletedUtc", FormatTimestamp(completedUtc ?? _timeProvider.GetUtcNow()))); + if (errorMessage is not null) updates.Add(new("ErrorMessage", errorMessage)); + if (progress.HasValue) updates.Add(new("Progress", FormatInt(progress.Value))); + if (attempt.HasValue) updates.Add(new("Attempt", FormatInt(attempt.Value))); + if (workerId is not null) updates.Add(new("WorkerId", workerId)); + if (status == MessageExecutionStatus.Processing) + { + updates.Add(new("Progress", FormatInt(progress ?? 0))); + updates.Add(new("ProgressMessage", string.Empty)); + updates.Add(new("LastHeartbeatUtc", FormatTimestamp(startedUtc ?? _timeProvider.GetUtcNow()))); + } + return MutateAsync("status", jobId, ResolveTtl(expiry, status), updates, cancellationToken); + } + + public Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null) + => MutateAsync("progress", jobId, ResolveTtl(expiry, MessageExecutionStatus.Processing), + [new("Progress", FormatInt(progress)), new("ProgressMessage", progressMessage ?? string.Empty)], cancellationToken, expectedAttempt); + + public Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null) + => MutateAsync("heartbeat", jobId, expiry is null ? null : ResolveTtl(expiry, MessageExecutionStatus.Processing), + [new("LastHeartbeatUtc", FormatTimestamp(_timeProvider.GetUtcNow()))], cancellationToken, expectedAttempt, preserveExpiry: expiry is null); + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + => MutateAsync("cancel", jobId, null, [], cancellationToken); + + public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) + { + // The flag has the same expiration as its job and is created atomically with the status check. + return _redis.GetDatabase().KeyExistsAsync(CancelKey(jobId)).WaitAsync(cancellationToken); + } + + public Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default) + => MutateAsync("remove", jobId, null, [], cancellationToken); + + private async Task MutateAsync(string operation, string jobId, TimeSpan? expiry, IReadOnlyList fields, + CancellationToken cancellationToken, int? expectedAttempt = null, bool preserveExpiry = false) + => await EvaluateAsync(operation, jobId, expiry, fields, cancellationToken, expectedAttempt, preserveExpiry).ConfigureAwait(false) != 0; + + private async Task EvaluateAsync(string operation, string jobId, TimeSpan? expiry, IReadOnlyList fields, + CancellationToken cancellationToken, int? expectedAttempt = null, bool preserveExpiry = false) + { + cancellationToken.ThrowIfCancellationRequested(); + RedisValue[] args = new RedisValue[6 + fields.Count * 2]; + args[0] = operation; + args[1] = _keyPrefix; + args[2] = jobId; + args[3] = preserveExpiry ? -2L : expiry is { } ttl ? Math.Max(1L, (long)ttl.TotalMilliseconds) : -1L; + args[4] = expectedAttempt.HasValue ? FormatInt(expectedAttempt.Value) : string.Empty; + args[5] = FormatTimestamp(_timeProvider.GetUtcNow()); + for (int i = 0; i < fields.Count; i++) + { + args[6 + i * 2] = fields[i].Name; + args[7 + i * 2] = fields[i].Value; + } + return (long)await _redis.GetDatabase().ScriptEvaluateAsync(RedisMessageExecutionScripts.Mutate, [JobKey(jobId)], args) + .WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task CleanupAsync(string queueName, CancellationToken cancellationToken) + { + while (await EvaluateAsync("clean", queueName, null, [], cancellationToken).ConfigureAwait(false) == 128) + cancellationToken.ThrowIfCancellationRequested(); + } + + public Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default) + { + var db = _redis.GetDatabase(); + var bucketKey = MessageExecutionCounterBucketKey(queueName, _timeProvider.GetUtcNow()); + + // Increment and retention refresh form one atomic server operation, without MULTI/EXEC overhead. + return db.ScriptEvaluateAsync(RedisMessageExecutionScripts.IncrementCounter, [bucketKey], + [counterName, value, (long)MessageExecutionCounterBucketRetention.TotalMilliseconds]).WaitAsync(cancellationToken); + } + + public async Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) + { + var db = _redis.GetDatabase(); + var now = _timeProvider.GetUtcNow(); + var effectiveWindow = window ?? TimeSpan.FromHours(24); + var startHour = TruncateToHour(now - effectiveWindow); + var endHour = TruncateToHour(now); + + var hours = new List(); + for (var hour = startHour; hour <= endHour; hour = hour.AddHours(1)) + hours.Add(hour); + + var batch = db.CreateBatch(); + var tasks = new Task[hours.Count]; + for (int i = 0; i < hours.Count; i++) + tasks[i] = batch.HashGetAllAsync(MessageExecutionCounterBucketKey(queueName, hours[i])); + batch.Execute(); + + var totals = new Dictionary(); + var buckets = new List(hours.Count); + + for (int i = 0; i < hours.Count; i++) + { + var entries = await tasks[i].WaitAsync(cancellationToken).ConfigureAwait(false); + var counters = new Dictionary(entries.Length); + + foreach (var entry in entries) + { + if (entry.Value.TryParse(out long val)) + { + var name = entry.Name.ToString(); + counters[name] = val; + totals[name] = totals.GetValueOrDefault(name) + val; + } + } + + buckets.Add(new MessageExecutionCounterBucket { Hour = hours[i], Counters = counters }); + } + + return new MessageExecutionCounters { Totals = totals, Buckets = buckets }; + } + + public async Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default) + { + if (take <= 0) + return []; + + var db = _redis.GetDatabase(); + await CleanupAsync(queueName, cancellationToken).ConfigureAwait(false); + var setKey = StatusSetKey(queueName, status); + var results = new List(take); + var dangling = new List(); + long cursor = Math.Max(skip, 0); + + // Keep paging past members whose hash has expired so the caller still gets a full page. + while (results.Count < take) + { + int wanted = take - results.Count; + var members = await db.SortedSetRangeByRankAsync(setKey, cursor, cursor + wanted - 1, Order.Descending).WaitAsync(cancellationToken).ConfigureAwait(false); + if (members.Length == 0) + break; + + cursor += members.Length; + + var batch = db.CreateBatch(); + var tasks = new Task[members.Length]; + for (int i = 0; i < members.Length; i++) + tasks[i] = batch.HashGetAllAsync(JobKey(members[i].ToString())); + batch.Execute(); + + for (int i = 0; i < tasks.Length; i++) + { + var entries = await tasks[i].WaitAsync(cancellationToken).ConfigureAwait(false); + if (entries.Length > 0) + { + var state = ParseJobState(entries); + if (state.Status == status && state.QueueName == queueName) + results.Add(state); + } + else + dangling.Add(members[i]); + } + + if (members.Length < wanted) + break; + } + + // Removal is deferred until after paging so ranks stay stable while reading. + foreach (var id in dangling) + await MutateAsync("prune", id.ToString(), null, [new(queueName, string.Empty)], cancellationToken).ConfigureAwait(false); + + return results; + } + + /// Counts current status-index entries after removing expired jobs using server deadlines. + public async Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default) + { + await CleanupAsync(queueName, cancellationToken).ConfigureAwait(false); + return await _redis.GetDatabase().SortedSetLengthAsync(StatusSetKey(queueName, status)).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private TimeSpan? ResolveTtl(TimeSpan? expiry, MessageExecutionStatus status) + { + var ttl = expiry ?? _options.DefaultExpiry; + if (ttl is null || IsTerminal(status)) + return ttl; + + return ttl.Value > _options.NonTerminalExpiry ? ttl : _options.NonTerminalExpiry; + } + + private static bool IsTerminal(MessageExecutionStatus status) + => status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled; + + private static HashEntry[] BuildEntries(MessageExecutionState state, long createdScore) + { + var entries = new List(12 + (state.Metadata?.Count ?? 0)) + { + new("JobId", state.JobId), + new("QueueName", state.QueueName), + new("MessageType", state.MessageType), + new("Status", FormatStatus(state.Status)), + new("Progress", FormatInt(state.Progress)), + new("ProgressMessage", state.ProgressMessage ?? string.Empty), + new("CreatedUtc", createdScore.ToString(CultureInfo.InvariantCulture)), + new("StartedUtc", state.StartedUtc is { } started ? FormatTimestamp(started) : string.Empty), + new("CompletedUtc", state.CompletedUtc is { } completed ? FormatTimestamp(completed) : string.Empty), + new("ErrorMessage", state.ErrorMessage ?? string.Empty), + new("Attempt", FormatInt(state.Attempt)), + new("WorkerId", state.WorkerId ?? string.Empty), + new("LastUpdatedUtc", FormatTimestamp(state.LastUpdatedUtc)), + new("LastHeartbeatUtc", state.LastHeartbeatUtc is { } heartbeat ? FormatTimestamp(heartbeat) : string.Empty) + }; + + if (state.Metadata is not null) + { + foreach (var kvp in state.Metadata) + entries.Add(new(MetadataFieldPrefix + kvp.Key, kvp.Value)); + } + + return entries.ToArray(); + } + + private static string FormatStatus(MessageExecutionStatus status) => ((int)status).ToString(CultureInfo.InvariantCulture); + private static string FormatInt(int value) => value.ToString(CultureInfo.InvariantCulture); + private static string FormatTimestamp(DateTimeOffset value) => value.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); + + private string JobKey(string jobId) => $"{_keyPrefix}:{jobId}"; + private string CancelKey(string jobId) => $"{_keyPrefix}:{jobId}:cancel"; + private string QueueSetKey(string queueName) => $"{_keyPrefix}:queues:{queueName}"; + private string StatusSetKey(string queueName, MessageExecutionStatus status) => $"{_keyPrefix}:queues:{queueName}:status:{(int)status}"; + private string MessageExecutionCounterBucketKey(string queueName, DateTimeOffset timestamp) => $"{_keyPrefix}:counters:{queueName}:{TruncateToHour(timestamp):yyyy-MM-ddTHH}"; + + private static DateTimeOffset TruncateToHour(DateTimeOffset timestamp) + => new(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0, TimeSpan.Zero); + + private static MessageExecutionState ParseJobState(HashEntry[] entries) + { + var dict = new Dictionary(entries.Length); + Dictionary? metadata = null; + + foreach (var entry in entries) + { + var name = entry.Name.ToString(); + if (name.StartsWith(MetadataFieldPrefix, StringComparison.Ordinal)) + (metadata ??= new Dictionary())[name[MetadataFieldPrefix.Length..]] = entry.Value.ToString(); + else + dict[name] = entry.Value.ToString(); + } + + return new MessageExecutionState + { + JobId = dict.GetValueOrDefault("JobId") ?? string.Empty, + QueueName = dict.GetValueOrDefault("QueueName") ?? string.Empty, + MessageType = dict.GetValueOrDefault("MessageType") ?? string.Empty, + Status = int.TryParse(dict.GetValueOrDefault("Status"), out var s) ? (MessageExecutionStatus)s : MessageExecutionStatus.Queued, + Progress = int.TryParse(dict.GetValueOrDefault("Progress"), out var p) ? p : 0, + ProgressMessage = NullIfEmpty(dict.GetValueOrDefault("ProgressMessage")), + CreatedUtc = ParseDateTimeOffset(dict.GetValueOrDefault("CreatedUtc")), + StartedUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("StartedUtc")), + CompletedUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("CompletedUtc")), + ErrorMessage = NullIfEmpty(dict.GetValueOrDefault("ErrorMessage")), + Attempt = int.TryParse(dict.GetValueOrDefault("Attempt"), out var a) ? a : 0, + WorkerId = string.IsNullOrEmpty(dict.GetValueOrDefault("WorkerId")) ? null : dict["WorkerId"], + LastUpdatedUtc = ParseDateTimeOffset(dict.GetValueOrDefault("LastUpdatedUtc")), + LastHeartbeatUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("LastHeartbeatUtc")), + Metadata = metadata + }; + } + + private static DateTimeOffset ParseDateTimeOffset(string? value) + => long.TryParse(value, out var ms) ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : DateTimeOffset.MinValue; + + private static DateTimeOffset? ParseNullableDateTimeOffset(string? value) + => string.IsNullOrEmpty(value) ? null : long.TryParse(value, out var ms) ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null; + + private static string? NullIfEmpty(string? value) + => string.IsNullOrEmpty(value) ? null : value; +} diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs new file mode 100644 index 000000000..34a31838a --- /dev/null +++ b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +/// +/// Options for configuring . +/// +public class RedisMessageExecutionStoreOptions +{ + /// + /// Key prefix for all Redis keys. Default is "fnd:executions". + /// + public string KeyPrefix { get; set; } = "fnd:executions"; + + /// + /// Optional prefix applied before for app-level scoping. + /// When set, all Redis keys become "{ResourcePrefix}:{KeyPrefix}:...". + /// When null or empty (default), only is used. + /// + /// + /// Use this to isolate multiple applications sharing the same Redis instance + /// (e.g., "myapp" produces keys like "myapp:fnd:executions:..."). + /// + public string? ResourcePrefix { get; set; } + + /// + /// TTL applied to a job's keys by any write whose caller passes no expiry. + /// Default is 24 hours. Set to null to disable auto-expiry for such writes. + /// + /// + /// The queue worker passes MessageExecutionOptions.StateRetention on every write, so this only + /// takes effect for direct callers of the store. Writes that leave a job + /// or are raised to at least . + /// + public TimeSpan? DefaultExpiry { get; set; } = TimeSpan.FromHours(24); + + /// + /// Minimum TTL for a job while it is or , + /// so a live job does not vanish before it reaches a terminal state. Default is 7 days. A longer caller-supplied + /// expiry still wins; a null effective expiry (no TTL) is left as is. + /// + public TimeSpan NonTerminalExpiry { get; set; } = TimeSpan.FromDays(7); +} diff --git a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs index 7a6c53dbc..a250768bc 100644 --- a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs +++ b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using Foundatio.Jobs; +using Foundatio.Lock; using Foundatio.Messaging; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -63,6 +64,29 @@ public static FoundatioBuilder.MessagingBuilder UseRedis(this FoundatioBuilder.M return builder.UseTransport(sp => new RedisStreamsMessageTransport(sp.GetRequiredService())); } + /// Shares progress, cancellation and history for broker-delivered executions through Redis. + public static FoundatioBuilder.MessagingBuilder UseRedisExecutionTracking(this FoundatioBuilder.MessagingBuilder builder, + Action? configure = null, string? connectionString = null) + { + var services = ((IFoundatioBuilder)builder).Services; + EnsureConnection(services, connectionString); + services.AddSingleton(sp => + { + var options = new RedisMessageExecutionStoreOptions(); + configure?.Invoke(options); + return new RedisMessageExecutionStore(sp.GetRequiredService(), options, sp.GetService()); + }); + return builder; + } + + /// Coordinates resources across workers with ownership-checked Redis locks. + public static FoundatioBuilder UseRedis(this FoundatioBuilder.LockingBuilder builder, string keyPrefix = "fnd:locks:", string? connectionString = null) + { + var services = ((IFoundatioBuilder)builder).Services; + EnsureConnection(services, connectionString); + return builder.Use(sp => new RedisLockProvider(sp.GetRequiredService(), keyPrefix)); + } + private static void EnsureConnection(IServiceCollection services, string? connectionString) { var settings = services.FirstOrDefault(d => d.ServiceType == typeof(RedisConnectionSettings))?.ImplementationInstance as RedisConnectionSettings; diff --git a/src/Foundatio.Redis/RedisLockProvider.cs b/src/Foundatio.Redis/RedisLockProvider.cs new file mode 100644 index 000000000..f06885f53 --- /dev/null +++ b/src/Foundatio.Redis/RedisLockProvider.cs @@ -0,0 +1,72 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace Foundatio.Lock; + +/// Redis resource locks with ownership-checked renewal and release. +public sealed class RedisLockProvider(IConnectionMultiplexer connection, string keyPrefix = "fnd:locks:") : ILockProvider +{ + private const string ReleaseScript = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end"; + private const string RenewScript = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end"; + + public async Task AcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) + => await TryAcquireAsync(resource, timeUntilExpires, releaseOnDispose, cancellationToken).ConfigureAwait(false) + ?? throw new LockAcquisitionTimeoutException(resource); + + public async Task TryAcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(resource); + var lifetime = timeUntilExpires ?? TimeSpan.FromMinutes(20); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(lifetime, TimeSpan.Zero); + string id = Guid.NewGuid().ToString("N"); + long started = Stopwatch.GetTimestamp(); + var database = connection.GetDatabase(); + do + { + // Observe every acquisition outcome even if cancellation arrives during Redis I/O, so a late successful acquisition can still be released by the caller. + if (await database.StringSetAsync(keyPrefix + resource, id, lifetime, When.NotExists).ConfigureAwait(false)) + return new Handle(this, resource, id, Stopwatch.GetElapsedTime(started), releaseOnDispose); + if (cancellationToken.IsCancellationRequested) return null; + try { await Task.Delay(TimeSpan.FromMilliseconds(50), cancellationToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return null; } + } while (true); + } + + public Task IsLockedAsync(string resource) => connection.GetDatabase().KeyExistsAsync(keyPrefix + resource); + public async Task ReleaseAsync(string resource, string lockId) + => _ = await connection.GetDatabase().ScriptEvaluateAsync(ReleaseScript, [keyPrefix + resource], [lockId]).ConfigureAwait(false); + public async Task ReleaseAsync(string resource) + => _ = await connection.GetDatabase().KeyDeleteAsync(keyPrefix + resource).ConfigureAwait(false); + public async Task RenewAsync(string resource, string lockId, TimeSpan? timeUntilExpires = null) + { + var lifetime = timeUntilExpires ?? TimeSpan.FromMinutes(20); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(lifetime, TimeSpan.Zero); + if ((int)await connection.GetDatabase().ScriptEvaluateAsync(RenewScript, [keyPrefix + resource], [lockId, (long)Math.Ceiling(lifetime.TotalMilliseconds)]).ConfigureAwait(false) == 0) + throw new LockOwnershipLostException($"The lock on '{resource}' is no longer owned by this holder."); + } + + private sealed class Handle(RedisLockProvider owner, string resource, string id, TimeSpan waited, bool releaseOnDispose) : ILock + { + private int _released; + private int _renewals; + public string LockId => id; + public string Resource => resource; + public DateTime AcquiredTimeUtc { get; } = DateTime.UtcNow; + public TimeSpan TimeWaitedForLock => waited; + public int RenewalCount => Volatile.Read(ref _renewals); + public async Task RenewAsync(TimeSpan? timeUntilExpires = null) + { + if (Volatile.Read(ref _released) != 0) throw new LockOwnershipLostException("This lock was already released."); + await owner.RenewAsync(resource, id, timeUntilExpires).ConfigureAwait(false); + Interlocked.Increment(ref _renewals); + } + public async Task ReleaseAsync() + { + if (Interlocked.Exchange(ref _released, 1) == 0) await owner.ReleaseAsync(resource, id).ConfigureAwait(false); + } + public ValueTask DisposeAsync() => releaseOnDispose ? new ValueTask(ReleaseAsync()) : default; + } +} diff --git a/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs new file mode 100644 index 000000000..9e61a546e --- /dev/null +++ b/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs @@ -0,0 +1,90 @@ +using System; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +/// Atomic transition guarantees for optional broker-driven execution tracking. +public abstract class MessageExecutionStoreConformanceTests +{ + protected abstract IMessageExecutionStore? CreateStore(); + private IMessageExecutionStore Store() { var store = CreateStore(); Assert.SkipWhen(store is null, "Execution store is not configured."); return store!; } + private static MessageExecutionState Queued(string id) => new() + { + JobId = id, + QueueName = "exports", + MessageType = "Export", + Status = MessageExecutionStatus.Queued, + CreatedUtc = DateTimeOffset.UtcNow, + LastUpdatedUtc = DateTimeOffset.UtcNow + }; + + [Fact] + public async Task ADeliveryAttempt_CanStartOnlyOnce() + { + var store = Store(); + string id = Guid.NewGuid().ToString("N"); + await store.SetJobStateAsync(Queued(id)); + Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1, workerId: "first")); + Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1, workerId: "second")); + Assert.Equal("first", (await store.GetJobStateAsync(id))!.WorkerId); + } + + [Fact] + public async Task OlderAttempt_CannotOverwriteANewerAttempt() + { + var store = Store(); + string id = Guid.NewGuid().ToString("N"); + await store.SetJobStateAsync(Queued(id)); + Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1)); + Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.RetryPending, attempt: 1)); + Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 2)); + await store.UpdateJobProgressAsync(id, 40, "current", expectedAttempt: 2); + await store.UpdateJobProgressAsync(id, 99, "stale", expectedAttempt: 1); + Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Completed, attempt: 1)); + var state = await store.GetJobStateAsync(id); + Assert.Equal(MessageExecutionStatus.Processing, state!.Status); + Assert.Equal(40, state.Progress); + Assert.Equal("current", state.ProgressMessage); + } + + [Fact] + public async Task RetryPending_IgnoresProgressFromTheFinishedAttempt() + { + var store = Store(); + string id = Guid.NewGuid().ToString("N"); + await store.SetJobStateAsync(Queued(id)); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.RetryPending, attempt: 1); + await store.UpdateJobProgressAsync(id, 80, "too late", expectedAttempt: 1); + Assert.Equal(0, (await store.GetJobStateAsync(id))!.Progress); + } + + [Fact] + public async Task Cancellation_SurvivesProgressAndStatusWrites() + { + var store = Store(); + string id = Guid.NewGuid().ToString("N"); + await store.SetJobStateAsync(Queued(id)); + Assert.True(await store.RequestCancellationAsync(id)); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); + await store.UpdateJobProgressAsync(id, 50, expectedAttempt: 1); + Assert.True(await store.IsCancellationRequestedAsync(id)); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Cancelled, attempt: 1); + Assert.False(await store.RequestCancellationAsync(id)); + } + + [Fact] + public async Task TerminalState_RejectsDelayedSendReconciliation() + { + var store = Store(); + string id = Guid.NewGuid().ToString("N"); + await store.SetJobStateAsync(Queued(id)); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); + await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Completed, attempt: 1, progress: 100); + Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.EnqueueUnknown, errorMessage: "Late send timeout")); + Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 2)); + Assert.Equal(MessageExecutionStatus.Completed, (await store.GetJobStateAsync(id))!.Status); + } +} diff --git a/src/Foundatio.Testing/RecordingMessageTransport.cs b/src/Foundatio.Testing/RecordingMessageTransport.cs index ce393ab64..0ebae742b 100644 --- a/src/Foundatio.Testing/RecordingMessageTransport.cs +++ b/src/Foundatio.Testing/RecordingMessageTransport.cs @@ -14,13 +14,17 @@ namespace Foundatio.Messaging.Testing; /// internal sealed class RecordingMessageTransport : IMessageTransport, ISupportsPull, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, - ISupportsEphemeralSubscriptions, ITransportInfo + ISupportsEphemeralSubscriptions, ITransportInfo, IMessageProcessingObserver { // A delayed redelivery lives only in the inner transport's timer until it fires — neither queued nor in flight — // so idle detection would report quiescent while a retry is pending. Give the timer this long past its due time to // materialize the redelivered message back into stats before the pending marker is dropped. private static readonly TimeSpan _redeliveryGrace = TimeSpan.FromMilliseconds(250); + private readonly ConcurrentDictionary _processing = new(); + public void ProcessingStarted(TransportEntry entry) => _processing[entry.Receipt] = entry.Destination; + public void ProcessingFinished(TransportEntry entry) => _processing.TryRemove(entry.Receipt, out _); + private readonly InMemoryMessageTransport _inner; private readonly TimeProvider _timeProvider; private readonly ConcurrentQueue _sent = new(); @@ -201,8 +205,9 @@ private static bool Consumes(DestinationAddress consumer, DestinationAddress sen { var stats = await _inner.GetStatsAsync(address, ct).ConfigureAwait(false); long queued = stats.Queued + scheduled.GetValueOrDefault(address); - if (queued > 0 || stats.Working > 0) - pending.Add((address.Key, queued, stats.Working)); + long working = Math.Max(stats.Working, _processing.Values.LongCount(source => source == address)); + if (queued > 0 || working > 0) + pending.Add((address.Key, queued, working)); } return pending; diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 00b68b82c..a4fb4c75a 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -362,6 +362,13 @@ public MessagingBuilder UseSchedulingStore(FuncTracks broker-delivered executions in this process. The broker remains the only source of runnable work. + public MessagingBuilder UseInMemoryExecutionTracking() + { + _services.ReplaceSingleton(sp => new InMemoryMessageExecutionStore(sp.GetService())); + return this; + } + public MessagingBuilder UseTransport(IMessageTransport transport) { ArgumentNullException.ThrowIfNull(transport); diff --git a/src/Foundatio/Lock/LockOwnershipLostException.cs b/src/Foundatio/Lock/LockOwnershipLostException.cs new file mode 100644 index 000000000..0c34e1f79 --- /dev/null +++ b/src/Foundatio/Lock/LockOwnershipLostException.cs @@ -0,0 +1,6 @@ +using System; + +namespace Foundatio.Lock; + +/// The resource lock is no longer owned by this lease holder. +public sealed class LockOwnershipLostException(string message) : InvalidOperationException(message); diff --git a/src/Foundatio/Messaging/IMessageContext.cs b/src/Foundatio/Messaging/IMessageContext.cs index fe40e7def..8b172363f 100644 --- a/src/Foundatio/Messaging/IMessageContext.cs +++ b/src/Foundatio/Messaging/IMessageContext.cs @@ -116,6 +116,12 @@ public sealed record RejectOptions public interface IMessageContext { + /// The receiving destination, when supplied by the bus. + DestinationAddress? Destination => null; + /// Original broker enqueue time, when available. + DateTimeOffset? EnqueuedUtc => null; + /// Whether this delivery lost ownership. A stale worker must not settle it. + bool IsLeaseLost => false; string Id { get; } /// Broker-assigned ID for diagnostics; may change when a message is re-sent. diff --git a/src/Foundatio/Messaging/IMessageProcessingObserver.cs b/src/Foundatio/Messaging/IMessageProcessingObserver.cs new file mode 100644 index 000000000..8f428265f --- /dev/null +++ b/src/Foundatio/Messaging/IMessageProcessingObserver.cs @@ -0,0 +1,11 @@ +namespace Foundatio.Messaging; + +/// Optional transport decoration for deterministic drains through processing, settlement, and observer updates. +public interface IMessageProcessingObserver +{ + /// Called when hosted processing begins. Implementations must not throw. + void ProcessingStarted(TransportEntry entry) { } + + /// Called once when core releases a delivered entry. Implementations must not throw. + void ProcessingFinished(TransportEntry entry); +} diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index d32a77af6..86de14452 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -38,7 +38,7 @@ public sealed partial class InMemoryMessageTransport : IMessageTransport, ISuppo private readonly ConcurrentDictionary _destinations = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _roles = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary> _topicSubscriptions = new(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary _redeliveryTimers = new(); + private readonly ConcurrentDictionary _redeliveryTimers = new(); private readonly ConcurrentDictionary _warnedDroppedTopics = new(StringComparer.OrdinalIgnoreCase); private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -339,6 +339,7 @@ public Task GetStatsAsync(DestinationAddress destinatio { Queued = state.QueuedCount, Working = state.InFlight.Count, + Delayed = _redeliveryTimers.Values.LongCount(key => key == ReceivableKey(destination)), Deadletter = state.DeadletterCount, Enqueued = Volatile.Read(ref state.Enqueued), Dequeued = Volatile.Read(ref state.Dequeued), @@ -502,9 +503,10 @@ private void ScheduleRedelivery(string destination, StoredMessage message, TimeS } catch (ObjectDisposedException) { } catch (InvalidOperationException) { } // destination was deleted / completed between scheduling and firing - }, null, delay, Timeout.InfiniteTimeSpan); + }, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); - _redeliveryTimers[timer] = 0; + _redeliveryTimers[timer] = destination; + timer.Change(delay, Timeout.InfiniteTimeSpan); // A redelivery scheduled right as the transport disposes could otherwise leak its timer; clean up the race. if (Volatile.Read(ref _isDisposed) == 1 && _redeliveryTimers.TryRemove(timer, out _)) diff --git a/src/Foundatio/Messaging/MessageAdministration.cs b/src/Foundatio/Messaging/MessageAdministration.cs new file mode 100644 index 000000000..92b35afb7 --- /dev/null +++ b/src/Foundatio/Messaging/MessageAdministration.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Messaging; + +/// Provider-neutral queue statistics and bounded dead-letter inspection and recovery. +public sealed class MessageAdministration(IMessageTransport transport, TimeProvider? timeProvider = null, ILogger? logger = null) +{ + private readonly ILogger _logger = logger ?? NullLogger.Instance; + private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + private static DestinationAddress DeadLetters(DestinationAddress source) => DestinationAddress.ForQueue(source.Name + ".deadletter"); + + public async Task GetStatsAsync(DestinationAddress source, CancellationToken cancellationToken = default) + { + if (transport is not ISupportsStats stats) throw new NotSupportedException("This transport does not report queue statistics."); + var value = await stats.GetStatsAsync(source, cancellationToken).ConfigureAwait(false); + if (transport is ISupportsDeadLetterSink) return value; + if (transport is ISupportsProvisioning provisioning && !await provisioning.ExistsAsync(DeadLetters(source), cancellationToken).ConfigureAwait(false)) return value; + var dead = await stats.GetStatsAsync(DeadLetters(source), cancellationToken).ConfigureAwait(false); + return value with { Deadletter = dead.Queued + dead.Working }; + } + + /// Returns a bounded snapshot. Inspection leaves the messages available for subsequent recovery. + public async Task> PeekDeadLettersAsync(DestinationAddress source, int limit = 20, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(limit, 1000); + if (transport is ISupportsDeadLetter native) + return await native.PeekDeadLetteredAsync(source, new DeadLetterQuery { Limit = limit }, cancellationToken).ConfigureAwait(false); + var found = new List(limit); + await VisitAsync(source, limit, (entry, _) => { found.Add(entry); return Task.FromResult(false); }, cancellationToken).ConfigureAwait(false); + return found; + } + + public async Task DeleteDeadLetterAsync(DestinationAddress source, string id, CancellationToken cancellationToken = default) + { + if (transport is ISupportsDeadLetter native) return await native.DeleteDeadLetteredAsync(source, id, cancellationToken).ConfigureAwait(false); + bool deleted = false; + await VisitAsync(source, 1000, async (entry, token) => + { + if (entry.Id != id) return false; + await transport.CompleteAsync(entry, token).ConfigureAwait(false); + deleted = true; + return true; + }, cancellationToken).ConfigureAwait(false); + return deleted; + } + + /// Replays one message with optional new execution metadata. A confirmed send precedes deleting its dead letter. + public async Task ReplayDeadLetterAsync(DestinationAddress source, string id, + Func>? prepare = null, CancellationToken cancellationToken = default) + { + if (transport is ISupportsDeadLetter native) + { + if (prepare is null) return await native.ReplayDeadLetteredAsync(source, id, source, cancellationToken).ConfigureAwait(false); + string? after = null; + for (int inspected = 0; inspected < 1000; inspected += 100) + { + var page = await native.PeekDeadLetteredAsync(source, new DeadLetterQuery { Limit = 100, AfterId = after }, cancellationToken).ConfigureAwait(false); + if (page.FirstOrDefault(entry => entry.Id == id) is { } selected) + { + await ReplayAsync(selected, cancellationToken).ConfigureAwait(false); + if (!await native.DeleteDeadLetteredAsync(source, id, cancellationToken).ConfigureAwait(false)) + throw new MessageBusException("Replay was accepted, but the dead letter was already removed by another operation."); + return true; + } + if (page.Count < 100) return false; + after = page[^1].Id; + } + return false; + } + bool replayed = false; + await VisitAsync(source, 1000, async (entry, token) => + { + if (entry.Id != id) return false; + await ReplayAsync(entry, token).ConfigureAwait(false); + await transport.CompleteAsync(entry, token).ConfigureAwait(false); + replayed = true; + return true; + }, cancellationToken).ConfigureAwait(false); + return replayed; + + async Task ReplayAsync(TransportEntry entry, CancellationToken token) + { + var message = prepare is null + ? new TransportMessage { Body = entry.Body, ContentType = entry.ContentType, Headers = MessageHeaders.Create(entry.Headers.Where(pair => pair.Key != KnownHeaders.Attempts && !pair.Key.StartsWith("message.dead_letter.", StringComparison.Ordinal)).ToDictionary()), MessageId = entry.ApplicationMessageId } + : await prepare(entry, token).ConfigureAwait(false); + var result = await transport.SendAsync(source, [message], new TransportSendOptions(), token).ConfigureAwait(false); + if (result.Items.Count != 1 || result.Items[0].Status != MessageSendStatus.Accepted) + throw new MessageBusException("Replay was not confirmed. The original dead letter was retained; an unknown send may still have been accepted."); + } + } + + // Providers without native peeking use bounded receive/hold/release. Hold non-matches for the whole scan so + // a broker cannot repeatedly return the same first ten entries and hide a selected message further back. + private async Task VisitAsync(DestinationAddress source, int limit, Func> visit, CancellationToken cancellationToken) + { + if (transport is not ISupportsPull pull) throw new NotSupportedException("This transport cannot inspect dead letters."); + var destination = DeadLetters(source); + if (transport is ISupportsProvisioning provisioning && !await provisioning.ExistsAsync(destination, cancellationToken).ConfigureAwait(false)) return; + var held = new List(); + var seen = new HashSet(StringComparer.Ordinal); + long started = Stopwatch.GetTimestamp(); + try + { + while (seen.Count < limit && Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(10)) + { + var request = new ReceiveRequest { MaxMessages = Math.Min(10, limit - seen.Count), MaxWaitTime = TimeSpan.FromSeconds(1) }; + var batch = transport is ISupportsVisibilityTimeout visibility + ? await visibility.ReceiveAsync(destination, request, TimeSpan.FromSeconds(30), cancellationToken).ConfigureAwait(false) + : await pull.ReceiveAsync(destination, request, cancellationToken).ConfigureAwait(false); + if (batch.Count == 0) return; + held.AddRange(batch); + foreach (var entry in batch) + { + if (!seen.Add(entry.Id)) return; + if (await visit(entry, cancellationToken).ConfigureAwait(false)) { held.Remove(entry); return; } + } + } + } + finally + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5), _time); + await Task.WhenAll(held.Select(async entry => + { + try { await transport.AbandonAsync(entry, cleanup.Token).WaitAsync(cleanup.Token).ConfigureAwait(false); } + catch (ReceiptExpiredException) { } + catch (OperationCanceledException) when (cleanup.IsCancellationRequested) { } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to return inspected dead letter {MessageId}; its visibility lease will expire", entry.Id); } + })).ConfigureAwait(false); + } + } +} diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index 25501c887..07b47a767 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -47,6 +47,24 @@ public abstract class MessageHandlerOptions /// Maximum in-flight messages across this endpoint's handlers on this process. Default 1. public int MaxConcurrency { get; set; } = 1; + /// Maximum entries requested in one receive, additionally bounded by free concurrency and provider limits. + public int? PrefetchCount { get; set; } + + /// Optional bounded delay for collecting capacity after a partially filled batch. Null uses the provider policy. + public TimeSpan? ReceiveBatchDelay { get; set; } + + /// Initial and automatically renewed delivery lease. Default one minute. + public TimeSpan VisibilityTimeout { get; set; } = TimeSpan.FromMinutes(1); + + /// Renew active delivery leases automatically. Lease loss still cancels processing when disabled. + public bool AutoRenewLock { get; set; } = true; + + /// Time admitted handlers may finish after receiving stops. Zero cancels immediately. + public TimeSpan ShutdownTimeout { get; set; } + + /// Keep manual deliveries active until explicit settlement. False releases processing on handler return. + public bool WaitForManualSettlement { get; set; } = true; + /// Maximum delivery attempts. Null uses the bus retry policy. public int? MaxAttempts { get; set; } @@ -62,8 +80,13 @@ public abstract class MessageHandlerOptions internal void Validate() { ArgumentOutOfRangeException.ThrowIfLessThan(MaxConcurrency, 1); + if (PrefetchCount is { } prefetch) + ArgumentOutOfRangeException.ThrowIfLessThan(prefetch, 1); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(VisibilityTimeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThan(ShutdownTimeout, TimeSpan.Zero); + if (ReceiveBatchDelay is { } batchDelay) ArgumentOutOfRangeException.ThrowIfLessThan(batchDelay, TimeSpan.Zero); if (MaxAttempts is { } attempts) - ArgumentOutOfRangeException.ThrowIfLessThan(attempts, 1); + ArgumentOutOfRangeException.ThrowIfEqual(attempts, 0); if (!Enum.IsDefined(AckMode)) throw new ArgumentOutOfRangeException(nameof(AckMode)); if (this is MessageConsumerOptions { Destination: { } destination }) @@ -133,6 +156,13 @@ internal interface IMessageBatchItem public interface IMessageBus : IAsyncDisposable { + /// Consumes work with an explicit outcome; core applies the endpoint retry and settlement policy. + Task ConsumeWithOutcomeAsync(Func, CancellationToken, ValueTask> handler, MessageConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class + => throw new NotSupportedException("This message bus does not support outcome handlers."); + + /// Consumes raw deliveries with an explicit outcome and core-owned retry and settlement. + Task ConsumeWithOutcomeAsync(Func> handler, MessageConsumerOptions options, CancellationToken cancellationToken = default) + => throw new NotSupportedException("This message bus does not support outcome handlers."); /// Whether per-instance, automatically expiring event subscriptions are available. bool SupportsTemporarySubscriptions => false; /// Receives queued work directly. Dispose an unsettled delivery to return it for redelivery. @@ -141,6 +171,10 @@ public interface IMessageBus : IAsyncDisposable /// Receives a raw message from an explicit queue without deserializing its body. Task ReceiveAsync(MessageReceiveOptions options, CancellationToken cancellationToken = default); + /// Receives one best-effort copy on this node, acknowledging before callbacks. Independent nodes do not compete. + Task SubscribeNodeAsync(Func handler, MessageNodeSubscriptionOptions options, CancellationToken cancellationToken = default) + => throw new NotSupportedException("This bus does not support node subscriptions."); + /// Enqueues work for a competing consumer. Returns its application message ID. Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; @@ -219,17 +253,52 @@ public sealed record MessageBusOptions public sealed class MessageBus : IMessageBus { private readonly MessageClientCore _core; + private readonly IMessageTransport _transport; private readonly ILogger _logger; public MessageBus(IMessageTransport transport, MessageBusOptions? options = null) { ArgumentNullException.ThrowIfNull(transport); + _transport = transport; options ??= new MessageBusOptions(); _logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, _logger, static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType, options.Topology); } + /// + public async Task SubscribeNodeAsync(Func handler, MessageNodeSubscriptionOptions options, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + ArgumentNullException.ThrowIfNull(options); + options.Validate(); + IManagedNodeSubscription? owner = null; + if (_transport is ISupportsManagedNodeSubscriptions managed) + owner = await managed.OpenNodeSubscriptionAsync(options, cancellationToken).ConfigureAwait(false); + else if (_transport is not ISupportsEphemeralSubscriptions) + throw new NotSupportedException("The transport supports neither expiring nor managed node subscriptions."); + try + { + var consumer = await SubscribeAsync(async (message, token) => + { + await message.CompleteAsync(token).ConfigureAwait(false); + await handler(message, token).ConfigureAwait(false); + }, new MessageSubscriptionOptions + { + Topic = options.Topic, + Subscription = owner?.Source.Name, + MaxConcurrency = options.MaxConcurrency, + AckMode = AckMode.Manual + }, cancellationToken).ConfigureAwait(false); + return owner is null ? consumer : new NodeMessageSubscription(consumer, owner); + } + catch + { + if (owner is not null) await owner.DisposeAsync().ConfigureAwait(false); + throw; + } + } + public Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); @@ -301,6 +370,20 @@ public Task> PublishBatchAsync(IEnumerable message public bool SupportsTemporarySubscriptions => _core.SupportsTemporarySubscriptions; + public Task ConsumeWithOutcomeAsync(Func, CancellationToken, ValueTask> handler, MessageConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(handler); + return ConsumeCoreAsync(options ?? new(), typeof(T), (config, token) => _core.StartOutcomeListenerAsync(config, handler, token), cancellationToken); + } + + public Task ConsumeWithOutcomeAsync(Func> handler, MessageConsumerOptions options, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Destination); + return ConsumeCoreAsync(options, typeof(object), (config, token) => _core.StartOutcomeListenerAsync(config, handler, token), cancellationToken); + } + public Task ConsumeAsync(Func, CancellationToken, Task> handler, MessageConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); @@ -368,6 +451,12 @@ private static ListenerConfig CreateListener(MessageHandlerOptions options, Type MessageType = messageType, AckMode = options.AckMode, MaxConcurrency = options.MaxConcurrency, + PrefetchCount = options.PrefetchCount, + ReceiveBatchDelay = options.ReceiveBatchDelay, + VisibilityTimeout = options.VisibilityTimeout, + AutoRenewLock = options.AutoRenewLock, + ShutdownTimeout = options.ShutdownTimeout, + WaitForManualSettlement = options.WaitForManualSettlement, MaxAttempts = options.MaxAttempts, RedeliveryBackoff = options.RedeliveryBackoff, DeadLetterWhen = options.DeadLetterWhen diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 5a7ab6a37..5d5bccc69 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -55,6 +55,12 @@ internal sealed record ListenerConfig public bool Ephemeral { get; init; } public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; + public int? PrefetchCount { get; init; } + public TimeSpan? ReceiveBatchDelay { get; init; } + public TimeSpan VisibilityTimeout { get; init; } = TimeSpan.FromMinutes(1); + public bool AutoRenewLock { get; init; } = true; + public TimeSpan ShutdownTimeout { get; init; } + public bool WaitForManualSettlement { get; init; } = true; // Null falls back to the client's default RetryPolicy. public int? MaxAttempts { get; init; } public Func? RedeliveryBackoff { get; init; } @@ -273,33 +279,43 @@ public async Task> SendBatchAsync(ScheduledDispatchKind ki public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(handler); - return RegisterConsumerAsync(config, async (entry, token) => + return RegisterConsumerAsync(config, async (entry, lease, token) => { - var received = CreateMessageContext(entry, token); + var received = CreateMessageContext(entry, token, lease); await HandleMessageAsync(received, config, handler, token).AnyContext(); }, cancellationToken); } + public Task StartOutcomeListenerAsync(ListenerConfig config, Func> handler, CancellationToken cancellationToken) + => RegisterConsumerAsync(config, (entry, lease, token) => HandleMessageOutcomeAsync(CreateMessageContext(entry, token, lease), config, handler, token), cancellationToken); + + public Task StartOutcomeListenerAsync(ListenerConfig config, Func, CancellationToken, ValueTask> handler, CancellationToken cancellationToken) where T : class + => RegisterConsumerAsync(config, async (entry, lease, token) => + { + var received = await CreateMessageContextAsync(entry, token, lease).AnyContext(); + await HandleMessageOutcomeAsync(received, config, handler, token).AnyContext(); + }, cancellationToken); + public Task?> ReceiveAsync(DestinationAddress source, TimeSpan wait, CancellationToken cancellationToken) where T : class { return ReceiveCoreAsync>(source, wait, async (entry, cancellation, supervision) => - new ReceivedMessage(await CreateMessageContextAsync(entry, cancellation.Token).AnyContext(), cancellation, supervision, ct => ReturnUnsettledAsync(entry, ct)), cancellationToken); + new ReceivedMessage(await CreateMessageContextAsync(entry, cancellation.Token, supervision).AnyContext(), cancellation, supervision, ct => ReturnUnsettledAsync(entry, ct)), cancellationToken); } public Task ReceiveAsync(DestinationAddress source, TimeSpan wait, CancellationToken cancellationToken) { return ReceiveCoreAsync(source, wait, (entry, cancellation, supervision) => - Task.FromResult(new ReceivedMessage(CreateMessageContext(entry, cancellation.Token), cancellation, supervision, ct => ReturnUnsettledAsync(entry, ct))), cancellationToken); + Task.FromResult(new ReceivedMessage(CreateMessageContext(entry, cancellation.Token, supervision), cancellation, supervision, ct => ReturnUnsettledAsync(entry, ct))), cancellationToken); } private async Task ReceiveCoreAsync(DestinationAddress source, TimeSpan wait, - Func, Task> create, CancellationToken cancellationToken) where T : class + Func> create, CancellationToken cancellationToken) where T : class { ThrowIfDisposed(); ArgumentOutOfRangeException.ThrowIfLessThan(wait, TimeSpan.Zero); var pull = RequirePull(); var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _lifetimeCancellation.Token); - Task supervision = Task.FromResult(false); + MessageDeliveryLease? supervision = null; bool transferred = false; try { @@ -311,7 +327,7 @@ public Task StartListenerAsync(ListenerConfig config, Fun if (entries.Count == 0) return null; - supervision = SuperviseLeaseAsync(entries[0], cancellation); + supervision = new MessageDeliveryLease(_transport, entries[0], TimeSpan.FromMinutes(1), true, _timeProvider, _logger, cancellation); var received = await create(entries[0], cancellation, supervision).AnyContext(); transferred = true; return received; @@ -321,7 +337,7 @@ public Task StartListenerAsync(ListenerConfig config, Fun if (!transferred) { await cancellation.CancelAsync().AnyContext(); - await supervision.AnyContext(); + if (supervision is not null) await supervision.DisposeAsync().AnyContext(); cancellation.Dispose(); } } @@ -330,9 +346,9 @@ public Task StartListenerAsync(ListenerConfig config, Fun public Task StartListenerAsync(ListenerConfig config, Func, CancellationToken, Task> handler, CancellationToken cancellationToken) where T : class { ArgumentNullException.ThrowIfNull(handler); - return RegisterConsumerAsync(config, async (entry, token) => + return RegisterConsumerAsync(config, async (entry, lease, token) => { - var received = await CreateMessageContextAsync(entry, token).AnyContext(); + var received = await CreateMessageContextAsync(entry, token, lease).AnyContext(); await HandleMessageAsync(received, config, handler, token).AnyContext(); }, cancellationToken); } @@ -358,7 +374,7 @@ public async ValueTask DisposeAsync() // Multiple typed consumers can share one destination. They attach to a single per-source listener whose loop // demultiplexes each message to the consumer registered for its type; a type with no registered consumer is // handled by HandleUnmatchedAsync. Duplicate registration on one endpoint is rejected. - private async Task RegisterConsumerAsync(ListenerConfig config, Func dispatch, CancellationToken cancellationToken) + private async Task RegisterConsumerAsync(ListenerConfig config, Func dispatch, CancellationToken cancellationToken) { ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); @@ -432,18 +448,28 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, DestinationAddress // (no head-of-line blocking) and steady-state utilization stays at the configured concurrency. A failure while // receiving or while processing a single entry (including a poison message that was already dead-lettered) must // never tear down the loop, otherwise one bad message or a transient transport blip silently stops consumption. - private async Task RunPullLoopAsync(DestinationAddress source, ISupportsPull pull, Func onMessage, int maxConcurrency, CancellationToken cancellationToken, Action? receivingHealth = null) + private async Task RunPullLoopAsync(DestinationAddress source, ISupportsPull pull, Func onMessage, ListenerConfig endpoint, CancellationToken cancellationToken, Action? receivingHealth = null) { - maxConcurrency = Math.Max(1, maxConcurrency); + int maxConcurrency = Math.Max(1, endpoint.MaxConcurrency); var capabilities = (_transport as ITransportInfo)?.GetCapabilities(source); int batchSize = Math.Clamp(capabilities?.MaxReceiveBatchSize ?? maxConcurrency, 1, maxConcurrency); - var batchDelay = capabilities?.ReceiveBatchDelay ?? TimeSpan.Zero; + if (endpoint.PrefetchCount is { } prefetch) + batchSize = Math.Min(batchSize, prefetch); + var batchDelay = endpoint.ReceiveBatchDelay.HasValue ? TimeSpan.Zero : capabilities?.ReceiveBatchDelay ?? TimeSpan.Zero; + int largestReceivedBatch = 0; var slots = new SemaphoreSlim(maxConcurrency, maxConcurrency); var inFlight = new ConcurrentDictionary(); var cleanupSlots = new SemaphoreSlim(maxConcurrency, maxConcurrency); using var collecting = new SemaphoreSlim(1, 1); using var receivingCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var receivingToken = receivingCancellation.Token; + using var processingCancellation = new CancellationTokenSource(); + var processingToken = processingCancellation.Token; + using var stopRegistration = receivingToken.Register(() => + { + if (endpoint.ShutdownTimeout == TimeSpan.Zero) processingCancellation.Cancel(); + else processingCancellation.CancelAfter(endpoint.ShutdownTimeout); + }); int receiveConcurrency = Math.Clamp(capabilities?.MaxConcurrentReceives ?? 1, 1, (maxConcurrency - 1) / batchSize + 1); try { @@ -474,6 +500,13 @@ async Task ReceiveAsync(CancellationToken cancellationToken) { await collecting.WaitAsync(cancellationToken).AnyContext(); collectingBatch = true; + if (endpoint.ReceiveBatchDelay is { } collectionDelay && collectionDelay > TimeSpan.Zero + && maxConcurrency - slots.CurrentCount >= (maxConcurrency + 1) / 2 + && slots.CurrentCount < largestReceivedBatch) + { + await Task.WhenAny(Task.WhenAll(inFlight.Keys.ToArray()), Task.Delay(collectionDelay, _timeProvider, cancellationToken)).AnyContext(); + cancellationToken.ThrowIfCancellationRequested(); + } await slots.WaitAsync(cancellationToken).AnyContext(); claimed = 1; long batchStart = batchDelay > TimeSpan.Zero ? Stopwatch.GetTimestamp() : 0; @@ -512,9 +545,18 @@ async Task ReceiveAsync(CancellationToken cancellationToken) MaxMessages = claimed, MaxWaitTime = pollWindow }; - entries = _transport is ISupportsVisibilityTimeout visibility - ? await visibility.ReceiveAsync(source, request, TimeSpan.FromMinutes(1), cancellationToken).AnyContext() - : await pull.ReceiveAsync(source, request, cancellationToken).AnyContext(); + var receive = _transport is ISupportsVisibilityTimeout visibility + ? visibility.ReceiveAsync(source, request, endpoint.VisibilityTimeout, cancellationToken) + : pull.ReceiveAsync(source, request, cancellationToken); + try { entries = await receive.WaitAsync(cancellationToken).AnyContext(); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + var cleanup = ReleaseLateReceiveAsync(receive); + // Join cooperative receive cancellation before disposal completes, while a provider that + // ignores cancellation gets a bounded grace period and releases its late deliveries separately. + await Task.WhenAny(cleanup, Task.Delay(TimeSpan.FromMilliseconds(50))).AnyContext(); + throw; + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -550,6 +592,7 @@ async Task ReceiveAsync(CancellationToken cancellationToken) // over-returns would otherwise release more slots than acquired (breaching the cap / overflowing the // semaphore). Any over-returned entries are left unsettled and redeliver after their visibility window. int toProcess = Math.Min(entries.Count, claimed); + largestReceivedBatch = Math.Max(largestReceivedBatch, toProcess); ReleaseSlots(slots, claimed - toProcess); // return slots we claimed but won't fill (always >= 0) // An empty poll should have blocked for MaxWaitTime; a transport that returns empty early (or @@ -563,7 +606,15 @@ async Task ReceiveAsync(CancellationToken cancellationToken) for (int index = 0; index < toProcess; index++) { - var task = SafeProcessAsync(entries[index], onMessage, source, cancellationToken, slots, cleanupSlots); + if (cancellationToken.IsCancellationRequested) + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5), _timeProvider); + await ReturnUnsettledAsync(entries[index], cleanup.Token).AnyContext(); + (_transport as IMessageProcessingObserver)?.ProcessingFinished(entries[index]); + slots.Release(); + continue; + } + var task = SafeProcessAsync(entries[index], onMessage, source, processingToken, slots, cleanupSlots, endpoint); if (!task.IsCompleted) { inFlight[task] = 0; @@ -586,48 +637,42 @@ private static void ReleaseSlots(SemaphoreSlim slots, int count) slots.Release(count); } - private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, DestinationAddress source, CancellationToken cancellationToken, SemaphoreSlim? slots = null, SemaphoreSlim? cleanupSlots = null) + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, DestinationAddress source, CancellationToken cancellationToken, SemaphoreSlim? slots = null, SemaphoreSlim? cleanupSlots = null, ListenerConfig? endpoint = null) { + (_transport as IMessageProcessingObserver)?.ProcessingStarted(entry); using var deliveryCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - var supervision = SuperviseLeaseAsync(entry, deliveryCancellation); + await using var lease = new MessageDeliveryLease(_transport, entry, endpoint?.VisibilityTimeout ?? TimeSpan.FromMinutes(1), + endpoint?.AutoRenewLock ?? true, _timeProvider, _logger, deliveryCancellation); try { deliveryCancellation.Token.ThrowIfCancellationRequested(); - await onMessage(entry, deliveryCancellation.Token).AnyContext(); - } - catch (OperationCanceledException) when (deliveryCancellation.IsCancellationRequested) - { - bool leaseLost = await supervision.AnyContext(); - if (cancellationToken.IsCancellationRequested && !leaseLost) - { - using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5), _timeProvider); - await ReturnUnsettledAsync(entry, cleanup.Token).AnyContext(); - } - } - catch (UnhandledMessageTypeException) - { - // Already settled AND classified (WARN-retryable / ERROR-terminal) by HandleUnmatchedAsync; re-logging - // here would emit an ERROR for every retryable attempt. + await onMessage(entry, lease, deliveryCancellation.Token).AnyContext(); } + catch (OperationCanceledException) when (deliveryCancellation.IsCancellationRequested) { } + catch (UnhandledMessageTypeException) { } catch (Exception ex) { - // The message has already been settled (dead-lettered on deserialize failure, abandoned/dead-lettered on - // handler error); swallowing here keeps the loop alive for the next message. _logger.LogError(ex, "Error processing message \"{MessageId}\" from \"{Source}\": {Message}", entry.Id, source, ex.Message); } finally { + if (cancellationToken.IsCancellationRequested && !lease.IsLost && !lease.IsSettled) + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5), _timeProvider); + await ReturnUnsettledAsync(entry, cleanup.Token).AnyContext(); + } + lease.Settled(); var cancellation = deliveryCancellation.CancelAsync(); - // Bound deferred cleanup separately so slow cancellation callbacks cannot retain unlimited deliveries. if (cleanupSlots is not null) await cleanupSlots.WaitAsync().AnyContext(); try { slots?.Release(); await cancellation.AnyContext(); - await supervision.AnyContext(); + await lease.Completion.AnyContext(); } finally { cleanupSlots?.Release(); } + (_transport as IMessageProcessingObserver)?.ProcessingFinished(entry); } } @@ -644,61 +689,29 @@ private async Task ReturnUnsettledAsync(TransportEntry entry, CancellationToken } } - private async Task SuperviseLeaseAsync(TransportEntry entry, CancellationTokenSource deliveryCancellation) + private async Task ReleaseLateReceiveAsync(Task> receive) { - if (entry.LockExpiresUtc is not { } expires) - return false; - - var token = deliveryCancellation.Token; - var duration = TimeSpan.FromMinutes(1); - if (_transport is ISupportsVisibilityTimeout { MaxVisibilityTimeout: { } maximum } && duration > maximum) - duration = maximum; - try { - while (!token.IsCancellationRequested) + foreach (var entry in await receive.AnyContext()) { - var remaining = expires - _timeProvider.GetUtcNow(); - if (remaining <= TimeSpan.Zero) - throw new ReceiptExpiredException(); - - if (_transport is not ISupportsLockRenewal renewal) - { - await Task.Delay(remaining, _timeProvider, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); - if (token.IsCancellationRequested) - return expires <= _timeProvider.GetUtcNow(); - throw new ReceiptExpiredException(); - } - - await Task.Delay(remaining / 2, _timeProvider, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); - if (token.IsCancellationRequested) - return expires <= _timeProvider.GetUtcNow(); - var started = _timeProvider.GetUtcNow(); - remaining = expires - started; - if (remaining <= TimeSpan.Zero) - throw new ReceiptExpiredException(); - - using var deadline = new CancellationTokenSource(remaining, _timeProvider); - using var renewalCancellation = CancellationTokenSource.CreateLinkedTokenSource(token, deadline.Token); - await renewal.RenewLockAsync(entry, duration, renewalCancellation.Token) - .WaitAsync(remaining, _timeProvider, token).AnyContext(); - expires = started.Add(duration); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5), _timeProvider); + await ReturnUnsettledAsync(entry, cleanup.Token).AnyContext(); + (_transport as IMessageProcessingObserver)?.ProcessingFinished(entry); } } - catch (OperationCanceledException) when (token.IsCancellationRequested) - { - return expires <= _timeProvider.GetUtcNow(); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Lease lost for message {MessageId} from {Source}; cancelling its handler", entry.Id, entry.Destination); - await deliveryCancellation.CancelAsync().AnyContext(); - return true; - } - return expires <= _timeProvider.GetUtcNow(); + catch (OperationCanceledException) { } + catch (Exception exception) { _logger.LogWarning(exception, "Receive finished after its listener stopped"); } } private async Task HandleMessageAsync(TMessage message, ListenerConfig config, Func handler, CancellationToken cancellationToken) where TMessage : IMessageContext + => await HandleMessageOutcomeAsync(message, config, async (context, token) => + { + await handler(context, token).AnyContext(); + return MessageOutcome.Success; + }, cancellationToken).AnyContext(); + + private async Task HandleMessageOutcomeAsync(TMessage message, ListenerConfig config, Func> handler, CancellationToken cancellationToken) where TMessage : IMessageContext { // Re-establish the producer's trace context on the consumer side so a cross-process trace continues here // instead of breaking at the transport boundary. @@ -706,11 +719,18 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig long startTimestamp = Stopwatch.GetTimestamp(); try { - await handler(message, cancellationToken).AnyContext(); + var outcome = await handler(message, cancellationToken).AnyContext(); + if (outcome.Kind is MessageOutcomeKind.Retry or MessageOutcomeKind.DeadLetter) + { + await outcome.SettleFailureAsync(message, config.MaxAttempts ?? _retryPolicy.MaxAttempts, config.RedeliveryBackoff ?? _retryPolicy.Backoff, cancellationToken).AnyContext(); + return; + } + if (outcome.Kind == MessageOutcomeKind.Unsettled) + return; if (config.AckMode == AckMode.Auto && !message.IsHandled) await message.CompleteAsync(cancellationToken).AnyContext(); - else if (config.AckMode == AckMode.Manual && message is MessageContext context) + else if (config.AckMode == AckMode.Manual && config.WaitForManualSettlement && message is MessageContext context) await context.WaitForSettlementAsync(cancellationToken).AnyContext(); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -744,7 +764,7 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig // A retry that can still happen is a warning; the terminal decision (unrecoverable or attempts exhausted) // is the error worth alerting on. - if (unrecoverable || message.Attempts >= maxAttempts) + if (unrecoverable || maxAttempts >= 0 && message.Attempts >= maxAttempts) _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}); dead-lettering: {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); else _logger.LogWarning(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}); will retry: {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); @@ -785,21 +805,17 @@ private static Task SettleFailedMessageAsync(IMessageContext message, bool unrec if (message.IsHandled) return Task.CompletedTask; - if (unrecoverable || message.Attempts >= maxAttempts) - return message.RejectAsync(new RejectOptions { Terminal = true, Reason = deadLetterReason, Exception = exception }, cancellationToken); - - // Policy-driven delays are best-effort: a transport that can't honor the delay redelivers immediately rather - // than failing the settle (an explicit caller-requested delay stays strict). - return message.RejectAsync(new RejectOptions { RedeliveryDelay = backoff?.Invoke(message.Attempts), BestEffortDelay = true }, cancellationToken); + var outcome = unrecoverable ? MessageOutcome.DeadLetter(deadLetterReason, exception) : MessageOutcome.Retry(deadLetterReason, exception); + return outcome.SettleFailureAsync(message, maxAttempts, backoff, cancellationToken); } - private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken) + private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken, MessageDeliveryLease? lease = null) { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination.Key)); - return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger, _topologyMode); + return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger, _topologyMode, lease); } - private async Task> CreateMessageContextAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class + private async Task> CreateMessageContextAsync(TransportEntry entry, CancellationToken cancellationToken, MessageDeliveryLease? lease = null) where T : class { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination.Key)); @@ -854,7 +870,7 @@ private async Task> CreateMessageContextAsync(TransportEnt throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); } - return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger, _topologyMode); + return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger, _topologyMode, lease); } private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, Exception? exception, CancellationToken cancellationToken) @@ -1098,7 +1114,7 @@ private sealed class ConsumerRegistration { public required string Key { get; init; } public required ListenerConfig Config { get; init; } - public required Func Dispatch { get; init; } + public required Func Dispatch { get; init; } public required bool IsCatchAll { get; init; } public required string? TypeName { get; init; } } @@ -1116,6 +1132,7 @@ private sealed class SourceListener private readonly ConcurrentDictionary _byType = new(StringComparer.Ordinal); private readonly ConsumerGroup _catchAll = new(); private int _maxConcurrency = 1; + private ListenerConfig? _endpoint; private bool _ephemeral; private Task? _loop; private int _status = (int)MessageSubscriptionStatus.Starting; @@ -1158,6 +1175,7 @@ public bool TryAddConsumer(ConsumerRegistration registration, out MessageListene if (_consumers.IsEmpty) { _maxConcurrency = desired; + _endpoint = registration.Config; _ephemeral = registration.Config.Ephemeral; created = true; } @@ -1165,6 +1183,14 @@ public bool TryAddConsumer(ConsumerRegistration registration, out MessageListene { throw new InvalidOperationException($"Source \"{_source}\" is already consumed with MaxConcurrency {_maxConcurrency}; a conflicting MaxConcurrency {desired} was requested. Consumers sharing a destination must use the same MaxConcurrency."); } + else if (_endpoint is { } endpoint && (endpoint.PrefetchCount != registration.Config.PrefetchCount + || endpoint.ReceiveBatchDelay != registration.Config.ReceiveBatchDelay + || endpoint.VisibilityTimeout != registration.Config.VisibilityTimeout + || endpoint.AutoRenewLock != registration.Config.AutoRenewLock + || endpoint.ShutdownTimeout != registration.Config.ShutdownTimeout)) + { + throw new InvalidOperationException($"Consumers sharing '{_source}' must agree on receive, lease, and shutdown settings."); + } handle = new MessageListenerHandle(_source, registration.Key, () => RemoveConsumerAsync(registration.Key), () => Status, () => RecoveryVersion, WaitUntilReadyAsync); _consumers[registration.Key] = new Registered(registration, handle); @@ -1234,7 +1260,7 @@ private async Task RunReceiverAsync(CancellationToken cancellationToken) { if (_core._transport is ISupportsPull pull) { - await _core.RunPullLoopAsync(_source, pull, DispatchAsync, _maxConcurrency, cancellationToken, healthy => + await _core.RunPullLoopAsync(_source, pull, DispatchAsync, _endpoint!, cancellationToken, healthy => { int previous = Interlocked.Exchange(ref _status, (int)(healthy ? MessageSubscriptionStatus.Healthy : MessageSubscriptionStatus.Recovering)); if (!healthy && previous == (int)MessageSubscriptionStatus.Healthy) @@ -1243,7 +1269,7 @@ await _core.RunPullLoopAsync(_source, pull, DispatchAsync, _maxConcurrency, canc return; } await using var push = await ((ISupportsPush)_core._transport).SubscribeAsync(_source, - (entry, token) => _core.SafeProcessAsync(entry, DispatchAsync, _source, token), + (entry, token) => _core.SafeProcessAsync(entry, DispatchAsync, _source, token, endpoint: _endpoint), new PushOptions { MaxConcurrentMessages = _maxConcurrency }, cancellationToken).AnyContext(); await Task.Delay(Timeout.Infinite, cancellationToken).AnyContext(); } @@ -1347,7 +1373,7 @@ private async Task ShutdownAsync() } } - private async Task DispatchAsync(TransportEntry entry, CancellationToken token) + private async Task DispatchAsync(TransportEntry entry, MessageDeliveryLease lease, CancellationToken token) { if (entry.EnvelopeError is { } error) { @@ -1361,7 +1387,7 @@ private async Task DispatchAsync(TransportEntry entry, CancellationToken token) return; } - await registration.Dispatch(entry, token).AnyContext(); + await registration.Dispatch(entry, lease, token).AnyContext(); } private ConsumerRegistration? Resolve(TransportEntry entry) @@ -1398,9 +1424,11 @@ internal class MessageContext : IMessageContext private readonly TopologyMode _topologyMode; private int _isHandled; private TaskCompletionSource? _settled; + private readonly MessageDeliveryLease? _lease; - public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null, TopologyMode topologyMode = TopologyMode.Ensure) + public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null, TopologyMode topologyMode = TopologyMode.Ensure, MessageDeliveryLease? lease = null) { + _lease = lease; _transport = transport; _entry = entry; _runtimeStore = runtimeStore; @@ -1412,6 +1440,9 @@ public MessageContext(IMessageTransport transport, TransportEntry entry, Cancell } public string Id => _entry.ApplicationMessageId ?? _entry.Headers.GetValueOrDefault(KnownHeaders.MessageId) ?? _entry.Id; + public DestinationAddress Destination => _entry.Destination; + public DateTimeOffset? EnqueuedUtc => _entry.EnqueuedUtc; + public bool IsLeaseLost => _lease?.IsLost == true; public string BrokerMessageId => _entry.Id; public ReadOnlyMemory Body => _entry.Body; public MessageHeaders Headers => _entry.Headers; @@ -1440,6 +1471,7 @@ public async Task CompleteAsync(CancellationToken cancellationToken = default) { await _transport.CompleteAsync(_entry, cancellationToken).AnyContext(); Volatile.Write(ref _isHandled, 2); + _lease?.Settled(); Volatile.Read(ref _settled)?.TrySetResult(); MessagingInstruments.Completed.Add(1, new KeyValuePair("source", _entry.Destination.Key)); } @@ -1462,6 +1494,7 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c { await RejectCoreAsync(options, cancellationToken).AnyContext(); Volatile.Write(ref _isHandled, 2); + _lease?.Settled(); Volatile.Read(ref _settled)?.TrySetResult(); var counter = options?.Terminal == true ? MessagingInstruments.DeadLettered : MessagingInstruments.Abandoned; counter.Add(1, new KeyValuePair("source", _entry.Destination.Key)); @@ -1540,6 +1573,8 @@ await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) { + if (_lease is not null) + return _lease.RenewAsync(duration, cancellationToken == default ? CancellationToken : cancellationToken); return _transport is ISupportsLockRenewal lockRenewal ? lockRenewal.RenewLockAsync(_entry, duration, cancellationToken == default ? CancellationToken : cancellationToken) : throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support lock renewal."); @@ -1547,7 +1582,7 @@ public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancella internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, string? deadLetterDestination, ILogger logger, CancellationToken cancellationToken, TopologyMode topologyMode = TopologyMode.Ensure) { - if (transport is ISupportsDeadLetter deadLetter) + if (transport is ISupportsDeadLetterSink deadLetter) { await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); return; @@ -1646,8 +1681,8 @@ private static int ParseAttemptsHeader(MessageHeaders headers) internal sealed class MessageContext : MessageContext, IMessageContext where T : class { - public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null, TopologyMode topologyMode = TopologyMode.Ensure) - : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination, logger, topologyMode) + public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null, TopologyMode topologyMode = TopologyMode.Ensure, MessageDeliveryLease? lease = null) + : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination, logger, topologyMode, lease) { Message = message; } diff --git a/src/Foundatio/Messaging/MessageDeliveryLease.cs b/src/Foundatio/Messaging/MessageDeliveryLease.cs new file mode 100644 index 000000000..9455e55f6 --- /dev/null +++ b/src/Foundatio/Messaging/MessageDeliveryLease.cs @@ -0,0 +1,118 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; +using Microsoft.Extensions.Logging; + +namespace Foundatio.Messaging; + +/// Coordinates automatic and explicit renewal against one conservative delivery deadline. +internal sealed class MessageDeliveryLease : IAsyncDisposable +{ + private readonly IMessageTransport _transport; + private readonly TransportEntry _entry; + private readonly TimeSpan _duration; + private readonly TimeProvider _time; + private readonly ILogger _logger; + private readonly CancellationTokenSource _processing; + private readonly CancellationTokenSource _renewal = new(); + private readonly SemaphoreSlim _gate = new(1); + private long _expiresTicks; + private int _lost; + private int _settled; + + public MessageDeliveryLease(IMessageTransport transport, TransportEntry entry, TimeSpan duration, bool autoRenew, + TimeProvider time, ILogger logger, CancellationTokenSource processing) + { + _transport = transport; + _entry = entry; + _duration = transport is ISupportsVisibilityTimeout { MaxVisibilityTimeout: { } maximum } && duration > maximum ? maximum : duration; + _time = time; + _logger = logger; + _processing = processing; + _expiresTicks = entry.LockExpiresUtc?.UtcTicks ?? DateTimeOffset.MaxValue.UtcTicks; + Completion = entry.LockExpiresUtc is null ? Task.CompletedTask : MonitorAsync(autoRenew); + } + + public Task Completion { get; } + public bool IsLost => Volatile.Read(ref _lost) != 0 || !IsSettled && Remaining <= TimeSpan.Zero; + public bool IsSettled => Volatile.Read(ref _settled) != 0; + private TimeSpan Remaining => new(Interlocked.Read(ref _expiresTicks) - _time.GetUtcNow().UtcTicks); + + public void Settled() + { + Interlocked.Exchange(ref _settled, 1); + _renewal.Cancel(); + } + + public async Task RenewAsync(TimeSpan? duration, CancellationToken cancellationToken) + { + var extension = duration ?? _duration; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(extension, TimeSpan.Zero); + if (_transport is ISupportsVisibilityTimeout { MaxVisibilityTimeout: { } maximum } && extension > maximum) + throw new ArgumentOutOfRangeException(nameof(duration), $"The transport supports a maximum lease of {maximum}."); + if (_transport is not ISupportsLockRenewal renewal) + throw new NotSupportedException($"Transport {_transport.GetType().Name} does not support delivery lease renewal."); + await _gate.WaitAsync(cancellationToken).AnyContext(); + try + { + if (IsSettled || IsLost || Remaining <= TimeSpan.Zero) + throw new ReceiptExpiredException(); + var remaining = Remaining; + var started = _time.GetUtcNow(); + using var deadline = new CancellationTokenSource(remaining > TimeSpan.FromSeconds(30) ? TimeSpan.FromSeconds(30) : remaining, _time); + using var operation = CancellationTokenSource.CreateLinkedTokenSource(deadline.Token, cancellationToken); + await renewal.RenewLockAsync(_entry, extension, operation.Token).WaitAsync(operation.Token).AnyContext(); + Interlocked.Exchange(ref _expiresTicks, started.Add(extension).UtcTicks); + } + finally { _gate.Release(); } + } + + private async Task MonitorAsync(bool autoRenew) + { + var token = _renewal.Token; + bool retry = false; + try + { + while (!token.IsCancellationRequested) + { + var remaining = Remaining; + if (remaining <= TimeSpan.Zero) break; + bool canRenew = autoRenew && _transport is ISupportsLockRenewal; + var delay = !canRenew ? remaining : retry + ? TimeSpan.FromTicks(Math.Min(TimeSpan.TicksPerSecond, remaining.Ticks / 4)) : remaining / 2; + await Task.Delay(delay, _time, token).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + if (token.IsCancellationRequested) return; + if (Remaining <= TimeSpan.Zero) break; + if (!canRenew) continue; + try + { + await RenewAsync(_duration, token).AnyContext(); + retry = false; + } + catch (ReceiptExpiredException) { break; } + catch (OperationCanceledException) when (token.IsCancellationRequested) { return; } + catch (Exception exception) + { + retry = true; + _logger.LogWarning(exception, "Unable to renew delivery {MessageId} at {Source}; retrying within its lease", _entry.Id, _entry.Destination); + } + } + if (!token.IsCancellationRequested && !IsSettled) + { + Interlocked.Exchange(ref _lost, 1); + _logger.LogWarning("Delivery lease lost for {MessageId} at {Source}; cancelling processing", _entry.Id, _entry.Destination); + await _processing.CancelAsync().AnyContext(); + } + } + catch (OperationCanceledException) when (token.IsCancellationRequested) { } + } + + public async ValueTask DisposeAsync() + { + await _renewal.CancelAsync().AnyContext(); + await Completion.AnyContext(); + _renewal.Dispose(); + // Explicit renewal may still be unwinding after settlement. SemaphoreSlim owns no wait handle here. + } +} diff --git a/src/Foundatio/Messaging/MessageHandlerRegistration.cs b/src/Foundatio/Messaging/MessageHandlerRegistration.cs index 56f2dc479..2bd5c728c 100644 --- a/src/Foundatio/Messaging/MessageHandlerRegistration.cs +++ b/src/Foundatio/Messaging/MessageHandlerRegistration.cs @@ -16,4 +16,5 @@ internal sealed class MessageHandlerRegistration } /// The DI-selected , applied at startup and by the message clients on use. -internal sealed record MessagingTopologyOptions(TopologyMode Mode); +/// The effective topology policy selected for this messaging client. +public sealed record MessagingTopologyOptions(TopologyMode Mode); diff --git a/src/Foundatio/Messaging/MessageNodeSubscription.cs b/src/Foundatio/Messaging/MessageNodeSubscription.cs new file mode 100644 index 000000000..d79780f13 --- /dev/null +++ b/src/Foundatio/Messaging/MessageNodeSubscription.cs @@ -0,0 +1,59 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Messaging; + +/// A best-effort subscription for one running node, independent of durable service subscriptions. +public sealed record MessageNodeSubscriptionOptions +{ + /// Topic broadcast to the running nodes. + public required string Topic { get; init; } + /// Diagnostic node identity; providers may add a unique resource suffix. + public string NodeId { get; init; } = Guid.NewGuid().ToString("N"); + /// Maximum callbacks running concurrently on this node. + public int MaxConcurrency { get; init; } = 10; + /// Interval for renewing managed node ownership. + public TimeSpan HeartbeatInterval { get; init; } = TimeSpan.FromMinutes(2); + /// Time without a heartbeat before another node may clean up resources. + public TimeSpan StaleAfter { get; init; } = TimeSpan.FromMinutes(10); + /// Maximum retained backlog on managed node resources. + public TimeSpan MessageRetention { get; init; } = TimeSpan.FromMinutes(5); + + /// Rejects invalid lifecycle and receiving settings. + public void Validate() + { + ArgumentException.ThrowIfNullOrWhiteSpace(Topic); + ArgumentException.ThrowIfNullOrWhiteSpace(NodeId); + ArgumentOutOfRangeException.ThrowIfLessThan(MaxConcurrency, 1); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(HeartbeatInterval, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(StaleAfter, HeartbeatInterval); + } +} + +/// A provider-managed node subscription. Disposal removes its infrastructure. +public interface IManagedNodeSubscription : IAsyncDisposable +{ + /// Native subscription destination owned by this node. + DestinationAddress Source { get; } +} + +/// Manages node resources on brokers without natively expiring subscriptions. Does not advertise native expiration. +public interface ISupportsManagedNodeSubscriptions : IMessageTransport +{ + /// Creates a node subscription and starts renewing its managed ownership. + Task OpenNodeSubscriptionAsync(MessageNodeSubscriptionOptions options, CancellationToken cancellationToken = default); +} + +internal sealed class NodeMessageSubscription(IMessageSubscription consumer, IManagedNodeSubscription owner) : IMessageSubscription +{ + public DestinationAddress Source => consumer.Source; + public MessageSubscriptionStatus Status => consumer.Status; + public long RecoveryVersion => consumer.RecoveryVersion; + public Task WaitUntilReadyAsync(CancellationToken cancellationToken = default) => consumer.WaitUntilReadyAsync(cancellationToken); + public async ValueTask DisposeAsync() + { + try { await consumer.DisposeAsync().ConfigureAwait(false); } + finally { await owner.DisposeAsync().ConfigureAwait(false); } + } +} diff --git a/src/Foundatio/Messaging/MessageOutcome.cs b/src/Foundatio/Messaging/MessageOutcome.cs new file mode 100644 index 000000000..283157275 --- /dev/null +++ b/src/Foundatio/Messaging/MessageOutcome.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Messaging; + +/// A handler's delivery decision. Returning an expected failure does not require throwing an exception. +public readonly record struct MessageOutcome +{ + private MessageOutcome(MessageOutcomeKind kind, string? reason, Exception? exception) + { + Kind = kind; + Reason = reason; + Exception = exception; + } + + /// The requested settlement. An already confirmed explicit settlement takes precedence. + public MessageOutcomeKind Kind { get; } + /// A diagnostic reason retained when dead-lettering. + public string? Reason { get; } + /// Optional exception retained as dead-letter evidence. + public Exception? Exception { get; } + /// Processing succeeded; automatic acknowledgement applies. + public static MessageOutcome Success => default; + /// Leave the delivery unsettled for its lease to expire. + public static MessageOutcome Unsettled => new(MessageOutcomeKind.Unsettled, null, null); + /// Retry using the endpoint policy, or dead-letter when its attempt budget is exhausted. + public static MessageOutcome Retry(string? reason = null, Exception? exception = null) => new(MessageOutcomeKind.Retry, reason, exception); + /// Dead-letter immediately with the supplied reason. + public static MessageOutcome DeadLetter(string reason, Exception? exception = null) => new(MessageOutcomeKind.DeadLetter, reason, exception); + + internal Task SettleFailureAsync(IMessageContext context, int maxAttempts, Func? backoff, CancellationToken cancellationToken) + { + if (context.IsHandled) + return Task.CompletedTask; + bool terminal = Kind == MessageOutcomeKind.DeadLetter || maxAttempts >= 0 && context.Attempts >= maxAttempts; + return context.RejectAsync(new RejectOptions + { + Terminal = terminal, + Reason = Reason, + Exception = Exception, + RedeliveryDelay = terminal ? null : backoff?.Invoke(context.Attempts), + BestEffortDelay = true + }, cancellationToken); + } +} + +/// The delivery decision returned by an outcome handler. +public enum MessageOutcomeKind +{ + Success, + Retry, + DeadLetter, + Unsettled +} diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 5ef91c409..88a90c7f2 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -163,6 +163,8 @@ public sealed record MessageDestinationStats // e.g. SQS ApproximateNumberOf*). public long Queued { get; init; } public long Working { get; init; } + /// Messages awaiting delayed redelivery, when the provider can report this gauge. + public long? Delayed { get; init; } public long Deadletter { get; init; } // Lifetime counters. Not universally available — a transport that does not track a counter leaves it null (e.g. @@ -372,9 +374,14 @@ public interface ISupportsRedeliveryDelay : IMessageTransport Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct = default); } -public interface ISupportsDeadLetter : IMessageTransport +/// A native dead-letter destination, independently of non-destructive administration support. +public interface ISupportsDeadLetterSink : IMessageTransport { Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct = default); +} + +public interface ISupportsDeadLetter : ISupportsDeadLetterSink +{ /// Inspects raw dead letters without consuming them. Task> PeekDeadLetteredAsync(DestinationAddress destination, DeadLetterQuery? query = null, CancellationToken cancellationToken = default); diff --git a/src/Foundatio/Messaging/ReceivedMessage.cs b/src/Foundatio/Messaging/ReceivedMessage.cs index 1246c410f..ff0016a46 100644 --- a/src/Foundatio/Messaging/ReceivedMessage.cs +++ b/src/Foundatio/Messaging/ReceivedMessage.cs @@ -21,10 +21,13 @@ public sealed record MessageReceiveOptions public TimeSpan WaitTime { get; init; } } -internal class ReceivedMessage(IMessageContext context, CancellationTokenSource cancellation, Task supervision, Func abandon) : IReceivedMessage +internal class ReceivedMessage(IMessageContext context, CancellationTokenSource cancellation, MessageDeliveryLease supervision, Func abandon) : IReceivedMessage { private int _disposed; public string Id => context.Id; + public DestinationAddress? Destination => context.Destination; + public DateTimeOffset? EnqueuedUtc => context.EnqueuedUtc; + public bool IsLeaseLost => context.IsLeaseLost; public string BrokerMessageId => context.BrokerMessageId; public ReadOnlyMemory Body => context.Body; public MessageHeaders Headers => context.Headers; @@ -58,7 +61,8 @@ public async ValueTask DisposeAsync() try { await cancellation.CancelAsync().AnyContext(); - bool leaseLost = await supervision.AnyContext(); + await supervision.DisposeAsync().AnyContext(); + bool leaseLost = supervision.IsLost; if (!context.IsHandled && !leaseLost) { using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(5)); @@ -68,13 +72,12 @@ public async ValueTask DisposeAsync() finally { await cancellation.CancelAsync().AnyContext(); - await supervision.AnyContext(); cancellation.Dispose(); } } } -internal sealed class ReceivedMessage(IMessageContext context, CancellationTokenSource cancellation, Task supervision, Func abandon) +internal sealed class ReceivedMessage(IMessageContext context, CancellationTokenSource cancellation, MessageDeliveryLease supervision, Func abandon) : ReceivedMessage(context, cancellation, supervision, abandon), IReceivedMessage where T : class { public T Message => context.Message; diff --git a/src/Foundatio/Messaging/Tracking/ExecutionHeaders.cs b/src/Foundatio/Messaging/Tracking/ExecutionHeaders.cs new file mode 100644 index 000000000..1905e67cc --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/ExecutionHeaders.cs @@ -0,0 +1,11 @@ +namespace Foundatio.Messaging; + +/// Optional execution metadata carried by ordinary Foundatio messages. +public static class ExecutionHeaders +{ + public const string ExecutionId = "message.execution.id"; + public const string OriginalExecutionId = "message.execution.original_id"; + public const string EnqueuedAt = "message.enqueued_at"; + public const string ReplayedAt = "message.replayed_at"; + public const string OriginNode = "message.origin_node"; +} diff --git a/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs new file mode 100644 index 000000000..61df4a898 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +/// +/// Stores the state of tracked queue jobs. Implementations must be safe for concurrent use. +/// +public interface IMessageExecutionStore +{ + /// Whether jobs and cancellation requests are shared across processes. Decorators must forward this capability. + bool IsShared { get; } + + /// + /// Creates or replaces a job's state. Called once at enqueue time with . + /// + Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default); + + /// Gets retained execution state, or null when absent or expired. + Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default); + + /// + /// Updates a job's status and optional fields. Implementations should apply the change atomically + /// relative to other status updates for the same job. Returns false for a missing job, a terminal + /// job, an older attempt, or a repeated start of an attempt already waiting for retry. + /// Starting an attempt resets progress and its message, initializes the heartbeat, and records workerId when supplied. + /// Administrative replay creates a new job identity; SetJobStateAsync is an explicit administrative replacement. + /// + Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null); + + /// Updates progress only for a live processing attempt; stale attempts are ignored. + Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null); + + /// + /// Signals that processing is still alive. The execution pipeline polls and heartbeats + /// independently of the broker lease supervisor. Only the current processing attempt may update it. + /// + Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null); + + /// Requests cooperative cancellation; returns false for missing or terminal executions. + Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); + + /// Checks whether cooperative cancellation was requested. + Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default); + + /// Removes retained history and cancellation state; does not remove broker work. + Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default); + + /// Adds an operational counter to the current hourly bucket. + Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default); + + /// Reads hourly operational counters within the requested time window. + Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default); + + /// Reads executions by status in descending creation order. + Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default); + + /// Counts retained executions with the requested status. + Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default); +} diff --git a/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs new file mode 100644 index 000000000..97394c4d2 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs @@ -0,0 +1,264 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Concurrent; + +namespace Foundatio.Messaging; + +/// +/// In-memory implementation of . +/// Suitable for development, testing, and single-node deployments. +/// Expired entries are lazily cleaned up on access. +/// +public sealed class InMemoryMessageExecutionStore : IMessageExecutionStore +{ + /// + public bool IsShared => false; + + private readonly object _gate = new(); + private readonly ConcurrentDictionary _jobs = new(); + private readonly ConcurrentDictionary _cancellations = new(); + private readonly ConcurrentDictionary> _counterBuckets = new(); + private readonly TimeProvider _timeProvider; + private int _accessCount; + + public InMemoryMessageExecutionStore(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + var now = _timeProvider.GetUtcNow(); + var expiresAt = expiry.HasValue ? now + expiry.Value : DateTimeOffset.MaxValue; + + _cancellations.TryRemove(state.JobId, out _); + _jobs[state.JobId] = new JobEntry(state, expiresAt, expiry); + + CleanupIfNeeded(); + + return Task.CompletedTask; + + } + } + + public Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (!_jobs.TryGetValue(jobId, out var entry)) + return Task.FromResult(null); + + if (!IsExpired(entry)) + return Task.FromResult(entry.State); + + // Remove expired entry on access + _jobs.TryRemove(jobId, out _); + _cancellations.TryRemove(jobId, out _); + + return Task.FromResult(null); + + } + } + + public Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (!_jobs.TryGetValue(jobId, out var entry) || IsExpired(entry) + || IsTerminal(entry.State.Status) || attempt < entry.State.Attempt + || (status == MessageExecutionStatus.Processing && attempt == entry.State.Attempt && entry.State.Status is MessageExecutionStatus.Processing or MessageExecutionStatus.RetryPending) + || (status == MessageExecutionStatus.EnqueueUnknown && entry.State.Status != MessageExecutionStatus.Queued)) + return Task.FromResult(false); + + var now = _timeProvider.GetUtcNow(); + var updated = entry.State with + { + Status = status, + StartedUtc = startedUtc ?? entry.State.StartedUtc, + CompletedUtc = IsTerminal(status) ? completedUtc ?? now : null, + ErrorMessage = errorMessage ?? (status == MessageExecutionStatus.Processing ? null : entry.State.ErrorMessage), + Progress = progress ?? (status == MessageExecutionStatus.Processing ? 0 : entry.State.Progress), + ProgressMessage = status == MessageExecutionStatus.Processing ? null : entry.State.ProgressMessage, + LastHeartbeatUtc = status == MessageExecutionStatus.Processing ? startedUtc ?? now : entry.State.LastHeartbeatUtc, + WorkerId = workerId ?? entry.State.WorkerId, + Attempt = attempt ?? entry.State.Attempt, + LastUpdatedUtc = now + }; + _jobs[jobId] = new JobEntry(updated, expiry.HasValue ? now + expiry.Value : entry.ExpiresAt, expiry ?? entry.Retention); + return Task.FromResult(true); + } + } + + public Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null) + => UpdateLiveJobAsync(jobId, expectedAttempt, expiry, state => state with + { + Progress = progress, + ProgressMessage = progressMessage, + LastUpdatedUtc = _timeProvider.GetUtcNow() + }, cancellationToken); + + public Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null) + => UpdateLiveJobAsync(jobId, expectedAttempt, expiry, state => state with + { + LastHeartbeatUtc = _timeProvider.GetUtcNow(), + LastUpdatedUtc = _timeProvider.GetUtcNow() + }, cancellationToken); + + private Task UpdateLiveJobAsync(string jobId, int? expectedAttempt, TimeSpan? expiry, Func update, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (_jobs.TryGetValue(jobId, out var entry) && !IsExpired(entry) && entry.State.Status == MessageExecutionStatus.Processing + && (!expectedAttempt.HasValue || expectedAttempt == entry.State.Attempt)) + _jobs[jobId] = new JobEntry(update(entry.State), (expiry ?? entry.Retention) is { } retention ? _timeProvider.GetUtcNow() + retention : entry.ExpiresAt, expiry ?? entry.Retention); + return Task.CompletedTask; + } + } + + private static bool IsTerminal(MessageExecutionStatus status) => status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled; + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + if (!_jobs.TryGetValue(jobId, out var entry) || IsExpired(entry)) + return Task.FromResult(false); + + // Only allow cancellation for non-terminal states + if (entry.State.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) + return Task.FromResult(false); + + _cancellations[jobId] = true; + return Task.FromResult(true); + + } + } + + public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + return Task.FromResult(_cancellations.ContainsKey(jobId)); + + } + } + + public Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_gate) + { + _jobs.TryRemove(jobId, out _); + _cancellations.TryRemove(jobId, out _); + return Task.CompletedTask; + + } + } + + public Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default) + { + var bucketKey = GetBucketKey(queueName, _timeProvider.GetUtcNow()); + var bucket = _counterBuckets.GetOrAdd(bucketKey, _ => new ConcurrentDictionary()); + bucket.AddOrUpdate(counterName, value, (_, existing) => existing + value); + return Task.CompletedTask; + } + + public Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var effectiveWindow = window ?? TimeSpan.FromHours(24); + var startHour = TruncateToHour(now - effectiveWindow); + var endHour = TruncateToHour(now); + + var totals = new Dictionary(); + var buckets = new List(); + + for (var hour = startHour; hour <= endHour; hour = hour.AddHours(1)) + { + var bucketKey = GetBucketKey(queueName, hour); + var counters = new Dictionary(); + + if (_counterBuckets.TryGetValue(bucketKey, out var bucket)) + { + foreach (var kvp in bucket) + { + counters[kvp.Key] = kvp.Value; + totals[kvp.Key] = totals.GetValueOrDefault(kvp.Key) + kvp.Value; + } + } + + buckets.Add(new MessageExecutionCounterBucket { Hour = hour, Counters = counters }); + } + + return Task.FromResult(new MessageExecutionCounters { Totals = totals, Buckets = buckets }); + } + + private static string GetBucketKey(string queueName, DateTimeOffset timestamp) + { + var hour = TruncateToHour(timestamp); + return $"{queueName}:{hour:yyyy-MM-ddTHH}"; + } + + private static DateTimeOffset TruncateToHour(DateTimeOffset timestamp) + => new(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0, TimeSpan.Zero); + + public Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var results = _jobs.Values + .Where(e => !IsExpired(e, now) + && string.Equals(e.State.QueueName, queueName, StringComparison.OrdinalIgnoreCase) + && e.State.Status == status) + .OrderByDescending(e => e.State.CreatedUtc) + .Skip(skip) + .Take(take) + .Select(e => e.State) + .ToList(); + + return Task.FromResult>(results); + } + + public Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var count = _jobs.Values.Count(e => !IsExpired(e, now) + && string.Equals(e.State.QueueName, queueName, StringComparison.OrdinalIgnoreCase) + && e.State.Status == status); + + return Task.FromResult((long)count); + } + + private bool IsExpired(JobEntry entry) => IsExpired(entry, _timeProvider.GetUtcNow()); + + private static bool IsExpired(JobEntry entry, DateTimeOffset now) => now >= entry.ExpiresAt; + + private void CleanupIfNeeded() + { + // Run cleanup every 100 writes to avoid accumulating expired entries + if (Interlocked.Increment(ref _accessCount) % 100 != 0) + return; + + var now = _timeProvider.GetUtcNow(); + foreach (var kvp in _jobs) + { + if (now >= kvp.Value.ExpiresAt) + { + _jobs.TryRemove(kvp.Key, out _); + _cancellations.TryRemove(kvp.Key, out _); + } + } + } + + private sealed record JobEntry(MessageExecutionState State, DateTimeOffset ExpiresAt, TimeSpan? Retention); +} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs new file mode 100644 index 000000000..83b5c8508 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +/// +/// Counter statistics for a queue, including totals and per-hour buckets for sparkline rendering. +/// +public sealed record MessageExecutionCounters +{ + /// + /// Sum of all counters across the requested time window. + /// Keys are counter names (e.g., "processed", "failed", "dead_lettered"). + /// + public required IReadOnlyDictionary Totals { get; init; } + + /// + /// Per-hour counter values ordered oldest to newest, suitable for sparkline rendering. + /// Each bucket represents one UTC hour. + /// + public required IReadOnlyList Buckets { get; init; } +} + +/// +/// Counter values for a single hour. +/// +public sealed record MessageExecutionCounterBucket +{ + /// + /// The UTC hour this bucket represents (truncated to the hour). + /// + public required DateTimeOffset Hour { get; init; } + + /// + /// Counter values for this hour. Keys are counter names. + /// + public required IReadOnlyDictionary Counters { get; init; } +} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs new file mode 100644 index 000000000..dde1942d5 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs @@ -0,0 +1,32 @@ +using System; + +namespace Foundatio.Messaging; + +/// Optional execution tracking for broker-delivered work. This never schedules a second job-store worker. +public sealed record MessageExecutionOptions +{ + /// Logical queue represented by this execution. + public required string QueueName { get; init; } + /// Application message type used for diagnostics. + public Type? MessageType { get; init; } + /// Total delivery budget, including the first attempt; negative means unlimited. + public int MaxAttempts { get; init; } = 3; + /// Delay for a failed attempt, using its one-based attempt number. + public Func RetryBackoff { get; init; } = RetryPolicy.DefaultBackoff; + /// Delivery duration used by explicit progress heartbeats. + public TimeSpan VisibilityTimeout { get; init; } = TimeSpan.FromMinutes(1); + /// Settle returned outcomes automatically; explicit settlement takes precedence. + public bool AutoComplete { get; init; } = true; + /// Read and update broker execution history in the supplied store. + public bool TrackProgress { get; init; } + /// Header containing the producer-created execution identifier. + public string ExecutionIdHeader { get; init; } = "message.execution.id"; + /// Retention refreshed by execution state updates. + public TimeSpan StateRetention { get; init; } = TimeSpan.FromHours(24); + /// Interval between cooperative cancellation checks and execution heartbeats. + public TimeSpan CancellationPollInterval { get; init; } = TimeSpan.FromSeconds(5); + /// Process identity recorded when an attempt starts. + public string WorkerId { get; init; } = $"{Environment.MachineName}:{Environment.ProcessId}"; + /// Receives success, failed-attempt, and dead-letter measurements. Must not throw. + public Action? OnProcessed { get; init; } +} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs new file mode 100644 index 000000000..3fae644fc --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs @@ -0,0 +1,257 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Messaging; + +/// +/// Executes a broker delivery with optional progress, cancellation, history, and per-attempt state fencing. +/// The message bus remains the sole owner of receiving and delivery leases; this pipeline creates no runnable jobs. +/// +public sealed class MessageExecutionPipeline +{ + private static readonly TimeSpan OperationTimeout = TimeSpan.FromSeconds(30); + private readonly MessageExecutionOptions _options; + private readonly IMessageExecutionStore? _store; + private readonly TimeProvider _time; + private readonly ILogger _logger; + + public MessageExecutionPipeline(MessageExecutionOptions options, IMessageExecutionStore? store = null, TimeProvider? timeProvider = null, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(options.QueueName); + ArgumentOutOfRangeException.ThrowIfEqual(options.MaxAttempts, 0); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.CancellationPollInterval, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.StateRetention, TimeSpan.Zero); + if (options.TrackProgress && store is null) + throw new ArgumentException("Execution tracking requires an IMessageExecutionStore.", nameof(store)); + _options = options; + _store = store; + _time = timeProvider ?? TimeProvider.System; + _logger = logger ?? NullLogger.Instance; + } + + /// Runs application processing and persists only confirmed settlement outcomes. + public async Task ProcessAsync(IMessageContext delivery, Func> handler, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(delivery); + ArgumentNullException.ThrowIfNull(handler); + string? jobId = _options.TrackProgress && _store is not null ? delivery.Headers.GetValueOrDefault(_options.ExecutionIdHeader) : null; + if (_options.MaxAttempts >= 0 && delivery.Attempts > _options.MaxAttempts) + { + await DeadLetterAsync(delivery, jobId, $"Exceeded max attempts ({_options.MaxAttempts})").AnyContext(); + return; + } + + using var processing = CancellationTokenSource.CreateLinkedTokenSource(delivery.CancellationToken, cancellationToken); + var token = processing.Token; + Task? poll = null; + long started = Stopwatch.GetTimestamp(); + var context = new MessageProcessingContext + { + Body = delivery.Body, + QueueName = _options.QueueName, + MessageId = delivery.BrokerMessageId, + MessageType = _options.MessageType, + DequeueCount = delivery.Attempts, + MaxAttempts = _options.MaxAttempts, + VisibilityTimeout = _options.VisibilityTimeout, + EnqueuedAt = delivery.EnqueuedUtc ?? _time.GetUtcNow(), + JobId = jobId, + Headers = delivery.Headers, + OnCancelProcessing = processing.Cancel, + OnRenewTimeout = (duration, ct) => delivery.RenewLockAsync(duration, ct), + OnComplete = ct => RunAsync(delivery.CompleteAsync, ct), + OnAbandon = (delay, ct) => RunAsync(t => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = delay }, t), ct), + OnReportProgress = async ct => + { + await delivery.RenewLockAsync(_options.VisibilityTimeout, ct).AnyContext(); + if (jobId is not null) + await UpdateAsync(t => _store!.HeartbeatAsync(jobId, t, delivery.Attempts, _options.StateRetention), ct).AnyContext(); + }, + OnReportDetailedProgress = jobId is null ? null : async (percent, message, ct) => + { + if (await IsCancelledAsync(jobId, ct).AnyContext()) + throw new OperationCanceledException("Job cancellation was requested."); + await UpdateAsync(t => _store!.UpdateJobProgressAsync(jobId, Math.Clamp(percent, 0, 100), message, _options.StateRetention, t, delivery.Attempts), ct).AnyContext(); + await UpdateAsync(t => _store!.HeartbeatAsync(jobId, t, delivery.Attempts, _options.StateRetention), ct).AnyContext(); + } + }; + try + { + if (jobId is not null) + { + if (await IsCancelledAsync(jobId, token).AnyContext()) + { + if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) + await StatusAsync(jobId, MessageExecutionStatus.Cancelled, delivery.Attempts).AnyContext(); + return; + } + poll = PollCancellationAsync(jobId, delivery.Attempts, processing); + bool accepted = await RunAsync(ct => _store!.UpdateJobStatusAsync(jobId, MessageExecutionStatus.Processing, + startedUtc: _time.GetUtcNow(), attempt: delivery.Attempts, expiry: _options.StateRetention, cancellationToken: ct, workerId: _options.WorkerId), token).AnyContext(); + if (!accepted) + { + var state = await RunAsync(ct => _store!.GetJobStateAsync(jobId, ct), token).AnyContext(); + if (state?.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) + await SettleAsync(delivery.CompleteAsync, delivery).AnyContext(); + else if (state is not null) + await RetryAsync(delivery).AnyContext(); + // Tracking retention must not become a second delivery scheduler. Expired history + // does not prevent broker-owned work from running; state updates become no-ops. + if (state is not null) return; + _logger.LogWarning("Execution history {JobId} expired before delivery; processing continues without retained history", jobId); + } + } + + var outcome = await handler(context, token).AnyContext(); + if (!context.IsCompleted && !context.IsAbandoned) + { + if (outcome.Kind == MessageOutcomeKind.Retry) + { + await FailureAsync(delivery, jobId, outcome.Reason ?? "Processing failed", Stopwatch.GetElapsedTime(started)).AnyContext(); + return; + } + if (outcome.Kind == MessageOutcomeKind.DeadLetter) + { + await DeadLetterAsync(delivery, jobId, outcome.Reason ?? "Processing rejected").AnyContext(); + return; + } + } + if (context.IsAbandoned) return; + token.ThrowIfCancellationRequested(); + if (_options.AutoComplete && !context.IsCompleted && outcome.Kind != MessageOutcomeKind.Unsettled) + await SettleAsync(context.CompleteAsync, delivery).AnyContext(); + if (!context.IsCompleted) + await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, "Handler finished without confirmed acknowledgment; delivery may recur.").AnyContext(); + } + catch (OperationCanceledException) when (delivery.IsLeaseLost) { } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + if (!context.IsCompleted && !context.IsAbandoned) + { + await SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.Zero }, ct), delivery).AnyContext(); + await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts).AnyContext(); + } + } + catch (OperationCanceledException) + { + if (context.IsCompleted || context.IsAbandoned) return; + if (jobId is not null && await IsCancelledAsync(jobId, CancellationToken.None).AnyContext()) + { + if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) + await StatusAsync(jobId, MessageExecutionStatus.Cancelled, delivery.Attempts).AnyContext(); + } + else + await FailureAsync(delivery, jobId, "Processing was cancelled", Stopwatch.GetElapsedTime(started)).AnyContext(); + } + catch (Exception exception) + { + _logger.LogError(exception, "Processing failed for {MessageId} at {Queue} on attempt {Attempt}", delivery.Id, _options.QueueName, delivery.Attempts); + if (!delivery.IsLeaseLost && !context.IsCompleted && !context.IsAbandoned) + await FailureAsync(delivery, jobId, exception.Message, Stopwatch.GetElapsedTime(started)).AnyContext(); + } + finally + { + await processing.CancelAsync().AnyContext(); + if (poll is not null) await poll.AnyContext(); + if (context.IsCompleted) + { + _options.OnProcessed?.Invoke(MessageOutcomeKind.Success, Stopwatch.GetElapsedTime(started)); + if (jobId is not null) + await UpdateAsync(ct => _store!.UpdateJobStatusAsync(jobId, MessageExecutionStatus.Completed, attempt: delivery.Attempts, + completedUtc: _time.GetUtcNow(), progress: 100, expiry: _options.StateRetention, cancellationToken: ct)).AnyContext(); + await CounterAsync("processed").AnyContext(); + } + else if (context.IsAbandoned) + await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts).AnyContext(); + } + } + + private async Task FailureAsync(IMessageContext delivery, string? jobId, string reason, TimeSpan elapsed) + { + if (_options.AutoComplete && _options.MaxAttempts > 0 && delivery.Attempts >= _options.MaxAttempts) + await DeadLetterAsync(delivery, jobId, reason).AnyContext(); + else + { + if (_options.AutoComplete) await RetryAsync(delivery).AnyContext(); + await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, reason).AnyContext(); + } + _options.OnProcessed?.Invoke(MessageOutcomeKind.Retry, elapsed); + await CounterAsync("failed").AnyContext(); + } + + private async Task DeadLetterAsync(IMessageContext delivery, string? jobId, string reason) + { + if (!await SettleAsync(ct => MessageOutcome.DeadLetter(reason).SettleFailureAsync(delivery, _options.MaxAttempts, _options.RetryBackoff, ct), delivery).AnyContext()) + { + await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, reason).AnyContext(); + return; + } + _options.OnProcessed?.Invoke(MessageOutcomeKind.DeadLetter, TimeSpan.Zero); + await StatusAsync(jobId, MessageExecutionStatus.Failed, delivery.Attempts, reason).AnyContext(); + await CounterAsync("dead_lettered").AnyContext(); + } + + private Task RetryAsync(IMessageContext delivery) => SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = _options.RetryBackoff(delivery.Attempts) }, ct), delivery); + private Task IsCancelledAsync(string jobId, CancellationToken token) => RunAsync(ct => _store!.IsCancellationRequestedAsync(jobId, ct), token); + private Task CounterAsync(string name) => _store is null ? Task.CompletedTask : UpdateAsync(ct => _store.IncrementCounterAsync(_options.QueueName, name, 1, ct)); + private Task StatusAsync(string? jobId, MessageExecutionStatus status, int attempt, string? error = null) + => jobId is null ? Task.CompletedTask : UpdateAsync(ct => _store!.UpdateJobStatusAsync(jobId, status, attempt: attempt, + completedUtc: status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled ? _time.GetUtcNow() : null, + errorMessage: error, expiry: _options.StateRetention, cancellationToken: ct)); + + private async Task PollCancellationAsync(string jobId, int attempt, CancellationTokenSource processing) + { + var token = processing.Token; + while (!token.IsCancellationRequested) + { + try + { + await Task.Delay(_options.CancellationPollInterval, _time, token).AnyContext(); + if (await IsCancelledAsync(jobId, token).AnyContext()) + { + await processing.CancelAsync().AnyContext(); + return; + } + await UpdateAsync(ct => _store!.HeartbeatAsync(jobId, ct, attempt, _options.StateRetention), token).AnyContext(); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) { return; } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to poll execution {JobId}; retrying", jobId); } + } + } + + private async Task SettleAsync(Func operation, IMessageContext delivery) + { + try { await RunAsync(operation).AnyContext(); return true; } + catch (Exception exception) + { + _logger.LogError(exception, "Settlement was not confirmed for {MessageId} at {Queue}; delivery may recur", delivery.Id, _options.QueueName); + return false; + } + } + private Task UpdateAsync(Func> operation, CancellationToken cancellationToken = default) + => UpdateAsync(async ct => { _ = await operation(ct).AnyContext(); }, cancellationToken); + + private async Task UpdateAsync(Func operation, CancellationToken cancellationToken = default) + { + try { await RunAsync(operation, cancellationToken).AnyContext(); } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to update execution state at {Queue}", _options.QueueName); } + } + private async Task RunAsync(Func operation, CancellationToken cancellationToken = default) + { + using var timeout = new CancellationTokenSource(OperationTimeout, _time); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + await operation(linked.Token).WaitAsync(linked.Token).AnyContext(); + } + private async Task RunAsync(Func> operation, CancellationToken cancellationToken = default) + { + using var timeout = new CancellationTokenSource(OperationTimeout, _time); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + return await operation(linked.Token).WaitAsync(linked.Token).AnyContext(); + } +} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs new file mode 100644 index 000000000..25b48c679 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +/// +/// Tracked state of a queued job. Immutable; use with expressions to derive updates. +/// +public sealed record MessageExecutionState +{ + public required string JobId { get; init; } + public required string QueueName { get; init; } + public string MessageType { get; init; } = string.Empty; + public MessageExecutionStatus Status { get; init; } = MessageExecutionStatus.Queued; + public int Progress { get; init; } + public string? ProgressMessage { get; init; } + public DateTimeOffset CreatedUtc { get; init; } + public DateTimeOffset? StartedUtc { get; init; } + public DateTimeOffset? CompletedUtc { get; init; } + public int Attempt { get; init; } + /// Identity of the worker process that started the current or most recent attempt. + public string? WorkerId { get; init; } + public string? ErrorMessage { get; init; } + public DateTimeOffset LastUpdatedUtc { get; init; } + + /// + /// When the worker last signalled that the job is alive, through a visibility renewal or a progress + /// report. A processing job whose heartbeat is stale has most likely lost its worker. + /// + public DateTimeOffset? LastHeartbeatUtc { get; init; } + + /// + /// Caller-supplied metadata captured by the producer at enqueue time, + /// such as a tenant or user id, so stores can index and display jobs by them. + /// + public IReadOnlyDictionary? Metadata { get; init; } +} + +public enum MessageExecutionStatus +{ + Queued = 0, + Processing = 1, + Completed = 2, + Failed = 3, + Cancelled = 4, + /// The current attempt did not settle successfully and may be delivered again. + RetryPending = 5, + /// The transport call failed without confirming whether the message was accepted. + EnqueueUnknown = 6 +} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs new file mode 100644 index 000000000..f37e5beee --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Messaging; + +public static class MessageExecutionStoreExtensions +{ + /// Observes a tracked execution until it reaches a terminal state. Cancellation stops observation, not the queued work. + public static async Task WaitForCompletionAsync(this IMessageExecutionStore store, string id, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentException.ThrowIfNullOrWhiteSpace(id); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var state = await store.GetJobStateAsync(id, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Execution '{id}' was not found or has expired."); + if (state.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) + return state; + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Foundatio/Messaging/Tracking/MessageProcessingContext.cs b/src/Foundatio/Messaging/Tracking/MessageProcessingContext.cs new file mode 100644 index 000000000..367aee393 --- /dev/null +++ b/src/Foundatio/Messaging/Tracking/MessageProcessingContext.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +namespace Foundatio.Messaging; + +/// Execution progress, cancellation and explicit settlement for a broker-delivered message. +public class MessageProcessingContext +{ + private readonly SemaphoreSlim _settlementGate = new(1); + private int _settlement; + + /// The serialized application payload. + public ReadOnlyMemory Body { get; init; } + + internal Action? OnSettled { get; init; } + internal Action? OnCancelProcessing { get; init; } + + /// Cancel cooperative processing after losing an application resource lock. + public void CancelProcessing() => OnCancelProcessing?.Invoke(); + + /// + /// The name of the queue this message was received from. + /// + public string QueueName { get; init; } = string.Empty; + + /// + /// The transport-assigned id of the message being processed. + /// + public string MessageId { get; init; } = string.Empty; + + /// + /// The visibility timeout the worker requested for this message. + /// + public TimeSpan VisibilityTimeout { get; init; } + + /// + /// The message type being processed. + /// + public Type? MessageType { get; init; } + + /// + /// The number of times this message has been dequeued (including the current attempt). + /// Useful for detecting poison messages or implementing backoff strategies. + /// + public int DequeueCount { get; init; } + + /// + /// The maximum number of attempts configured for this queue. + /// After this many attempts, the message will be dead-lettered. + /// + public int MaxAttempts { get; init; } + + /// + /// When the message was originally enqueued. + /// + public DateTimeOffset EnqueuedAt { get; init; } + + /// + /// The unique job identifier for progress tracking, or null if tracking is not enabled. + /// + public string? JobId { get; init; } + + /// Message headers, including correlation, propagated context, and replay lineage. + public IReadOnlyDictionary Headers { get; init; } = new Dictionary(); + + /// + /// Delegate invoked by to signal that the handler + /// is still actively working. This acts as a heartbeat keep-alive that extends the + /// message visibility by the configured timeout. Set by the worker infrastructure. + /// + internal Func? OnReportProgress { get; init; } + + /// + /// Delegate invoked by + /// to update progress percentage and message in the state store. + /// Set by the worker infrastructure when progress tracking is enabled. + /// + internal Func? OnReportDetailedProgress { get; init; } + + /// + /// Delegate invoked by to extend the message lock + /// or visibility timeout by a specific duration. Set by the worker infrastructure. + /// + internal Func? OnRenewTimeout { get; init; } + + /// + /// Delegate invoked by to remove the message from the queue. + /// Set by the worker infrastructure. + /// + internal Func? OnComplete { get; init; } + + /// + /// Delegate invoked by to + /// return the message to the queue for redelivery. Set by the worker infrastructure. + /// + internal Func? OnAbandon { get; init; } + + /// + /// Indicates whether the handler explicitly completed the message via . + /// When true, the worker infrastructure will skip automatic completion. + /// + public bool IsCompleted => Volatile.Read(ref _settlement) == 1; + + /// + /// Indicates whether the handler explicitly abandoned the message via + /// or . + /// When true, the worker infrastructure will skip automatic abandonment. + /// + public bool IsAbandoned => Volatile.Read(ref _settlement) == 2; + + /// + /// Reports that the handler is still actively processing the message. + /// For transports that support it, this extends the visibility timeout + /// by the configured default duration, preventing the message from being + /// redelivered during long-running operations. + /// + public Task ReportProgressAsync(CancellationToken cancellationToken = default) + => OnReportProgress?.Invoke(cancellationToken) ?? Task.CompletedTask; + + /// + /// Reports progress with a percentage and optional message. + /// When progress tracking is enabled, this updates the job state store + /// and checks for cancellation. If cancellation has been requested, + /// an is thrown. + /// Also acts as a heartbeat to extend the message visibility timeout. + /// + /// Progress percentage (0–100). + /// Optional description of current work. + /// A cancellation token. + public async Task ReportProgressAsync(int progressPercent, string? message = null, CancellationToken cancellationToken = default) + { + // Always renew the visibility timeout as a heartbeat + if (OnReportProgress is not null) + await OnReportProgress(cancellationToken).ConfigureAwait(false); + + // Update state store and check for cancellation + if (OnReportDetailedProgress is not null) + await OnReportDetailedProgress(progressPercent, message, cancellationToken).ConfigureAwait(false); + } + + /// + /// Extends the message lock or visibility timeout by the specified duration. + /// Use this for long-running handlers to prevent the message from being + /// redelivered to another consumer. + /// + public Task RenewTimeoutAsync(TimeSpan extension, CancellationToken cancellationToken = default) + => OnRenewTimeout?.Invoke(extension, cancellationToken) ?? Task.CompletedTask; + + /// + /// Completes the message, removing it from the queue permanently. + /// Use this when AutoComplete is disabled and the handler has finished + /// processing successfully. If AutoComplete is enabled, the worker + /// infrastructure will skip its own completion when this has been called. + /// + public Task CompleteAsync(CancellationToken cancellationToken = default) + => SettleAsync(1, OnComplete, cancellationToken); + + private async Task SettleAsync(int outcome, Func? operation, CancellationToken cancellationToken) + { + await _settlementGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_settlement == outcome) + return; + if (_settlement != 0) + throw new InvalidOperationException("This delivery has already been settled with a different outcome."); + if (operation is not null) + await operation(cancellationToken).ConfigureAwait(false); + Volatile.Write(ref _settlement, outcome); + OnSettled?.Invoke(); + } + finally + { + _settlementGate.Release(); + } + } + + /// + /// Abandons the message so it becomes immediately visible for redelivery. + /// Use this when AutoComplete is disabled and the handler cannot + /// process the message successfully. + /// + public Task AbandonAsync(CancellationToken cancellationToken = default) + => AbandonAsync(TimeSpan.Zero, cancellationToken); + + /// + /// Abandons the message so it becomes visible for redelivery after the specified delay. + /// Use this when AutoComplete is disabled and the handler wants to retry + /// the message after a backoff period. + /// + /// How long before the message becomes visible again. Use for immediate redelivery. + /// A cancellation token. + public Task AbandonAsync(TimeSpan delay, CancellationToken cancellationToken = default) + => SettleAsync(2, ct => OnAbandon?.Invoke(delay, ct) ?? Task.CompletedTask, cancellationToken); +} diff --git a/tests/Foundatio.Aws.Tests/AwsEnvelopeTests.cs b/tests/Foundatio.Aws.Tests/AwsEnvelopeTests.cs index 325db4bf4..c42a52454 100644 --- a/tests/Foundatio.Aws.Tests/AwsEnvelopeTests.cs +++ b/tests/Foundatio.Aws.Tests/AwsEnvelopeTests.cs @@ -16,14 +16,14 @@ namespace Foundatio.Aws.Tests; public class AwsEnvelopeTests { [Fact] - public async Task ReceiveAsync_SystemAttributes_RequestsOnlyDeliveryCount() + public async Task ReceiveAsync_SystemAttributes_RequestsDeliveryCountAndEnqueueTime() { var sqs = new Mock(); sqs.Setup(s => s.GetQueueUrlAsync(It.IsAny(), It.IsAny())).ReturnsAsync(new GetQueueUrlResponse { QueueUrl = "http://test/queue" }); sqs.Setup(s => s.ReceiveMessageAsync(It.IsAny(), It.IsAny())) .ReturnsAsync((ReceiveMessageRequest request, CancellationToken _) => { - Assert.Equal(["ApproximateReceiveCount"], request.MessageSystemAttributeNames); + Assert.Equal(["ApproximateReceiveCount", "SentTimestamp"], request.MessageSystemAttributeNames); Assert.Equal(["All"], request.MessageAttributeNames); return new ReceiveMessageResponse { diff --git a/tests/Foundatio.Aws.Tests/AwsMessageAdministrationTests.cs b/tests/Foundatio.Aws.Tests/AwsMessageAdministrationTests.cs new file mode 100644 index 000000000..6e5d91da3 --- /dev/null +++ b/tests/Foundatio.Aws.Tests/AwsMessageAdministrationTests.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Aws.Tests; + +public class AwsMessageAdministrationTests +{ + [Fact] + public async Task ReplayBeyondFirstReceiveBatch_ResetsRetryStateAndPreservesOtherDeadLetters() + { + string? connection = Environment.GetEnvironmentVariable("FOUNDATIO_AWS_CONNECTION_STRING"); + Assert.SkipWhen(String.IsNullOrEmpty(connection), "FOUNDATIO_AWS_CONNECTION_STRING not set."); + var options = AwsMessageTransportOptions.FromConnectionString(connection); + options.ResourcePrefix = $"admin-test-{Guid.NewGuid():N}-"; + await using var transport = new AwsMessageTransport(options); + var token = TestContext.Current.CancellationToken; + var source = DestinationAddress.ForQueue("work"); + var dead = DestinationAddress.ForQueue("work.deadletter"); + await transport.EnsureAsync([new() { Address = source }, new() { Address = dead }], token); + try + { + var messages = Enumerable.Range(0, 15).Select(index => new TransportMessage + { + Body = "{}"u8.ToArray(), + ContentType = "application/json", + MessageId = $"original-{index}", + Headers = MessageHeaders.Create(new Dictionary + { + [KnownHeaders.Attempts] = "9", + [KnownHeaders.DeadLetterReason] = "Original failure" + }) + }).ToArray(); + (await transport.SendAsync(dead, messages, new(), token)).EnsureAccepted(messages.Length); + var administration = new MessageAdministration(transport); + var snapshot = await administration.PeekDeadLettersAsync(source, 15, token); + Assert.Equal(15, snapshot.Count); + var selected = snapshot[^1]; + Assert.True(await administration.ReplayDeadLetterAsync(source, selected.Id, cancellationToken: token)); + var replay = Assert.Single(await transport.ReceiveAsync(source, new() { MaxMessages = 1, MaxWaitTime = TimeSpan.FromSeconds(2) }, token)); + Assert.Equal(selected.ApplicationMessageId, replay.ApplicationMessageId); + Assert.NotNull(replay.EnqueuedUtc); + Assert.False(replay.Headers.ContainsKey(KnownHeaders.Attempts)); + Assert.False(replay.Headers.ContainsKey(KnownHeaders.DeadLetterReason)); + await transport.CompleteAsync(replay, token); + var remaining = await administration.PeekDeadLettersAsync(source, 15, token); + Assert.Equal(14, remaining.Count); + Assert.DoesNotContain(remaining, entry => entry.Id == selected.Id); + } + finally + { + await transport.DeleteAsync(source, token); + await transport.DeleteAsync(dead, token); + } + } +} diff --git a/tests/Foundatio.Aws.Tests/AwsNodeSubscriptionTests.cs b/tests/Foundatio.Aws.Tests/AwsNodeSubscriptionTests.cs new file mode 100644 index 000000000..543580b90 --- /dev/null +++ b/tests/Foundatio.Aws.Tests/AwsNodeSubscriptionTests.cs @@ -0,0 +1,40 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Aws.Tests; + +public class AwsNodeSubscriptionTests +{ + [Fact] + public async Task Nodes_ReceiveIndependentCopies_AndDisposeTheirResources() + { + string? connection = Environment.GetEnvironmentVariable("FOUNDATIO_AWS_CONNECTION_STRING"); + Assert.SkipWhen(String.IsNullOrEmpty(connection), "FOUNDATIO_AWS_CONNECTION_STRING not set."); + var options = AwsMessageTransportOptions.FromConnectionString(connection); + options.ResourcePrefix = $"node-test-{Guid.NewGuid():N}-"; + await using var transport = new AwsMessageTransport(options); + await using var bus = new MessageBus(transport, new MessageBusOptions { OwnsTransport = false }); + var token = TestContext.Current.CancellationToken; + var firstReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var secondReceived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = await bus.SubscribeNodeAsync((_, _) => { firstReceived.TrySetResult(); return Task.CompletedTask; }, new() { Topic = "events", NodeId = "first" }, token); + var second = await bus.SubscribeNodeAsync((_, _) => { secondReceived.TrySetResult(); return Task.CompletedTask; }, new() { Topic = "events", NodeId = "second" }, token); + try + { + Assert.NotEqual(first.Source, second.Source); + await bus.PublishAsync("changed", new MessagePublishOptions { Topic = "events" }, token); + await Task.WhenAll(firstReceived.Task, secondReceived.Task).WaitAsync(TimeSpan.FromSeconds(15), token); + } + finally + { + await first.DisposeAsync(); + await second.DisposeAsync(); + await transport.DeleteAsync(DestinationAddress.ForTopic("events"), token); + } + Assert.False(await transport.ExistsAsync(first.Source, token)); + Assert.False(await transport.ExistsAsync(second.Source, token)); + } +} diff --git a/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs b/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs new file mode 100644 index 000000000..e481e27b6 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs @@ -0,0 +1,12 @@ +using System; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; + +namespace Foundatio.Redis.Tests; + +public sealed class RedisMessageExecutionStoreTests : MessageExecutionStoreConformanceTests +{ + protected override IMessageExecutionStore? CreateStore() => RedisTestConnection.Multiplexer is { } connection + ? new RedisMessageExecutionStore(connection, new RedisMessageExecutionStoreOptions { KeyPrefix = $"native-execution-test:{Guid.NewGuid():N}" }) + : null; +} diff --git a/tests/Foundatio.Redis.Tests/RedisResourceLockTests.cs b/tests/Foundatio.Redis.Tests/RedisResourceLockTests.cs new file mode 100644 index 000000000..3c0b242c3 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisResourceLockTests.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Lock; +using Xunit; + +namespace Foundatio.Redis.Tests; + +public class RedisResourceLockTests +{ + [Fact] + public async Task ExpiredOwner_CannotReleaseOrRenewANewOwnersLock() + { + var connection = RedisTestConnection.Multiplexer; + Assert.SkipWhen(connection is null, "FOUNDATIO_REDIS_CONNECTION_STRING not set."); + var provider = new RedisLockProvider(connection!, $"native-lock:{Guid.NewGuid():N}:"); + var token = TestContext.Current.CancellationToken; + await using var first = await provider.AcquireAsync("report", TimeSpan.FromMilliseconds(100), cancellationToken: token); + await Task.Delay(150, token); + await using var second = await provider.AcquireAsync("report", TimeSpan.FromSeconds(10), cancellationToken: token); + await Assert.ThrowsAsync(() => first.RenewAsync()); + await first.ReleaseAsync(); + Assert.True(await provider.IsLockedAsync("report")); + await second.RenewAsync(TimeSpan.FromSeconds(20)); + Assert.Equal(1, second.RenewalCount); + await second.ReleaseAsync(); + Assert.False(await provider.IsLockedAsync("report")); + } + + [Fact] + public async Task ContendedAcquisition_StopsWaitingWhenCancelled() + { + var connection = RedisTestConnection.Multiplexer; + Assert.SkipWhen(connection is null, "FOUNDATIO_REDIS_CONNECTION_STRING not set."); + var provider = new RedisLockProvider(connection!, $"native-lock:{Guid.NewGuid():N}:"); + await using var held = await provider.AcquireAsync("report", cancellationToken: TestContext.Current.CancellationToken); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + timeout.CancelAfter(TimeSpan.FromMilliseconds(100)); + Assert.Null(await provider.TryAcquireAsync("report", cancellationToken: timeout.Token)); + Assert.True(await provider.IsLockedAsync("report")); + } +} diff --git a/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs b/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs new file mode 100644 index 000000000..1053358b0 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Moq; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class MessageEndpointPolicyTests +{ + [Fact] + public async Task ExpiredTrackingHistory_DoesNotPreventBrokerWorkFromRunning() + { + var store = new InMemoryMessageExecutionStore(); + var delivery = new Mock(); + delivery.SetupGet(value => value.Headers).Returns(MessageHeaders.Create(new Dictionary { [ExecutionHeaders.ExecutionId] = "expired" })); + delivery.SetupGet(value => value.Attempts).Returns(1); + delivery.Setup(value => value.CompleteAsync(It.IsAny())).Returns(Task.CompletedTask); + var pipeline = new MessageExecutionPipeline(new MessageExecutionOptions { QueueName = "exports", TrackProgress = true }, store); + bool invoked = false; + await pipeline.ProcessAsync(delivery.Object, (_, _) => + { + invoked = true; + return ValueTask.FromResult(MessageOutcome.Success); + }, TestContext.Current.CancellationToken); + Assert.True(invoked); + delivery.Verify(value => value.CompleteAsync(It.IsAny()), Times.Once); + Assert.Null(await store.GetJobStateAsync("expired", TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task ManualRenewal_ExtendsTheSupervisedDeadlineWhenAutoRenewIsDisabled() + { + var time = new FakeTimeProvider(); + await using var transport = new InMemoryMessageTransport(time); + await using var bus = new MessageBus(transport, new MessageBusOptions { TimeProvider = time, OwnsTransport = false }); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var consumer = await bus.ConsumeAsync(async (message, token) => + { + entered.TrySetResult(message); + try { await Task.Delay(Timeout.InfiniteTimeSpan, token); } + catch (OperationCanceledException) when (token.IsCancellationRequested) { cancelled.TrySetResult(); } + }, new MessageConsumerOptions { Destination = "manual-lease", AutoRenewLock = false, VisibilityTimeout = TimeSpan.FromSeconds(30) }, TestContext.Current.CancellationToken); + await bus.SendAsync("work", new MessageSendOptions { Destination = "manual-lease" }, TestContext.Current.CancellationToken); + var delivery = await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + time.Advance(TimeSpan.FromSeconds(10)); + await delivery.RenewLockAsync(TimeSpan.FromSeconds(60), TestContext.Current.CancellationToken); + time.Advance(TimeSpan.FromSeconds(25)); + await Task.Delay(50, TestContext.Current.CancellationToken); + Assert.False(delivery.CancellationToken.IsCancellationRequested); + time.Advance(TimeSpan.FromSeconds(40)); + await cancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + Assert.True(delivery.IsLeaseLost); + } + + [Fact] + public async Task FailedAcknowledgment_DoesNotPersistCompletion() + { + var store = new InMemoryMessageExecutionStore(); + await store.SetJobStateAsync(new MessageExecutionState + { + JobId = "ack-failure", + QueueName = "exports", + MessageType = "Export", + Status = MessageExecutionStatus.Queued, + CreatedUtc = DateTimeOffset.UtcNow, + LastUpdatedUtc = DateTimeOffset.UtcNow + }, cancellationToken: TestContext.Current.CancellationToken); + var delivery = new Mock(); + delivery.SetupGet(value => value.Headers).Returns(MessageHeaders.Create(new Dictionary { [ExecutionHeaders.ExecutionId] = "ack-failure" })); + delivery.SetupGet(value => value.Attempts).Returns(1); + delivery.Setup(value => value.CompleteAsync(It.IsAny())).ThrowsAsync(new TimeoutException("Unknown broker acknowledgment")); + var pipeline = new MessageExecutionPipeline(new MessageExecutionOptions { QueueName = "exports", TrackProgress = true }, store); + await pipeline.ProcessAsync(delivery.Object, (_, _) => ValueTask.FromResult(MessageOutcome.Success), TestContext.Current.CancellationToken); + Assert.Equal(MessageExecutionStatus.RetryPending, (await store.GetJobStateAsync("ack-failure", TestContext.Current.CancellationToken))!.Status); + } + + [Fact] + public async Task ConsumeAsync_EndpointPolicy_ControlsVisibilityAndReceiveCapacity() + { + var token = TestContext.Current.CancellationToken; + var transport = new Mock(); + transport.As(); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + transport.Setup(t => t.ReceiveAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) => + { + Assert.Equal("exports", source.Name); + Assert.InRange(request.MaxMessages, 1, 2); + Assert.Equal(TimeSpan.FromSeconds(12), visibility); + received.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, ct); + return Array.Empty(); + }); + await using var bus = new MessageBus(transport.Object); + await using var consumer = await bus.ConsumeAsync((_, _) => Task.CompletedTask, new MessageConsumerOptions + { + Destination = "exports", + MaxConcurrency = 5, + PrefetchCount = 2, + VisibilityTimeout = TimeSpan.FromSeconds(12) + }, token); + await received.Task.WaitAsync(TimeSpan.FromSeconds(5), token); + } + + [Fact] + public async Task ConsumeAsync_GracefulShutdown_CompletesAdmittedWorkBeforeReturning() + { + var token = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var finish = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken handlerToken = default; + var consumer = await bus.ConsumeAsync(async (_, ct) => + { + handlerToken = ct; + started.TrySetResult(); + await finish.Task.WaitAsync(ct); + }, new MessageConsumerOptions { Destination = "exports", ShutdownTimeout = TimeSpan.FromSeconds(5) }, token); + try + { + await bus.SendAsync(new Work(), new MessageSendOptions { Destination = "exports" }, token); + await started.Task.WaitAsync(TimeSpan.FromSeconds(5), token); + var stopping = consumer.DisposeAsync().AsTask(); + Assert.False(handlerToken.IsCancellationRequested); + Assert.False(stopping.IsCompleted); + finish.TrySetResult(); + await stopping.WaitAsync(TimeSpan.FromSeconds(5), token); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("exports"), token); + Assert.Equal(1, stats.Completed); + Assert.Equal(0, stats.Queued); + } + finally + { + finish.TrySetResult(); + await consumer.DisposeAsync(); + } + } + + [Fact] + public async Task ConsumeWithOutcomeAsync_RetryBudget_DeadLettersReturnedFailure() + { + var token = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + int calls = 0; + await using var consumer = await bus.ConsumeWithOutcomeAsync((_, _) => + { + Interlocked.Increment(ref calls); + return new ValueTask(MessageOutcome.Retry("service unavailable")); + }, new MessageConsumerOptions { Destination = "exports", MaxAttempts = 2, RedeliveryBackoff = _ => TimeSpan.Zero }, token); + await bus.SendAsync(new Work(), new MessageSendOptions { Destination = "exports" }, token); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(token); + deadline.CancelAfter(TimeSpan.FromSeconds(5)); + IReadOnlyList deadLetters; + do + { + deadLetters = await transport.PeekDeadLetteredAsync(DestinationAddress.ForQueue("exports"), cancellationToken: deadline.Token); + if (deadLetters.Count == 0) await Task.Delay(10, deadline.Token); + } while (deadLetters.Count == 0); + Assert.Equal(2, calls); + Assert.Equal("service unavailable", Assert.Single(deadLetters).Headers[KnownHeaders.DeadLetterReason]); + } + + private sealed record Work; +} diff --git a/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs b/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs new file mode 100644 index 000000000..ba2206506 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs @@ -0,0 +1,8 @@ +using Foundatio.Messaging; + +namespace Foundatio.Tests.Messaging; + +public sealed class MessageExecutionStoreTests : MessageExecutionStoreConformanceTests +{ + protected override IMessageExecutionStore CreateStore() => new InMemoryMessageExecutionStore(); +} From 872d66900636347a5b76ccca41096efaab290332 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 7 Sep 2026 21:02:54 -0500 Subject: [PATCH 2/4] Track broker executions in the shared job runtime store --- .agents/skills/foundatio/SKILL.md | 2 +- docs/guide/messaging.md | 2 +- .../Messaging/RedisMessageExecutionStore.cs | 19 +- .../RedisJobRuntimeStore.Broker.cs | 224 ++++++++++++++++++ .../RedisJobRuntimeStore.Claims.cs | 37 ++- src/Foundatio.Redis/RedisJobRuntimeStore.cs | 52 ++-- .../Jobs/JobRuntimeStoreConformanceTests.cs | 141 +++++++++++ .../Jobs/InMemoryJobRuntimeStore.Broker.cs | 127 ++++++++++ .../Jobs/InMemoryJobRuntimeStore.Claims.cs | 24 +- .../JobCounterStats.cs} | 8 +- src/Foundatio/Jobs/JobMonitorExtensions.cs | 24 ++ src/Foundatio/Jobs/JobRuntime.cs | 147 ++++++++++-- .../Tracking/IMessageExecutionStore.cs | 3 +- .../Tracking/InMemoryMessageExecutionStore.cs | 9 +- .../Tracking/MessageExecutionOptions.cs | 2 - .../Tracking/MessageExecutionPipeline.cs | 208 ++++++++-------- .../Jobs/LeaseSupervisionTests.cs | 9 + .../Messaging/MessageEndpointPolicyTests.cs | 24 +- 18 files changed, 870 insertions(+), 192 deletions(-) create mode 100644 src/Foundatio.Redis/RedisJobRuntimeStore.Broker.cs create mode 100644 src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs rename src/Foundatio/{Messaging/Tracking/MessageExecutionCounters.cs => Jobs/JobCounterStats.cs} (83%) create mode 100644 src/Foundatio/Jobs/JobMonitorExtensions.cs diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index 9ab76d8c8..8bf7a3a17 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -361,7 +361,7 @@ Validate a custom transport or job store against the shared conformance suites i ## Broker execution integration -- Optional broker execution history uses `IMessageExecutionStore` with `.Messaging.UseInMemoryExecutionTracking()` or `.UseRedisExecutionTracking()`. `MessageExecutionPipeline` and native `MessageProcessingContext` support progress, cancellation, and attempt-fenced state. Do not enqueue the same delivery through `IJobRuntimeStore`; the bus owns receiving and settlement. +- Optional broker history uses the normal job store: `.Jobs.UseInMemory()` / `.Jobs.UseRedis()`, `IJobRuntimeStore`, `JobState`, and `IJobMonitor`. Create records with `ExecutionOwner = Broker`; runtime claims and recovery exclude them. `BeginBrokerAttemptAsync` returns a fresh claim token for `ReportJobProgressAsync` / `CompleteJobAsync`. Broker delivery leases stay in the message bus. `MessageExecutionPipeline` and `MessageProcessingContext` connect processing to the shared job store without scheduling the same work twice. - `ConsumeWithOutcomeAsync` handles returned Success/Retry/DeadLetter/Unsettled outcomes. Endpoint receive capacity, visibility, automatic renewal, and graceful drain are configured with MessageHandlerOptions. - `SubscribeNodeAsync` supplies independent best-effort node broadcasts. AWS uses managed tagged resources and startup stale cleanup, not native TTL. Acknowledge-before-callback intentionally allows lost notifications. - Native `MessageAdministration` owns bounded dead-letter inspection/replay. Send-before-delete is at least once, not atomic. Optional replay preparation can create fresh tracked execution IDs. diff --git a/docs/guide/messaging.md b/docs/guide/messaging.md index a2cd7b776..e43e132e8 100644 --- a/docs/guide/messaging.md +++ b/docs/guide/messaging.md @@ -156,7 +156,7 @@ The former publish-only interfaces live under `Foundatio.Messaging.Legacy`; `Mes ## Broker-driven execution tracking -For queues that need progress, cancellation, and operational history, configure `.Messaging.UseInMemoryExecutionTracking()` or `.Messaging.UseRedisExecutionTracking()`. This registers `IMessageExecutionStore`; it does not create runnable jobs, a job worker, or a second scheduler. Delivery remains owned by the message bus. +For queues that need progress, cancellation, and operational history, configure `.Jobs.UseInMemory()` or `.Jobs.UseRedis()`. Tracked deliveries use the existing `IJobRuntimeStore`, `JobState`, and `IJobMonitor`. Set `ExecutionOwner = JobExecutionOwner.Broker` when recording enqueue acceptance: the ordinary job worker cannot claim or recover these records. The message bus owns delivery leases and settlement, while each delivery attempt gets a fresh token for atomic progress and completion updates. Registering the store starts no job worker or scheduler. `MessageExecutionPipeline` runs a raw delivery callback with `MessageProcessingContext`, applies returned `MessageOutcome` values, and persists only confirmed settlement. Pass it to a manual-ack raw `ConsumeAsync` endpoint with `WaitForManualSettlement = false`. The producer creates a unique execution state before sending and puts its ID in `ExecutionHeaders.ExecutionId`. Integrations such as Foundatio.Mediator perform this composition automatically. diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs index 9980cd7e7..01f8545b8 100644 --- a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs +++ b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs @@ -1,3 +1,4 @@ +using Foundatio.Jobs; using System; using System.Collections.Generic; using System.Linq; @@ -33,7 +34,7 @@ public sealed class RedisMessageExecutionStore : IMessageExecutionStore public bool IsShared => true; private const string MetadataFieldPrefix = "meta:"; - private static readonly TimeSpan MessageExecutionCounterBucketRetention = TimeSpan.FromHours(48); + private static readonly TimeSpan JobCounterBucketRetention = TimeSpan.FromHours(48); private readonly IConnectionMultiplexer _redis; private readonly RedisMessageExecutionStoreOptions _options; @@ -131,14 +132,14 @@ private async Task CleanupAsync(string queueName, CancellationToken cancellation public Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default) { var db = _redis.GetDatabase(); - var bucketKey = MessageExecutionCounterBucketKey(queueName, _timeProvider.GetUtcNow()); + var bucketKey = JobCounterBucketKey(queueName, _timeProvider.GetUtcNow()); // Increment and retention refresh form one atomic server operation, without MULTI/EXEC overhead. return db.ScriptEvaluateAsync(RedisMessageExecutionScripts.IncrementCounter, [bucketKey], - [counterName, value, (long)MessageExecutionCounterBucketRetention.TotalMilliseconds]).WaitAsync(cancellationToken); + [counterName, value, (long)JobCounterBucketRetention.TotalMilliseconds]).WaitAsync(cancellationToken); } - public async Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) + public async Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) { var db = _redis.GetDatabase(); var now = _timeProvider.GetUtcNow(); @@ -153,11 +154,11 @@ public async Task GetCounterStatsAsync(string queueNam var batch = db.CreateBatch(); var tasks = new Task[hours.Count]; for (int i = 0; i < hours.Count; i++) - tasks[i] = batch.HashGetAllAsync(MessageExecutionCounterBucketKey(queueName, hours[i])); + tasks[i] = batch.HashGetAllAsync(JobCounterBucketKey(queueName, hours[i])); batch.Execute(); var totals = new Dictionary(); - var buckets = new List(hours.Count); + var buckets = new List(hours.Count); for (int i = 0; i < hours.Count; i++) { @@ -174,10 +175,10 @@ public async Task GetCounterStatsAsync(string queueNam } } - buckets.Add(new MessageExecutionCounterBucket { Hour = hours[i], Counters = counters }); + buckets.Add(new JobCounterBucket { Hour = hours[i], Counters = counters }); } - return new MessageExecutionCounters { Totals = totals, Buckets = buckets }; + return new JobCounterStats { Totals = totals, Buckets = buckets }; } public async Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default) @@ -288,7 +289,7 @@ private static HashEntry[] BuildEntries(MessageExecutionState state, long create private string CancelKey(string jobId) => $"{_keyPrefix}:{jobId}:cancel"; private string QueueSetKey(string queueName) => $"{_keyPrefix}:queues:{queueName}"; private string StatusSetKey(string queueName, MessageExecutionStatus status) => $"{_keyPrefix}:queues:{queueName}:status:{(int)status}"; - private string MessageExecutionCounterBucketKey(string queueName, DateTimeOffset timestamp) => $"{_keyPrefix}:counters:{queueName}:{TruncateToHour(timestamp):yyyy-MM-ddTHH}"; + private string JobCounterBucketKey(string queueName, DateTimeOffset timestamp) => $"{_keyPrefix}:counters:{queueName}:{TruncateToHour(timestamp):yyyy-MM-ddTHH}"; private static DateTimeOffset TruncateToHour(DateTimeOffset timestamp) => new(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0, TimeSpan.Zero); diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.Broker.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.Broker.cs new file mode 100644 index 000000000..b793760a8 --- /dev/null +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.Broker.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace Foundatio.Jobs; + +public sealed partial class RedisJobRuntimeStore +{ + public bool IsShared => true; + + private const string MonitoringFunctions = """ + local function hex(value) + return (string.gsub(value, '.', function(c) return string.format('%02X', string.byte(c)) end)) + end + local function monitoringKeys(job, status) + local id = redis.call('HGET', job, 'jobId') + local prefix = string.sub(job, 1, #job - #id - 4) + local keys = {prefix .. 'created', prefix .. 'created-status:' .. status} + local name = redis.call('HGET', job, 'name') + if name then + table.insert(keys, prefix .. 'created-name:' .. hex(name)) + table.insert(keys, prefix .. 'created-name:' .. hex(name) .. ':' .. status) + end + local queue = redis.call('HGET', job, 'queueName') + if queue then + table.insert(keys, prefix .. 'created-queue:' .. hex(queue)) + table.insert(keys, prefix .. 'created-queue:' .. hex(queue) .. ':' .. status) + end + return keys, prefix, id + end + local function removeMonitoring(job) + local status = redis.call('HGET', job, 'monitorStatus') or redis.call('HGET', job, 'status') + if not status then return end + local keys, prefix, id = monitoringKeys(job, status) + for _, key in ipairs(keys) do redis.call('ZREM', key, id) end + redis.call('ZREM', prefix .. 'broker-expiry', id) + end + local function syncMonitoring(job) + local status = redis.call('HGET', job, 'status') + if not status then return end + local keys, prefix, id = monitoringKeys(job, status) + if redis.call('HGET', job, 'monitorStatus') ~= status then + removeMonitoring(job) + local created = redis.call('HGET', job, 'createdUtc') + for _, key in ipairs(keys) do redis.call('ZADD', key, created, id) end + redis.call('HSET', job, 'monitorStatus', status) + end + local expiry = redis.call('HGET', job, 'historyExpiresUtc') + if expiry then redis.call('ZADD', prefix .. 'broker-expiry', expiry, id) end + end + local function refreshBrokerHistory(job, now) + if redis.call('HGET', job, 'executionOwner') ~= 'Broker' then return end + local retention = tonumber(redis.call('HGET', job, 'historyRetention') or '0') + if retention > 0 then redis.call('HSET', job, 'historyExpiresUtc', string.format('%.0f', now + retention)) end + end + local function forgetJob(job, prefix, id) + removeMonitoring(job) + local status, name = redis.call('HGET', job, 'status'), redis.call('HGET', job, 'name') + if status then redis.call('ZREM', prefix .. 'status:' .. status, id) end + if name then redis.call('ZREM', prefix .. 'name:' .. name, id) end + for _, suffix in ipairs({'all', 'terminal', 'unclaimed'}) do redis.call('ZREM', prefix .. suffix, id) end + if redis.call('HGET', job, 'executionOwner') == 'Broker' then redis.call('ZREM', prefix .. 'deduplication', id) end + local ready, schedule = redis.call('HGET', job, 'readyKey'), redis.call('HGET', job, 'activeScheduleKey') + if ready then redis.call('ZREM', ready, id) end + if schedule then redis.call('SREM', schedule, id) end + redis.call('DEL', job) + end + local function expireJob(job, now) + local expires = tonumber(redis.call('HGET', job, 'historyExpiresUtc')) + if not expires or expires > now or redis.call('HGET', job, 'executionOwner') ~= 'Broker' then return false end + local _, prefix, id = monitoringKeys(job, redis.call('HGET', job, 'status')) + forgetJob(job, prefix, id) + return true + end + """; + + public async Task BeginBrokerAttemptAsync(string jobId, int attempt, string nodeId, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(attempt, 1); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + const string script = MonitoringFunctions + "\n" + """ + if expireJob(KEYS[1], tonumber(ARGV[4])) then return {} end + if redis.call('HGET', KEYS[1], 'executionOwner') ~= 'Broker' then return {} end + local status = redis.call('HGET', KEYS[1], 'status') + if status ~= 'Queued' and status ~= 'Processing' and status ~= 'RetryPending' and status ~= 'EnqueueUnknown' then return {} end + if tonumber(redis.call('HGET', KEYS[1], 'attempt') or '0') >= tonumber(ARGV[1]) then return {} end + redis.call('ZREM', ARGV[5] .. 'status:' .. status, ARGV[6]) + redis.call('ZADD', ARGV[5] .. 'status:Processing', 0, ARGV[6]) + redis.call('HSET', KEYS[1], 'status', 'Processing', 'attempt', ARGV[1], 'nodeId', ARGV[2], 'claimToken', ARGV[3], + 'startedUtc', ARGV[4], 'lastHeartbeatUtc', ARGV[4], 'lastUpdatedUtc', ARGV[4], 'progress', 0) + redis.call('HDEL', KEYS[1], 'progressMessage', 'completedUtc') + refreshBrokerHistory(KEYS[1], tonumber(ARGV[4])) + syncMonitoring(KEYS[1]) + return redis.call('HGETALL', KEYS[1]) + """; + var raw = await _db.ScriptEvaluateAsync(script, [JobKey(jobId)], [attempt, nodeId, Guid.NewGuid().ToString("N"), Ticks(_timeProvider.GetUtcNow()), _prefix, jobId]).WaitAsync(cancellationToken).ConfigureAwait(false); + return ((RedisResult[])raw!).Length == 0 ? null : ReadJobSnapshot(raw); + } + + public async Task MarkEnqueueUnknownAsync(string jobId, string error, CancellationToken cancellationToken = default) + { + const string script = MonitoringFunctions + "\n" + """ + if expireJob(KEYS[1], tonumber(ARGV[2])) then return 0 end + if redis.call('HGET', KEYS[1], 'executionOwner') ~= 'Broker' or redis.call('HGET', KEYS[1], 'status') ~= 'Queued' or redis.call('HGET', KEYS[1], 'attempt') ~= '0' then return 0 end + redis.call('HSET', KEYS[1], 'status', 'EnqueueUnknown', 'error', ARGV[1], 'lastUpdatedUtc', ARGV[2]) + redis.call('ZREM', ARGV[3] .. 'status:Queued', ARGV[4]) + redis.call('ZADD', ARGV[3] .. 'status:EnqueueUnknown', 0, ARGV[4]) + refreshBrokerHistory(KEYS[1], tonumber(ARGV[2])) + syncMonitoring(KEYS[1]) + return 1 + """; + return (long)await _db.ScriptEvaluateAsync(script, [JobKey(jobId)], [error, Ticks(_timeProvider.GetUtcNow()), _prefix, jobId]).WaitAsync(cancellationToken).ConfigureAwait(false) == 1; + } + + public Task HeartbeatJobAsync(string jobId, string claimToken, CancellationToken cancellationToken = default) + => ReportJobProgressAsync(jobId, claimToken, cancellationToken: cancellationToken); + + public async Task RemoveAsync(string jobId, CancellationToken cancellationToken = default) + { + const string script = MonitoringFunctions + "\n" + """ + local status = redis.call('HGET', KEYS[1], 'status') + if not status then return 0 end + if redis.call('HGET', KEYS[1], 'executionOwner') ~= 'Broker' and (status == 'Queued' or status == 'Scheduled' or status == 'Processing') then return 0 end + forgetJob(KEYS[1], ARGV[1], ARGV[2]) + return 1 + """; + return (long)await _db.ScriptEvaluateAsync(script, [JobKey(jobId)], [_prefix, jobId]).WaitAsync(cancellationToken).ConfigureAwait(false) == 1; + } + + private async Task PurgeBrokerHistoryAsync(CancellationToken cancellationToken) + { + const string script = MonitoringFunctions + "\n" + """ + local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 128) + for _, id in ipairs(ids) do + local job = ARGV[2] .. 'job:' .. id + if redis.call('HGET', job, 'executionOwner') == 'Broker' then forgetJob(job, ARGV[2], id) end + redis.call('ZREM', KEYS[1], id) + end + return #ids + """; + while ((long)await _db.ScriptEvaluateAsync(script, [$"{_prefix}broker-expiry"], [Ticks(_timeProvider.GetUtcNow()), _prefix]).WaitAsync(cancellationToken).ConfigureAwait(false) == 128) + cancellationToken.ThrowIfCancellationRequested(); + } + + private RedisKey MonitoringIndex(JobQuery query) + { + string scope = query.QueueName is { } queue ? "created-queue:" + EncodeKey(queue) : query.Name is { } name ? "created-name:" + EncodeKey(name) : "created"; + return $"{_prefix}{scope}{(query.Status is { } status ? (scope == "created" ? "-status:" : ":") + status : "")}"; + } + + public async Task CountAsync(JobQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + query.Validate(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); + if (query.QueueName is null || query.Name is null) + return await _db.SortedSetLengthAsync(MonitoringIndex(query)).WaitAsync(cancellationToken).ConfigureAwait(false); + const string script = """ + local count, offset = 0, 0 + repeat + local ids = redis.call('ZRANGE', KEYS[1], offset, offset + 199) + for _, id in ipairs(ids) do + if redis.call('HGET', ARGV[1] .. 'job:' .. id, 'name') == ARGV[2] then count = count + 1 end + end + offset = offset + #ids + until #ids < 200 + return count + """; + return (long)await _db.ScriptEvaluateAsync(script, [MonitoringIndex(query)], [_prefix, query.Name]).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task QueryNewestAsync(JobQuery query, CancellationToken cancellationToken) + { + const string script = """ + local result, offset, skipped = {}, 0, 0 + repeat + local ids = redis.call('ZREVRANGE', KEYS[1], offset, offset + 199) + for _, id in ipairs(ids) do + local job = ARGV[1] .. 'job:' .. id + if ARGV[2] == '' or redis.call('HGET', job, 'name') == ARGV[2] then + if skipped < tonumber(ARGV[3]) then skipped = skipped + 1 + else table.insert(result, redis.call('HGETALL', job)) end + if #result >= tonumber(ARGV[4]) then return result end + end + end + offset = offset + #ids + until #ids < 200 + return result + """; + var result = (RedisResult[])(await _db.ScriptEvaluateAsync(script, [MonitoringIndex(query)], [_prefix, query.Name ?? "", query.Skip, query.Limit]).WaitAsync(cancellationToken).ConfigureAwait(false))!; + return new JobPage(result.Select(ReadJobSnapshot).ToArray(), null); + } + + public Task IncrementCounterAsync(string name, string counterName, long value = 1, CancellationToken cancellationToken = default) + => _db.ScriptEvaluateAsync("redis.call('HINCRBY', KEYS[1], ARGV[1], ARGV[2]); redis.call('PEXPIRE', KEYS[1], 172800000); return 1", + [CounterKey(name, _timeProvider.GetUtcNow())], [counterName, value]).WaitAsync(cancellationToken); + + public async Task GetCounterStatsAsync(string name, TimeSpan? window = null, CancellationToken cancellationToken = default) + { + var duration = window ?? TimeSpan.FromHours(24); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(duration, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(duration, TimeSpan.FromHours(48)); + var now = _timeProvider.GetUtcNow(); + var hours = new List(); + for (var hour = TruncateHour(now - duration); hour <= TruncateHour(now); hour = hour.AddHours(1)) hours.Add(hour); + var snapshots = await Task.WhenAll(hours.Select(hour => _db.HashGetAllAsync(CounterKey(name, hour)))).WaitAsync(cancellationToken).ConfigureAwait(false); + var totals = new Dictionary(); + var buckets = new List(); + for (int i = 0; i < hours.Count; i++) + { + var counters = snapshots[i].ToDictionary(pair => (string)pair.Name!, pair => (long)pair.Value); + foreach (var pair in counters) totals[pair.Key] = totals.GetValueOrDefault(pair.Key) + pair.Value; + buckets.Add(new() { Hour = hours[i], Counters = counters }); + } + return new JobCounterStats { Totals = totals, Buckets = buckets }; + } + + private RedisKey CounterKey(string name, DateTimeOffset timestamp) => $"{_prefix}counters:{EncodeKey(name)}:{timestamp:yyyy-MM-ddTHH}"; + private static DateTimeOffset TruncateHour(DateTimeOffset value) => new(value.Year, value.Month, value.Day, value.Hour, 0, 0, TimeSpan.Zero); +} diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.Claims.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.Claims.cs index 6007ac2ad..652b6a304 100644 --- a/src/Foundatio.Redis/RedisJobRuntimeStore.Claims.cs +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.Claims.cs @@ -15,13 +15,14 @@ public async Task CreateOccurrenceAsync(JobState initial, b { ArgumentNullException.ThrowIfNull(initial); ArgumentException.ThrowIfNullOrWhiteSpace(initial.ScheduleName); + if (initial.ExecutionOwner != JobExecutionOwner.Runtime) throw new ArgumentException("Scheduled occurrences must be owned by the job runtime.", nameof(initial)); ArgumentException.ThrowIfNullOrWhiteSpace(initial.JobType); cancellationToken.ThrowIfCancellationRequested(); ValidatePayload(initial.Payload?.Length ?? 0); initial.RetryPolicy.Validate(); var now = _timeProvider.GetUtcNow(); var state = initial with { CreatedUtc = initial.CreatedUtc == default ? now : initial.CreatedUtc, LastUpdatedUtc = now }; - const string script = """ + const string script = MonitoringFunctions + "\n" + """ if redis.call('EXISTS', KEYS[1]) == 1 then return 0 end if ARGV[2] == '0' and redis.call('SCARD', KEYS[6]) > 0 then return 2 end redis.call('ZREMRANGEBYSCORE', KEYS[7], '-inf', ARGV[5]) @@ -37,6 +38,7 @@ public async Task CreateOccurrenceAsync(JobState initial, b redis.call('ZADD', KEYS[4], 0, ARGV[1]) redis.call('ZADD', KEYS[5], ARGV[3], ARGV[1]) redis.call('SADD', KEYS[6], ARGV[1]) + syncMonitoring(KEYS[1]) return 1 """; var arguments = new List { state.JobId, allowOverlap ? "1" : "0", Ticks(state.AvailableUtc ?? state.CreatedUtc), _options.MaxActiveJobs, Ticks(now), _options.MaxDeduplicationRecords }; @@ -75,7 +77,7 @@ local candidate if not id then return {} end local job = ARGV[5] .. 'job:' .. id local status = redis.call('HGET', job, 'status') - if status ~= 'Queued' and status ~= 'Scheduled' and status ~= 'Processing' then + if redis.call('HGET', job, 'executionOwner') == 'Broker' or (status ~= 'Queued' and status ~= 'Scheduled' and status ~= 'Processing') then redis.call('ZREM', key, id) else local due = redis.call('HGET', job, status == 'Processing' and 'leaseExpiresUtc' or 'availableUtc') @@ -100,6 +102,7 @@ local candidate redis.call('ZREM', key, id) local active = redis.call('HGET', job, 'activeScheduleKey') if active then redis.call('SREM', active, id) end + syncMonitoring(job) finishJob(job, id, ARGV[5], now, tonumber(ARGV[7]), tonumber(ARGV[8]), tonumber(ARGV[9])) else redis.call('HSET', job, 'status', 'Processing', 'nodeId', ARGV[2], 'claimToken', ARGV[3], @@ -108,6 +111,7 @@ local candidate redis.call('ZADD', ARGV[5] .. 'status:Processing', 0, id) redis.call('ZADD', key, ARGV[4], id) redis.call('ZREM', ARGV[5] .. 'unclaimed', id) + syncMonitoring(job) return redis.call('HGETALL', job) end end @@ -117,7 +121,26 @@ local candidate """; private const string CompleteJobScript = RetentionFunctions + "\n" + """ + if expireJob(KEYS[1], tonumber(ARGV[2])) then return 0 end if redis.call('HGET', KEYS[1], 'status') ~= 'Processing' or redis.call('HGET', KEYS[1], 'claimToken') ~= ARGV[1] then return 0 end + if redis.call('HGET', KEYS[1], 'executionOwner') == 'Broker' then + local kind = tonumber(ARGV[3]) + local status = kind == 0 and 'Completed' or kind == 2 and 'Cancelled' or (kind == 3 or (kind == 1 and ARGV[7] == '1')) and 'RetryPending' or 'Failed' + local id = redis.call('HGET', KEYS[1], 'jobId') + redis.call('ZREM', ARGV[5] .. 'status:Processing', id) + redis.call('ZADD', ARGV[5] .. 'status:' .. status, 0, id) + redis.call('HSET', KEYS[1], 'status', status, 'lastUpdatedUtc', ARGV[2], 'error', ARGV[4], 'resultMessage', ARGV[6]) + redis.call('HDEL', KEYS[1], 'claimToken', 'leaseExpiresUtc', 'availableUtc') + refreshBrokerHistory(KEYS[1], tonumber(ARGV[2])) + syncMonitoring(KEYS[1]) + if status ~= 'RetryPending' then + redis.call('HSET', KEYS[1], 'completedUtc', ARGV[2]) + if status == 'Completed' then redis.call('HSET', KEYS[1], 'progress', 100) end + redis.call('ZADD', ARGV[5] .. 'terminal', ARGV[2], id) + finishJob(KEYS[1], id, ARGV[5], tonumber(ARGV[2]), tonumber(ARGV[9]), tonumber(ARGV[10]), tonumber(ARGV[11])) + end + return 1 + end if tonumber(redis.call('HGET', KEYS[1], 'leaseExpiresUtc') or '0') <= tonumber(ARGV[2]) then return 0 end local kind = tonumber(ARGV[3]) if redis.call('HGET', KEYS[1], 'cancellationRequested') == '1' then kind = 2 end @@ -154,6 +177,7 @@ local candidate redis.call('ZADD', ARGV[5] .. 'terminal', ARGV[2], id) finishJob(KEYS[1], id, ARGV[5], tonumber(ARGV[2]), tonumber(ARGV[9]), tonumber(ARGV[10]), tonumber(ARGV[11])) end + syncMonitoring(KEYS[1]) return 1 """; @@ -166,12 +190,15 @@ local candidate return 1 """; - private const string ReportJobProgressScript = """ + private const string ReportJobProgressScript = MonitoringFunctions + "\n" + """ + if expireJob(KEYS[1], tonumber(ARGV[2])) then return 0 end if redis.call('HGET', KEYS[1], 'status') ~= 'Processing' or redis.call('HGET', KEYS[1], 'claimToken') ~= ARGV[1] then return 0 end - if tonumber(redis.call('HGET', KEYS[1], 'leaseExpiresUtc') or '0') <= tonumber(ARGV[2]) then return 0 end + if redis.call('HGET', KEYS[1], 'executionOwner') ~= 'Broker' and tonumber(redis.call('HGET', KEYS[1], 'leaseExpiresUtc') or '0') <= tonumber(ARGV[2]) then return 0 end if ARGV[3] ~= '' then redis.call('HSET', KEYS[1], 'progress', ARGV[3]) end if ARGV[4] == '1' then redis.call('HSET', KEYS[1], 'progressMessage', ARGV[5]) end - redis.call('HSET', KEYS[1], 'lastUpdatedUtc', ARGV[2]) + redis.call('HSET', KEYS[1], 'lastUpdatedUtc', ARGV[2], 'lastHeartbeatUtc', ARGV[2]) + refreshBrokerHistory(KEYS[1], tonumber(ARGV[2])) + syncMonitoring(KEYS[1]) return 1 """; diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.cs index 3460b9dcb..db3e20de4 100644 --- a/src/Foundatio.Redis/RedisJobRuntimeStore.cs +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.cs @@ -61,10 +61,12 @@ public async Task CreateIfAbsentAsync(JobState initial, CancellationToken cancel ArgumentNullException.ThrowIfNull(initial); cancellationToken.ThrowIfCancellationRequested(); ValidatePayload(initial.Payload?.Length ?? 0); + initial.Validate(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); initial.RetryPolicy.Validate(); var now = _timeProvider.GetUtcNow(); var state = initial with { CreatedUtc = initial.CreatedUtc == default ? now : initial.CreatedUtc, LastUpdatedUtc = initial.LastUpdatedUtc == default ? now : initial.LastUpdatedUtc }; - const string script = """ + const string script = MonitoringFunctions + "\n" + """ if redis.call('EXISTS', KEYS[1]) == 1 then return 0 end redis.call('ZREMRANGEBYSCORE', KEYS[6], '-inf', ARGV[6]) if redis.call('ZSCORE', KEYS[6], ARGV[1]) then return 0 end @@ -86,9 +88,11 @@ public async Task CreateIfAbsentAsync(JobState initial, CancellationToken cancel local completed = redis.call('HGET', KEYS[1], 'completedUtc') if completed then redis.call('ZADD', KEYS[5], completed, ARGV[1]) end end + refreshBrokerHistory(KEYS[1], tonumber(ARGV[6])) + syncMonitoring(KEYS[1]) return 1 """; - var args = new List { state.JobId, _options.MaxActiveJobs, Ticks(state.Status == JobStatus.Processing ? state.LeaseExpiresUtc ?? now : state.AvailableUtc ?? state.CreatedUtc), state.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing ? "1" : "0", _options.MaxDeduplicationRecords, Ticks(now), state.CompletedUtc is { } completed ? Ticks(completed.Add(_options.DeduplicationRetention)) : "+inf" }; + var args = new List { state.JobId, _options.MaxActiveJobs, Ticks(state.Status == JobStatus.Processing ? state.LeaseExpiresUtc ?? now : state.AvailableUtc ?? state.CreatedUtc), state.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing or JobStatus.RetryPending or JobStatus.EnqueueUnknown ? "1" : "0", _options.MaxDeduplicationRecords, Ticks(now), state.CompletedUtc is { } completed ? Ticks(completed.Add(_options.DeduplicationRetention)) : "+inf" }; foreach (var field in ToHash(state)) { args.Add(field.Name); args.Add(field.Value); } var result = await _db.ScriptEvaluateAsync(script, new RedisKey[] { JobKey(state.JobId), AllKey, StatusKey(state.Status), NameKey(state.Name), TerminalKey, DeduplicationKey, UnclaimedKey }, args.ToArray()).ConfigureAwait(false); ThrowIfCapacityExceeded((long)result); @@ -97,6 +101,7 @@ public async Task CreateIfAbsentAsync(JobState initial, CancellationToken cancel public async Task GetAsync(string jobId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); var entries = await _db.HashGetAllAsync(JobKey(jobId)).ConfigureAwait(false); return entries.Length == 0 ? null : FromHash(entries); } @@ -106,6 +111,8 @@ public async Task QueryAsync(JobQuery query, CancellationToken cancella ArgumentNullException.ThrowIfNull(query); query.Validate(); cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); + if (query.NewestFirst) return await QueryNewestAsync(query, cancellationToken).ConfigureAwait(false); var index = query.Name is not null ? NameKey(query.Name) : query.Status is { } status ? StatusKey(status) : AllKey; const string script = """ local ids = redis.call('ZRANGEBYLEX', KEYS[1], ARGV[1], '+', 'LIMIT', 0, 1001) @@ -113,7 +120,7 @@ public async Task QueryAsync(JobQuery query, CancellationToken cancella for i = 1, math.min(#ids, 1000) do local id = ids[i] local job = ARGV[2] .. id - if ARGV[3] == '' or redis.call('HGET', job, 'status') == ARGV[3] then + if (ARGV[3] == '' or redis.call('HGET', job, 'status') == ARGV[3]) and (ARGV[5] == '' or redis.call('HGET', job, 'queueName') == ARGV[5]) then table.insert(result, redis.call('HGETALL', job)) end if #result >= tonumber(ARGV[4]) or i == 1000 then @@ -123,7 +130,7 @@ public async Task QueryAsync(JobQuery query, CancellationToken cancella end return {cursor, result} """; - var raw = (RedisResult[])(await _db.ScriptEvaluateAsync(script, new RedisKey[] { index }, new RedisValue[] { query.AfterJobId is null ? "-" : "(" + query.AfterJobId, $"{_prefix}job:", query.Status?.ToString() ?? "", query.Limit }).ConfigureAwait(false))!; + var raw = (RedisResult[])(await _db.ScriptEvaluateAsync(script, new RedisKey[] { index }, new RedisValue[] { query.AfterJobId is null ? "-" : "(" + query.AfterJobId, $"{_prefix}job:", query.Status?.ToString() ?? "", query.Limit, query.QueueName ?? "" }).ConfigureAwait(false))!; var states = ((RedisResult[])raw[1]!).Select(ReadJobSnapshot).ToArray(); string? cursor = (string?)raw[0]; return new JobPage(states, String.IsNullOrEmpty(cursor) ? null : cursor); @@ -137,7 +144,7 @@ private static JobState ReadJobSnapshot(RedisResult snapshot) return FromHash(fields); } - private const string RetentionFunctions = """ + private const string RetentionFunctions = MonitoringFunctions + "\n" + """ local function trimHistory(prefix, now, maximum, retention, limit) if limit <= 0 then return 0 end local terminal = prefix .. 'terminal' @@ -148,19 +155,15 @@ local function trimHistory(prefix, now, maximum, retention, limit) if count - removed <= maximum and tonumber(candidates[i+1]) > now - retention then break end local id = candidates[i] local job = prefix .. 'job:' .. id - local status = redis.call('HGET', job, 'status') - local name = redis.call('HGET', job, 'name') - if status then redis.call('ZREM', prefix .. 'status:' .. status, id) end - if name then redis.call('ZREM', prefix .. 'name:' .. name, id) end - redis.call('ZREM', prefix .. 'all', id) - redis.call('ZREM', terminal, id) - redis.call('DEL', job) + forgetJob(job, prefix, id) removed = removed + 1 end return removed end local function finishJob(job, id, prefix, now, maximum, retention, dedupRetention) - redis.call('ZADD', prefix .. 'deduplication', string.format('%.0f', now + dedupRetention), id) + if redis.call('HGET', job, 'executionOwner') ~= 'Broker' then + redis.call('ZADD', prefix .. 'deduplication', string.format('%.0f', now + dedupRetention), id) + end redis.call('ZREM', prefix .. 'unclaimed', id) return trimHistory(prefix, now, maximum, retention, 1000) end @@ -183,6 +186,7 @@ private void ThrowIfCapacityExceeded(long result) public async Task GetStatsAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); var raw = (RedisResult[])(await _db.ScriptEvaluateAsync("return {redis.call('ZCARD', KEYS[1]), redis.call('ZCARD', KEYS[2]), redis.call('ZCARD', KEYS[3]), redis.call('ZCARD', KEYS[4])}", [AllKey, TerminalKey, DeduplicationKey, DueKey]).ConfigureAwait(false))!; return new JobRuntimeStoreStats((long)raw[0] - (long)raw[1], (long)raw[1], (long)raw[2], (long)raw[3]); @@ -193,6 +197,7 @@ public async Task CleanupAsync(int limit = 1000, CancellationToken cancella ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1); ArgumentOutOfRangeException.ThrowIfGreaterThan(limit, 1000); cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); const string script = RetentionFunctions + "\n" + """ local now, prefix = tonumber(ARGV[1]), ARGV[3] redis.call('ZREMRANGEBYSCORE', KEYS[3], '-inf', ARGV[1]) @@ -206,6 +211,7 @@ public async Task CleanupAsync(int limit = 1000, CancellationToken cancella redis.call('ZADD', prefix .. 'status:Cancelled', 0, id) redis.call('HSET', job, 'status', 'Cancelled', 'completedUtc', ARGV[1], 'lastUpdatedUtc', ARGV[1], 'resultMessage', 'Unclaimed per-node occurrence expired.') redis.call('ZADD', KEYS[1], ARGV[1], id) + syncMonitoring(job) local ready, active = redis.call('HGET', job, 'readyKey'), redis.call('HGET', job, 'activeScheduleKey') if ready then redis.call('ZREM', ready, id) end if active then redis.call('SREM', active, id) end @@ -223,11 +229,13 @@ public async Task CleanupAsync(int limit = 1000, CancellationToken cancella public async Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); const string script = RetentionFunctions + "\n" + """ if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end local status = redis.call('HGET', KEYS[1], 'status') + if redis.call('HGET', KEYS[1], 'executionOwner') == 'Broker' and (status == 'Completed' or status == 'Failed' or status == 'Cancelled' or status == 'DeadLettered') then return 0 end redis.call('HSET', KEYS[1], 'cancellationRequested', '1', 'lastUpdatedUtc', ARGV[2]) - if status == 'Queued' or status == 'Scheduled' then + if redis.call('HGET', KEYS[1], 'executionOwner') ~= 'Broker' and (status == 'Queued' or status == 'Scheduled') then redis.call('HSET', KEYS[1], 'status', 'Cancelled', 'completedUtc', ARGV[2]) redis.call('ZREM', KEYS[2], ARGV[1]) redis.call('ZREM', KEYS[3], ARGV[1]) @@ -239,6 +247,7 @@ public async Task RequestCancellationAsync(string jobId, CancellationToken if active then redis.call('SREM', active, ARGV[1]) end finishJob(KEYS[1], ARGV[1], ARGV[3], tonumber(ARGV[2]), tonumber(ARGV[4]), tonumber(ARGV[5]), tonumber(ARGV[6])) end + syncMonitoring(KEYS[1]) return 1 """; var result = await _db.ScriptEvaluateAsync(script, @@ -250,6 +259,7 @@ public async Task RequestCancellationAsync(string jobId, CancellationToken public async Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + await PurgeBrokerHistoryAsync(cancellationToken).ConfigureAwait(false); var value = await _db.HashGetAsync(JobKey(jobId), "cancellationRequested").ConfigureAwait(false); return value == "1"; } @@ -347,6 +357,7 @@ private HashEntry[] ToHash(JobState state) { new("jobId", state.JobId), new("name", state.Name), + new("executionOwner", state.ExecutionOwner.ToString()), new("status", state.Status.ToString()), new("attempt", state.Attempt), new("maxAttempts", state.MaxAttempts), @@ -360,10 +371,15 @@ private HashEntry[] ToHash(JobState state) new("lastUpdatedUtc", Ticks(state.LastUpdatedUtc)) }; + if (state.QueueName is not null) entries.Add(new("queueName", state.QueueName)); + if (state.Metadata is not null) entries.Add(new("metadata", JsonSerializer.Serialize(state.Metadata))); + if (state.LastHeartbeatUtc is { } heartbeat) entries.Add(new("lastHeartbeatUtc", Ticks(heartbeat))); + if (state.HistoryRetention is { } retention) entries.Add(new("historyRetention", retention.Ticks)); + if (state.HistoryExpiresUtc is { } historyExpiry) entries.Add(new("historyExpiresUtc", Ticks(historyExpiry))); if (state.JobType is not null) { entries.Add(new("jobType", state.JobType)); - entries.Add(new("readyKey", ReadyKey(state.JobType, state.RequiredNodeId).ToString())); + if (state.ExecutionOwner == JobExecutionOwner.Runtime) entries.Add(new("readyKey", ReadyKey(state.JobType, state.RequiredNodeId).ToString())); } if (state.ExpiresUtc is { } expires) entries.Add(new("expiresUtc", Ticks(expires))); if (state.RequiredNodeId is not null) entries.Add(new("requiredNodeId", state.RequiredNodeId)); @@ -398,6 +414,12 @@ private static JobState FromHash(HashEntry[] entries) JobId = (string)Get("jobId")!, Name = (string)Get("name")!, JobType = ToStringOrNull(Get("jobType")), + ExecutionOwner = Get("executionOwner") == "Broker" ? JobExecutionOwner.Broker : JobExecutionOwner.Runtime, + QueueName = ToStringOrNull(Get("queueName")), + Metadata = Get("metadata").IsNullOrEmpty ? null : JsonSerializer.Deserialize>((string)Get("metadata")!), + LastHeartbeatUtc = ParseTime(Get("lastHeartbeatUtc")), + HistoryRetention = Get("historyRetention").IsNullOrEmpty ? null : TimeSpan.FromTicks((long)Get("historyRetention")), + HistoryExpiresUtc = ParseTime(Get("historyExpiresUtc")), Payload = Get("payload").IsNullOrEmpty ? null : Convert.FromBase64String((string)Get("payload")!), PayloadType = ToStringOrNull(Get("payloadType")), Status = Enum.Parse((string)Get("status")!), diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs index 9797df5e8..be5daafd4 100644 --- a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -23,6 +23,147 @@ namespace Foundatio.Tests.Jobs; /// public abstract class JobRuntimeStoreConformanceTests : TestWithLoggingBase { + [Fact] + public virtual async Task BrokerHistoryPressure_ReleasesCapacityAndExpiredIdentityAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time, new JobRuntimeStoreOptions { MaxActiveJobs = 1, MaxHistoryJobs = 1, MaxDeduplicationRecords = 2 }); + Assert.SkipWhen(store is null, "Job runtime store not configured."); + var token = TestCancellationToken; + for (int i = 0; i < 5; i++) + { + var id = $"broker-{i}"; + await store.CreateIfAbsentAsync(new JobState { JobId = id, Name = "exports", QueueName = "exports", ExecutionOwner = JobExecutionOwner.Broker, HistoryRetention = TimeSpan.FromMinutes(1) }, token); + var claim = Assert.IsType(await store.BeginBrokerAttemptAsync(id, 1, "worker", token)); + Assert.True(await store.CompleteJobAsync(id, claim.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Succeeded }, token)); + time.Advance(TimeSpan.FromSeconds(1)); + } + Assert.Equal(new JobRuntimeStoreStats(0, 1, 1, 0), await store.GetStatsAsync(token)); + time.Advance(TimeSpan.FromMinutes(2)); + await store.CreateIfAbsentAsync(new JobState { JobId = "broker-4", Name = "new", QueueName = "exports", ExecutionOwner = JobExecutionOwner.Broker, HistoryRetention = TimeSpan.FromMinutes(1) }, token); + Assert.Equal("new", (await store.GetAsync("broker-4", token))!.Name); + time.Advance(TimeSpan.FromMinutes(2)); + Assert.Equal(new JobRuntimeStoreStats(0, 0, 0, 0), await store.GetStatsAsync(token)); + } + + [Fact] + public virtual async Task BrokerAndRuntimeJobs_ShareQueriesMetadataAndCountersAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + Assert.SkipWhen(store is null, "Job runtime store not configured."); + var token = TestCancellationToken; + await store.CreateIfAbsentAsync(new JobState { JobId = "runtime", Name = "reports", JobType = "work" }, token); + for (int i = 0; i < 28; i++) + { + time.Advance(TimeSpan.FromMilliseconds(1)); + await store.CreateIfAbsentAsync(new JobState + { + JobId = $"broker-{i:D2}", + Name = "reports", + ExecutionOwner = JobExecutionOwner.Broker, + QueueName = "exports", + PayloadType = "Export", + HistoryRetention = TimeSpan.FromDays(1), + Metadata = new Dictionary { ["tenant"] = "acme" } + }, token); + } + Assert.Equal(29, await store.CountAsync(new JobQuery { Name = "reports" }, token)); + Assert.Equal(28, await store.CountAsync(new JobQuery { QueueName = "exports" }, token)); + var query = new JobQuery { QueueName = "exports", NewestFirst = true, Limit = 25 }; + var first = await store.QueryAsync(query, token); + var second = await store.QueryAsync(query with { Skip = 25 }, token); + Assert.Equal(25, first.Count); + Assert.Equal(3, second.Count); + Assert.Equal("broker-27", first[0].JobId); + Assert.Equal("broker-02", second[0].JobId); + Assert.Empty(first.Select(j => j.JobId).Intersect(second.Select(j => j.JobId))); + Assert.All(first, job => Assert.Equal("acme", job.Metadata!["tenant"])); + var claim = Assert.IsType(await store.BeginBrokerAttemptAsync("broker-00", 1, "worker", token)); + Assert.Equal(1, await store.CountAsync(query with { Status = JobStatus.Processing }, token)); + Assert.Equal("broker-00", Assert.Single(await store.QueryAsync(query with { Status = JobStatus.Processing }, token)).JobId); + Assert.Equal(27, await store.CountAsync(query with { Status = JobStatus.Queued }, token)); + await store.IncrementCounterAsync("exports", "processed", 2, token); + time.Advance(TimeSpan.FromHours(1)); + await store.IncrementCounterAsync("exports", "processed", 3, token); + var counters = await store.GetCounterStatsAsync("exports", cancellationToken: token); + Assert.Equal(5, counters.Totals["processed"]); + Assert.Equal(2, counters.Buckets.Count(bucket => bucket.Counters.ContainsKey("processed"))); + Assert.True(await store.RemoveAsync(claim.JobId, token)); + Assert.Equal(0, await store.CountAsync(query with { Status = JobStatus.Processing }, token)); + } + + [Fact] + public virtual async Task BrokerJobs_ShareMonitoringButNeverEnterRuntimeClaimsAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + Assert.SkipWhen(store is null, "Job runtime store not configured."); + var token = TestCancellationToken; + await store.CreateIfAbsentAsync(new JobState + { + JobId = "broker", + Name = "work", + JobType = "work", + QueueName = "exports", + ExecutionOwner = JobExecutionOwner.Broker, + HistoryRetention = TimeSpan.FromHours(1) + }, token); + var request = new JobClaimRequest { NodeId = "runtime", JobTypes = ["work"] }; + Assert.Null(await store.ClaimNextAsync(request, token)); + Assert.Null(await store.ClaimJobAsync("broker", request, token)); + var first = Assert.IsType(await store.BeginBrokerAttemptAsync("broker", 1, "worker", token)); + Assert.Null(await store.BeginBrokerAttemptAsync("broker", 1, "duplicate", token)); + Assert.True(await store.ReportJobProgressAsync("broker", first.ClaimToken!, 25, "Running", token)); + Assert.False(await store.RenewJobLeaseAsync("broker", first.ClaimToken!, TimeSpan.FromMinutes(1), token)); + time.Advance(TimeSpan.FromMinutes(5)); + Assert.Null(await store.ClaimNextAsync(request, token)); + var second = Assert.IsType(await store.BeginBrokerAttemptAsync("broker", 2, "replacement", token)); + Assert.NotEqual(first.ClaimToken, second.ClaimToken); + Assert.False(await store.ReportJobProgressAsync("broker", first.ClaimToken!, 90, cancellationToken: token)); + Assert.False(await store.CompleteJobAsync("broker", first.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Succeeded }, token)); + Assert.True(await store.CompleteJobAsync("broker", second.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Interrupted }, token)); + Assert.Equal(JobStatus.RetryPending, (await store.GetAsync("broker", token))!.Status); + Assert.False(await store.ReportJobProgressAsync("broker", second.ClaimToken!, 90, cancellationToken: token)); + Assert.Null(await store.ClaimNextAsync(request, token)); + Assert.Null(await store.BeginBrokerAttemptAsync("broker", 2, "duplicate", token)); + var third = Assert.IsType(await store.BeginBrokerAttemptAsync("broker", 3, "worker", token)); + Assert.True(await store.CompleteJobAsync("broker", third.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Succeeded }, token)); + Assert.Equal(JobStatus.Completed, Assert.Single(await store.QueryAsync(new JobQuery { QueueName = "exports", NewestFirst = true }, token)).Status); + Assert.Equal(1, await store.CountAsync(new JobQuery { QueueName = "exports", Status = JobStatus.Completed }, token)); + Assert.False(await store.MarkEnqueueUnknownAsync("broker", "Late send timeout", token)); + Assert.Null(await store.BeginBrokerAttemptAsync("broker", 4, "late", token)); + } + + [Fact] + public virtual async Task BrokerCancellationAndRetention_DoNotScheduleOrCancelRuntimeJobsAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + Assert.SkipWhen(store is null, "Job runtime store not configured."); + var token = TestCancellationToken; + await store.CreateIfAbsentAsync(new JobState { JobId = "runtime", Name = "work", JobType = "work" }, token); + await store.CreateIfAbsentAsync(new JobState { JobId = "broker", Name = "work", JobType = "work", QueueName = "exports", ExecutionOwner = JobExecutionOwner.Broker, HistoryRetention = TimeSpan.FromMinutes(1) }, token); + Assert.True(await store.RequestCancellationAsync("broker", token)); + Assert.Equal(JobStatus.Queued, (await store.GetAsync("broker", token))!.Status); + Assert.False(await store.RemoveAsync("runtime", token)); + var claim = Assert.IsType(await store.BeginBrokerAttemptAsync("broker", 1, "worker", token)); + Assert.True(claim.CancellationRequested); + Assert.True(await store.ReportJobProgressAsync("broker", claim.ClaimToken!, 50, cancellationToken: token)); + Assert.True(await store.IsCancellationRequestedAsync("broker", token)); + Assert.True(await store.CompleteJobAsync("broker", claim.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Interrupted }, token)); + Assert.Equal(JobStatus.RetryPending, (await store.GetAsync("broker", token))!.Status); + claim = Assert.IsType(await store.BeginBrokerAttemptAsync("broker", 2, "replacement", token)); + Assert.True(claim.CancellationRequested); + Assert.True(await store.CompleteJobAsync("broker", claim.ClaimToken!, new JobCompletion { Kind = JobCompletionKind.Cancelled }, token)); + Assert.False(await store.RequestCancellationAsync("broker", token)); + time.Advance(TimeSpan.FromMinutes(2)); + await store.CleanupAsync(cancellationToken: token); + Assert.Null(await store.GetAsync("broker", token)); + Assert.Equal(0, await store.CountAsync(new JobQuery { QueueName = "exports" }, token)); + Assert.Equal("runtime", (await store.ClaimNextAsync(new JobClaimRequest { NodeId = "node", JobTypes = ["work"] }, token))!.JobId); + } + [Fact] public virtual async Task RetentionPressure_EvictsHistoryAndPreservesIdempotencyAsync() { diff --git a/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs new file mode 100644 index 000000000..922dbce15 --- /dev/null +++ b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Jobs; + +public sealed partial class InMemoryJobRuntimeStore +{ + public bool IsShared => false; + private readonly SortedSet<(DateTimeOffset Expires, string Id)> _brokerExpiry = new(); + private readonly Dictionary<(string Name, DateTimeOffset Hour), Dictionary> _counters = new(); + + public Task BeginBrokerAttemptAsync(string jobId, int attempt, string nodeId, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(attempt, 1); + ArgumentException.ThrowIfNullOrWhiteSpace(nodeId); + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + PurgeBrokerHistory(); + if (!_jobs.TryGetValue(jobId, out var state) || state.ExecutionOwner != JobExecutionOwner.Broker || !IsActive(state) || attempt <= state.Attempt) + return Task.FromResult(null); + var now = _timeProvider.GetUtcNow(); + StoreJob(state with + { + Status = JobStatus.Processing, + Attempt = attempt, + ClaimToken = Guid.NewGuid().ToString("N"), + NodeId = nodeId, + StartedUtc = now, + LastHeartbeatUtc = now, + LastUpdatedUtc = now, + CompletedUtc = null, + Progress = 0, + ProgressMessage = null + }); + return Task.FromResult(_jobs[jobId]); + } + } + + public Task MarkEnqueueUnknownAsync(string jobId, string error, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + PurgeBrokerHistory(); + if (!_jobs.TryGetValue(jobId, out var state) || state.ExecutionOwner != JobExecutionOwner.Broker || state.Status != JobStatus.Queued || state.Attempt != 0) + return Task.FromResult(false); + StoreJob(state with { Status = JobStatus.EnqueueUnknown, Error = error, LastUpdatedUtc = _timeProvider.GetUtcNow() }); + return Task.FromResult(true); + } + } + + public Task HeartbeatJobAsync(string jobId, string claimToken, CancellationToken cancellationToken = default) + => ReportJobProgressAsync(jobId, claimToken, cancellationToken: cancellationToken); + + public Task RemoveAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + PurgeBrokerHistory(); + if (!_jobs.TryGetValue(jobId, out var state) || state.ExecutionOwner == JobExecutionOwner.Runtime && IsActive(state)) + return Task.FromResult(false); + ForgetJob(state); + return Task.FromResult(true); + } + } + + private void ForgetJob(JobState state) + { + if (state.HistoryExpiresUtc is { } expires) _brokerExpiry.Remove((expires, state.JobId)); + _jobs.TryRemove(state.JobId, out _); + if (IsActive(state)) _activeJobs--; + _active.Remove(state.JobId); + if (state.ExecutionOwner == JobExecutionOwner.Broker) _deduplication.Remove(state.JobId); + } + + private void PurgeBrokerHistory() + { + var now = _timeProvider.GetUtcNow(); + while (_brokerExpiry.Count > 0 && _brokerExpiry.Min.Expires <= now) + { + var item = _brokerExpiry.Min; + _brokerExpiry.Remove(item); + if (_jobs.TryGetValue(item.Id, out var state) && state.HistoryExpiresUtc == item.Expires) ForgetJob(state); + } + } + + public Task IncrementCounterAsync(string name, string counterName, long value = 1, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + var hour = TruncateHour(_timeProvider.GetUtcNow()); + foreach (var key in _counters.Keys.Where(k => k.Hour < hour.AddHours(-48)).ToArray()) _counters.Remove(key); + if (!_counters.TryGetValue((name, hour), out var bucket)) _counters[(name, hour)] = bucket = new(); + bucket[counterName] = bucket.GetValueOrDefault(counterName) + value; + } + return Task.CompletedTask; + } + + public Task GetCounterStatsAsync(string name, TimeSpan? window = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var duration = window ?? TimeSpan.FromHours(24); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(duration, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(duration, TimeSpan.FromHours(48)); + lock (_lock) + { + var now = _timeProvider.GetUtcNow(); + var totals = new Dictionary(); + var buckets = new List(); + for (var hour = TruncateHour(now - duration); hour <= TruncateHour(now); hour = hour.AddHours(1)) + { + var values = _counters.TryGetValue((name, hour), out var bucket) ? new Dictionary(bucket) : new(); + foreach (var pair in values) totals[pair.Key] = totals.GetValueOrDefault(pair.Key) + pair.Value; + buckets.Add(new() { Hour = hour, Counters = values }); + } + return Task.FromResult(new JobCounterStats { Totals = totals, Buckets = buckets }); + } + } + + private static DateTimeOffset TruncateHour(DateTimeOffset value) => new(value.Year, value.Month, value.Day, value.Hour, 0, 0, TimeSpan.Zero); +} diff --git a/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Claims.cs b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Claims.cs index 554ad62cc..0d0810d76 100644 --- a/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Claims.cs +++ b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Claims.cs @@ -11,6 +11,7 @@ public Task CreateOccurrenceAsync(JobState initial, bool al { ArgumentNullException.ThrowIfNull(initial); ArgumentException.ThrowIfNullOrWhiteSpace(initial.ScheduleName); + if (initial.ExecutionOwner != JobExecutionOwner.Runtime) throw new ArgumentException("Scheduled occurrences must be owned by the job runtime.", nameof(initial)); cancellationToken.ThrowIfCancellationRequested(); lock (_lock) { @@ -51,7 +52,7 @@ public Task CreateOccurrenceAsync(JobState initial, bool al var now = _timeProvider.GetUtcNow(); if (_active.Count == 0) return Task.FromResult(null); - var candidates = _active.Values.Where(s => (jobId is null || s.JobId == jobId) + var candidates = _active.Values.Where(s => s.ExecutionOwner == JobExecutionOwner.Runtime && (jobId is null || s.JobId == jobId) && (s.RequiredNodeId is null || s.RequiredNodeId == request.NodeId) && s.JobType is not null && request.JobTypes.Contains(s.JobType, StringComparer.Ordinal) && ((s.Status is JobStatus.Queued or JobStatus.Scheduled && (s.AvailableUtc ?? s.CreatedUtc) <= now) @@ -106,14 +107,15 @@ public Task CompleteJobAsync(string jobId, string claimToken, JobCompletio if (!TryGetOwnedJob(jobId, claimToken, now, out var state)) return Task.FromResult(false); - var kind = state.CancellationRequested ? JobCompletionKind.Cancelled : completion.Kind; - bool retry = kind == JobCompletionKind.Failed && completion.Retryable && state.Attempt < state.MaxAttempts; + bool broker = state.ExecutionOwner == JobExecutionOwner.Broker; + var kind = !broker && state.CancellationRequested ? JobCompletionKind.Cancelled : completion.Kind; + bool retry = kind == JobCompletionKind.Failed && completion.Retryable && (broker || state.Attempt < state.MaxAttempts); var status = kind switch { JobCompletionKind.Succeeded => JobStatus.Completed, JobCompletionKind.Cancelled => JobStatus.Cancelled, - JobCompletionKind.Interrupted => JobStatus.Queued, - JobCompletionKind.Failed => retry ? JobStatus.Queued : JobStatus.Failed, + JobCompletionKind.Interrupted => broker ? JobStatus.RetryPending : JobStatus.Queued, + JobCompletionKind.Failed => retry ? (broker ? JobStatus.RetryPending : JobStatus.Queued) : JobStatus.Failed, _ => throw new ArgumentOutOfRangeException(nameof(completion)) }; StoreJob(state with @@ -121,12 +123,12 @@ public Task CompleteJobAsync(string jobId, string claimToken, JobCompletio Status = status, Error = kind == JobCompletionKind.Failed ? completion.Error : null, ResultMessage = completion.Message, - NodeId = null, + NodeId = broker ? state.NodeId : null, ClaimToken = null, LeaseExpiresUtc = null, LastUpdatedUtc = now, - CompletedUtc = status == JobStatus.Queued ? null : now, - AvailableUtc = retry ? now.Add(state.RetryPolicy.GetDelay(state.Attempt)) : now, + CompletedUtc = status is JobStatus.Queued or JobStatus.RetryPending ? null : now, + AvailableUtc = broker ? null : retry ? now.Add(state.RetryPolicy.GetDelay(state.Attempt)) : now, Progress = status == JobStatus.Completed ? 100 : state.Progress }); return Task.FromResult(true); @@ -142,6 +144,7 @@ public Task RenewJobLeaseAsync(string jobId, string claimToken, TimeSpan l var now = _timeProvider.GetUtcNow(); if (!TryGetOwnedJob(jobId, claimToken, now, out var state)) return Task.FromResult(false); + if (state.ExecutionOwner == JobExecutionOwner.Broker) return Task.FromResult(false); StoreJob(state with { LeaseExpiresUtc = now.Add(lease), LastUpdatedUtc = now }); return Task.FromResult(true); } @@ -157,7 +160,7 @@ public Task ReportJobProgressAsync(string jobId, string claimToken, int? p var now = _timeProvider.GetUtcNow(); if (!TryGetOwnedJob(jobId, claimToken, now, out var state)) return Task.FromResult(false); - StoreJob(state with { Progress = percent ?? state.Progress, ProgressMessage = message ?? state.ProgressMessage, LastUpdatedUtc = now }); + StoreJob(state with { Progress = percent ?? state.Progress, ProgressMessage = message ?? state.ProgressMessage, LastHeartbeatUtc = now, LastUpdatedUtc = now }); return Task.FromResult(true); } } @@ -166,7 +169,8 @@ private bool TryGetOwnedJob(string jobId, string claimToken, DateTimeOffset now, { ArgumentException.ThrowIfNullOrWhiteSpace(jobId); ArgumentException.ThrowIfNullOrWhiteSpace(claimToken); + PurgeBrokerHistory(); return _jobs.TryGetValue(jobId, out state!) && state.Status == JobStatus.Processing - && state.ClaimToken == claimToken && state.LeaseExpiresUtc > now; + && state.ClaimToken == claimToken && (state.ExecutionOwner == JobExecutionOwner.Broker || state.LeaseExpiresUtc > now); } } diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs b/src/Foundatio/Jobs/JobCounterStats.cs similarity index 83% rename from src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs rename to src/Foundatio/Jobs/JobCounterStats.cs index 83b5c8508..e1215d7fb 100644 --- a/src/Foundatio/Messaging/Tracking/MessageExecutionCounters.cs +++ b/src/Foundatio/Jobs/JobCounterStats.cs @@ -3,12 +3,12 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -namespace Foundatio.Messaging; +namespace Foundatio.Jobs; /// /// Counter statistics for a queue, including totals and per-hour buckets for sparkline rendering. /// -public sealed record MessageExecutionCounters +public sealed record JobCounterStats { /// /// Sum of all counters across the requested time window. @@ -20,13 +20,13 @@ public sealed record MessageExecutionCounters /// Per-hour counter values ordered oldest to newest, suitable for sparkline rendering. /// Each bucket represents one UTC hour. /// - public required IReadOnlyList Buckets { get; init; } + public required IReadOnlyList Buckets { get; init; } } /// /// Counter values for a single hour. /// -public sealed record MessageExecutionCounterBucket +public sealed record JobCounterBucket { /// /// The UTC hour this bucket represents (truncated to the hour). diff --git a/src/Foundatio/Jobs/JobMonitorExtensions.cs b/src/Foundatio/Jobs/JobMonitorExtensions.cs new file mode 100644 index 000000000..5d73ae8a7 --- /dev/null +++ b/src/Foundatio/Jobs/JobMonitorExtensions.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Jobs; + +/// Observation helpers for runtime jobs and tracked broker work. +public static class JobMonitorExtensions +{ + /// Waits for terminal state. Cancelling observation does not cancel the job. + public static async Task WaitForCompletionAsync(this IJobMonitor monitor, string jobId, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(monitor); + ArgumentException.ThrowIfNullOrWhiteSpace(jobId); + while (true) + { + var state = await monitor.GetAsync(jobId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Job '{jobId}' was not found or has expired."); + if (state.Status is JobStatus.Completed or JobStatus.Failed or JobStatus.Cancelled or JobStatus.DeadLettered) + return state; + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index c8e8cc380..f969cc5da 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -33,9 +33,16 @@ public enum JobStatus Completed, Failed, Cancelled, - DeadLettered + DeadLettered, + /// A broker delivery may recur; the job runtime must not schedule it. + RetryPending, + /// Sending did not confirm whether the broker accepted the work. + EnqueueUnknown } +/// Identifies the system responsible for delivering and recovering work. +public enum JobExecutionOwner { Runtime, Broker } + public enum ScheduledDispatchKind { QueueMessage, @@ -46,6 +53,18 @@ public sealed record JobState { public required string JobId { get; init; } public required string Name { get; init; } + /// Runtime is the default. Broker records are never claimed by the job worker. + public JobExecutionOwner ExecutionOwner { get; init; } + /// Queue destination for broker-delivered work. + public string? QueueName { get; init; } + /// Optional operational metadata, shared by runtime and broker jobs. + public IReadOnlyDictionary? Metadata { get; init; } + /// Last progress report or explicit execution heartbeat. + public DateTimeOffset? LastHeartbeatUtc { get; init; } + /// Optional retention of broker history since its last update. Expiry never removes broker work. + public TimeSpan? HistoryRetention { get; init; } + /// Store-maintained expiration of optional broker history. + public DateTimeOffset? HistoryExpiresUtc { get; init; } public string? JobType { get; init; } /// Serialized per-invocation arguments (see ); null when the job takes none. @@ -81,6 +100,19 @@ public sealed record JobState public DateTimeOffset? ScheduledForUtc { get; init; } /// Expires an unclaimed per-node occurrence when its intended node never returns. public DateTimeOffset? ExpiresUtc { get; init; } + /// Validates execution ownership and retention settings. + public void Validate() + { + if (!Enum.IsDefined(ExecutionOwner)) throw new ArgumentOutOfRangeException(nameof(ExecutionOwner)); + if (HistoryRetention <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(HistoryRetention)); + if (ExecutionOwner == JobExecutionOwner.Broker) + { + ArgumentException.ThrowIfNullOrWhiteSpace(QueueName); + if (ScheduleName is not null || RequiredNodeId is not null || LeaseExpiresUtc is not null) + throw new ArgumentException("Broker jobs cannot carry runtime scheduling or lease ownership."); + } + } + } public sealed record JobQuery @@ -88,12 +120,23 @@ public sealed record JobQuery public string? Name { get; init; } public JobStatus? Status { get; init; } public int Limit { get; init; } = 100; + /// Restricts monitoring to a queue destination. + public string? QueueName { get; init; } + /// Orders by creation time descending instead of the default job-ID cursor order. + public bool NewestFirst { get; init; } + /// Offset for creation-ordered monitoring pages. + public int Skip { get; init; } + /// Continue after the token returned by the preceding page, using the same filters. public string? AfterJobId { get; init; } public void Validate() { + ArgumentOutOfRangeException.ThrowIfNegative(Skip); + ArgumentOutOfRangeException.ThrowIfGreaterThan(Skip, 1000); + if (NewestFirst && AfterJobId is not null || !NewestFirst && Skip != 0) + throw new ArgumentException("Use Skip with NewestFirst, or AfterJobId with job-ID ordering."); ArgumentOutOfRangeException.ThrowIfLessThan(Limit, 1); ArgumentOutOfRangeException.ThrowIfGreaterThan(Limit, 1000); } @@ -360,6 +403,8 @@ public interface IJobMonitor { Task GetAsync(string jobId, CancellationToken cancellationToken = default); Task QueryAsync(JobQuery query, CancellationToken cancellationToken = default); + /// Counts retained jobs matching the query filters; pagination is ignored. + Task CountAsync(JobQuery query, CancellationToken cancellationToken = default); } public interface IJobClient @@ -406,11 +451,26 @@ public interface IScheduledDispatchStore /// /// The full job runtime store: job state persistence, queries, lease/ownership management, cancellation signaling, /// and scheduled-dispatch storage. The state/lease/cancellation members are deliberately one contract — transitions -/// verify current unexpired claim tokens atomically, so +/// verify current ownership tokens and runtime lease expiry atomically, so /// splitting them would break the compare-and-set semantics correctness depends on. /// public interface IJobRuntimeStore : IJobMonitor, IScheduledDispatchStore, IScheduledJobStore { + /// Whether state and cancellation are shared across processes. + bool IsShared { get; } + /// Records a broker attempt with a new ownership token. Missing, terminal, and stale attempts return null; this never schedules work. + Task BeginBrokerAttemptAsync(string jobId, int attempt, string nodeId, CancellationToken cancellationToken = default); + /// Records uncertain acceptance only before any broker attempt begins. + Task MarkEnqueueUnknownAsync(string jobId, string error, CancellationToken cancellationToken = default); + /// Refreshes history only for the current processing owner. It does not renew a broker delivery lease. + Task HeartbeatJobAsync(string jobId, string claimToken, CancellationToken cancellationToken = default); + /// Removes operational history. Active jobs owned by the runtime cannot be removed. + Task RemoveAsync(string jobId, CancellationToken cancellationToken = default); + /// Adds a named operational counter to the current hourly bucket. + Task IncrementCounterAsync(string name, string counterName, long value = 1, CancellationToken cancellationToken = default); + /// Reads retained hourly operational counters. + Task GetCounterStatsAsync(string name, TimeSpan? window = null, CancellationToken cancellationToken = default); + /// Atomically creates an occurrence, enforcing its unique ID and optional overlap exclusion. Task CreateOccurrenceAsync(JobState initial, bool allowOverlap = false, CancellationToken cancellationToken = default); /// Atomically claims the oldest eligible due job, including recoverable expired executions. @@ -421,7 +481,7 @@ public interface IJobRuntimeStore : IJobMonitor, IScheduledDispatchStore, ISched Task CompleteJobAsync(string jobId, string claimToken, JobCompletion completion, CancellationToken cancellationToken = default); /// Renews only the current, unexpired execution claim. Task RenewJobLeaseAsync(string jobId, string claimToken, TimeSpan lease, CancellationToken cancellationToken = default); - /// Updates progress only for the current, unexpired execution claim. + /// Updates progress only for the current owner, requiring an unexpired lease for runtime-owned jobs. Task ReportJobProgressAsync(string jobId, string claimToken, int? percent = null, string? message = null, CancellationToken cancellationToken = default); Task GetStatsAsync(CancellationToken cancellationToken = default); /// Applies configured history, idempotency and unclaimed occurrence retention in bounded batches. @@ -463,7 +523,9 @@ public Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellation lock (_lock) { ValidatePayload(initial.Payload?.Length ?? 0); + initial.Validate(); initial.RetryPolicy.Validate(); + PurgeBrokerHistory(); PurgeDeduplication(); if (_jobs.ContainsKey(initial.JobId) || _deduplication.ContainsKey(initial.JobId)) return Task.CompletedTask; @@ -482,25 +544,46 @@ public Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellation public Task GetAsync(string jobId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - _jobs.TryGetValue(jobId, out var state); - return Task.FromResult(state); + lock (_lock) + { + PurgeBrokerHistory(); + _jobs.TryGetValue(jobId, out var state); + return Task.FromResult(state); + } } + private IEnumerable MatchingJobs(JobQuery query) => _jobs.Values.Where(s => + (query.Name is null || s.Name == query.Name) && (query.Status is null || s.Status == query.Status) + && (query.QueueName is null || s.QueueName == query.QueueName)); + public Task QueryAsync(JobQuery query, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(query); + query.Validate(); cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) + { + PurgeBrokerHistory(); + var matching = MatchingJobs(query); + var candidates = (query.NewestFirst + ? matching.OrderByDescending(s => s.CreatedUtc).ThenByDescending(s => s.JobId, StringComparer.Ordinal).Skip(query.Skip) + : matching.Where(s => query.AfterJobId is null || StringComparer.Ordinal.Compare(s.JobId, query.AfterJobId) > 0).OrderBy(s => s.JobId, StringComparer.Ordinal)) + .Take(query.Limit + 1).ToArray(); + return Task.FromResult(new JobPage(candidates.Take(query.Limit).ToArray(), !query.NewestFirst && candidates.Length > query.Limit ? candidates[query.Limit - 1].JobId : null)); + } + } + public Task CountAsync(JobQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); query.Validate(); - var candidates = _jobs.Values - .Where(s => (query.Name is null || s.Name == query.Name) && (query.Status is null || s.Status == query.Status)) - .Where(s => query.AfterJobId is null || StringComparer.Ordinal.Compare(s.JobId, query.AfterJobId) > 0) - .OrderBy(s => s.JobId, StringComparer.Ordinal).Take(query.Limit + 1).ToArray(); - return Task.FromResult(new JobPage(candidates.Take(query.Limit).ToArray(), candidates.Length > query.Limit ? candidates[query.Limit - 1].JobId : null)); + cancellationToken.ThrowIfCancellationRequested(); + lock (_lock) { PurgeBrokerHistory(); return Task.FromResult(MatchingJobs(query).LongCount()); } } private void EnsureCapacity() { + PurgeBrokerHistory(); PurgeDeduplication(); if (_activeJobs >= _options.MaxActiveJobs) throw CapacityExceeded("active jobs", _options.MaxActiveJobs); @@ -520,7 +603,7 @@ private void ValidatePayload(long bytes) throw new JobException($"Payload exceeds the configured {_options.MaxPayloadBytes} byte limit."); } - private static bool IsActive(JobState state) => state.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing; + private static bool IsActive(JobState state) => state.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing or JobStatus.RetryPending or JobStatus.EnqueueUnknown; private void StoreJob(JobState state) { @@ -529,6 +612,12 @@ private void StoreJob(JobState state) _activeJobs += (active ? 1 : 0) - (previous is not null && IsActive(previous) ? 1 : 0); if (!active && state.CompletedUtc is null) state = state with { CompletedUtc = _timeProvider.GetUtcNow() }; + if (state.ExecutionOwner == JobExecutionOwner.Broker && state.HistoryRetention is { } retention) + { + if (previous?.HistoryExpiresUtc is { } previousExpiry) _brokerExpiry.Remove((previousExpiry, state.JobId)); + state = state with { HistoryExpiresUtc = _timeProvider.GetUtcNow().Add(retention) }; + _brokerExpiry.Add((state.HistoryExpiresUtc.Value, state.JobId)); + } _jobs[state.JobId] = state; if (active) _active[state.JobId] = state; else _active.Remove(state.JobId); @@ -538,8 +627,11 @@ private void StoreJob(JobState state) { var completed = state.CompletedUtc!.Value; var expires = completed.Add(_options.DeduplicationRetention); - _deduplication[state.JobId] = expires; - _deduplicationExpiry.Enqueue((state.JobId, expires), expires); + if (state.ExecutionOwner == JobExecutionOwner.Runtime) + { + _deduplication[state.JobId] = expires; + _deduplicationExpiry.Enqueue((state.JobId, expires), expires); + } _history.Enqueue((state.JobId, completed), completed); TrimHistory(1000, pressureOnly: true); } @@ -562,11 +654,19 @@ private int TrimHistory(int limit, bool pressureOnly = false) var cutoff = _timeProvider.GetUtcNow().Subtract(_options.HistoryRetention); while (removed < limit && _history.TryPeek(out var item, out var completed)) { + if (!_jobs.TryGetValue(item.Id, out var retained) || retained.CompletedUtc != item.Completed) + { + _history.Dequeue(); + continue; + } if (_jobs.Count - _activeJobs <= _options.MaxHistoryJobs && (pressureOnly || completed > cutoff)) break; _history.Dequeue(); - if (_jobs.TryGetValue(item.Id, out var state) && !IsActive(state) && state.CompletedUtc == item.Completed && _jobs.TryRemove(item.Id, out _)) + if (!IsActive(retained)) + { + ForgetJob(retained); removed++; + } } return removed; } @@ -575,7 +675,10 @@ public Task GetStatsAsync(CancellationToken cancellationTo { cancellationToken.ThrowIfCancellationRequested(); lock (_lock) + { + PurgeBrokerHistory(); return Task.FromResult(new JobRuntimeStoreStats(_activeJobs, _jobs.Count - _activeJobs, _deduplication.Count, _dispatches.Count)); + } } public Task CleanupAsync(int limit = 1000, CancellationToken cancellationToken = default) @@ -585,9 +688,10 @@ public Task CleanupAsync(int limit = 1000, CancellationToken cancellationTo cancellationToken.ThrowIfCancellationRequested(); lock (_lock) { + PurgeBrokerHistory(); PurgeDeduplication(); var now = _timeProvider.GetUtcNow(); - foreach (var state in _jobs.Values.Where(s => s.RequiredNodeId is not null && s.Attempt == 0 && s.Status is JobStatus.Queued or JobStatus.Scheduled && s.ExpiresUtc <= now).Take(limit)) + foreach (var state in _jobs.Values.Where(s => s.ExecutionOwner == JobExecutionOwner.Runtime && s.RequiredNodeId is not null && s.Attempt == 0 && s.Status is JobStatus.Queued or JobStatus.Scheduled && s.ExpiresUtc <= now).Take(limit)) StoreJob(state with { Status = JobStatus.Cancelled, CompletedUtc = now, LastUpdatedUtc = now, ResultMessage = "Unclaimed per-node occurrence expired." }); return Task.FromResult(TrimHistory(limit)); } @@ -599,8 +703,8 @@ public Task RequestCancellationAsync(string jobId, CancellationToken cance return Task.FromResult(UpdateJob(jobId, state => state with { CancellationRequested = true, - Status = state.Status is JobStatus.Queued or JobStatus.Scheduled ? JobStatus.Cancelled : state.Status, - CompletedUtc = state.Status is JobStatus.Queued or JobStatus.Scheduled ? _timeProvider.GetUtcNow() : state.CompletedUtc, + Status = state.ExecutionOwner == JobExecutionOwner.Runtime && state.Status is JobStatus.Queued or JobStatus.Scheduled ? JobStatus.Cancelled : state.Status, + CompletedUtc = state.ExecutionOwner == JobExecutionOwner.Runtime && state.Status is JobStatus.Queued or JobStatus.Scheduled ? _timeProvider.GetUtcNow() : state.CompletedUtc, LastUpdatedUtc = _timeProvider.GetUtcNow() })); } @@ -608,7 +712,11 @@ public Task RequestCancellationAsync(string jobId, CancellationToken cance public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - return Task.FromResult(_jobs.TryGetValue(jobId, out var state) && state.CancellationRequested); + lock (_lock) + { + PurgeBrokerHistory(); + return Task.FromResult(_jobs.TryGetValue(jobId, out var state) && state.CancellationRequested); + } } public Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default) @@ -692,8 +800,11 @@ private bool UpdateJob(string jobId, Func update) { lock (_lock) { + PurgeBrokerHistory(); if (!_jobs.TryGetValue(jobId, out var current)) return false; + if (current.ExecutionOwner == JobExecutionOwner.Broker && !IsActive(current)) + return false; StoreJob(update(current)); return true; diff --git a/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs index 61df4a898..e404ae6f3 100644 --- a/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs +++ b/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs @@ -1,3 +1,4 @@ +using Foundatio.Jobs; using System; using System.Collections.Generic; using System.Linq; @@ -52,7 +53,7 @@ public interface IMessageExecutionStore Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default); /// Reads hourly operational counters within the requested time window. - Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default); + Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default); /// Reads executions by status in descending creation order. Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default); diff --git a/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs index 97394c4d2..f39bbb80c 100644 --- a/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs +++ b/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs @@ -1,3 +1,4 @@ +using Foundatio.Jobs; using System; using System.Collections.Generic; using System.Linq; @@ -174,7 +175,7 @@ public Task IncrementCounterAsync(string queueName, string counterName, long val return Task.CompletedTask; } - public Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) + public Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) { var now = _timeProvider.GetUtcNow(); var effectiveWindow = window ?? TimeSpan.FromHours(24); @@ -182,7 +183,7 @@ public Task GetCounterStatsAsync(string queueName, Tim var endHour = TruncateToHour(now); var totals = new Dictionary(); - var buckets = new List(); + var buckets = new List(); for (var hour = startHour; hour <= endHour; hour = hour.AddHours(1)) { @@ -198,10 +199,10 @@ public Task GetCounterStatsAsync(string queueName, Tim } } - buckets.Add(new MessageExecutionCounterBucket { Hour = hour, Counters = counters }); + buckets.Add(new JobCounterBucket { Hour = hour, Counters = counters }); } - return Task.FromResult(new MessageExecutionCounters { Totals = totals, Buckets = buckets }); + return Task.FromResult(new JobCounterStats { Totals = totals, Buckets = buckets }); } private static string GetBucketKey(string queueName, DateTimeOffset timestamp) diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs index dde1942d5..1a4b0e0aa 100644 --- a/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionOptions.cs @@ -21,8 +21,6 @@ public sealed record MessageExecutionOptions public bool TrackProgress { get; init; } /// Header containing the producer-created execution identifier. public string ExecutionIdHeader { get; init; } = "message.execution.id"; - /// Retention refreshed by execution state updates. - public TimeSpan StateRetention { get; init; } = TimeSpan.FromHours(24); /// Interval between cooperative cancellation checks and execution heartbeats. public TimeSpan CancellationPollInterval { get; init; } = TimeSpan.FromSeconds(5); /// Process identity recorded when an attempt starts. diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs index 3fae644fc..8bc0cef5f 100644 --- a/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs +++ b/src/Foundatio/Messaging/Tracking/MessageExecutionPipeline.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using Foundatio.Jobs; using Foundatio.Utility; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -9,48 +10,60 @@ namespace Foundatio.Messaging; /// -/// Executes a broker delivery with optional progress, cancellation, history, and per-attempt state fencing. -/// The message bus remains the sole owner of receiving and delivery leases; this pipeline creates no runnable jobs. +/// Executes broker deliveries with optional job progress, cancellation, and history. +/// The broker owns delivery leases and retries; the job store records each attempt without scheduling it. /// public sealed class MessageExecutionPipeline { private static readonly TimeSpan OperationTimeout = TimeSpan.FromSeconds(30); private readonly MessageExecutionOptions _options; - private readonly IMessageExecutionStore? _store; + private readonly IJobRuntimeStore? _store; private readonly TimeProvider _time; private readonly ILogger _logger; - public MessageExecutionPipeline(MessageExecutionOptions options, IMessageExecutionStore? store = null, TimeProvider? timeProvider = null, ILogger? logger = null) + public MessageExecutionPipeline(MessageExecutionOptions options, IJobRuntimeStore? store = null, TimeProvider? timeProvider = null, ILogger? logger = null) { ArgumentNullException.ThrowIfNull(options); ArgumentException.ThrowIfNullOrWhiteSpace(options.QueueName); ArgumentOutOfRangeException.ThrowIfEqual(options.MaxAttempts, 0); ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.CancellationPollInterval, TimeSpan.Zero); - ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.StateRetention, TimeSpan.Zero); if (options.TrackProgress && store is null) - throw new ArgumentException("Execution tracking requires an IMessageExecutionStore.", nameof(store)); + throw new ArgumentException("Execution tracking requires a job store. Configure AddFoundatio().Jobs.UseInMemory() or Jobs.UseRedis().", nameof(store)); _options = options; _store = store; _time = timeProvider ?? TimeProvider.System; _logger = logger ?? NullLogger.Instance; } - /// Runs application processing and persists only confirmed settlement outcomes. + /// Processes a delivery, recording terminal job state only after confirmed broker settlement. public async Task ProcessAsync(IMessageContext delivery, Func> handler, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(delivery); ArgumentNullException.ThrowIfNull(handler); - string? jobId = _options.TrackProgress && _store is not null ? delivery.Headers.GetValueOrDefault(_options.ExecutionIdHeader) : null; - if (_options.MaxAttempts >= 0 && delivery.Attempts > _options.MaxAttempts) + string? jobId = _options.TrackProgress ? delivery.Headers.GetValueOrDefault(_options.ExecutionIdHeader) : null; + JobState? attempt = null; + if (jobId is not null) { - await DeadLetterAsync(delivery, jobId, $"Exceeded max attempts ({_options.MaxAttempts})").AnyContext(); - return; + attempt = await RunAsync(ct => _store!.BeginBrokerAttemptAsync(jobId, delivery.Attempts, _options.WorkerId, ct), cancellationToken).AnyContext(); + if (attempt is null) + { + var state = await RunAsync(ct => _store!.GetAsync(jobId, ct), cancellationToken).AnyContext(); + if (state is not null) + { + if (state.Status is JobStatus.Completed or JobStatus.Failed or JobStatus.Cancelled or JobStatus.DeadLettered) + await SettleAsync(delivery.CompleteAsync, delivery).AnyContext(); + else + await SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = _options.RetryBackoff(delivery.Attempts) }, ct), delivery).AnyContext(); + return; + } + _logger.LogWarning("Job history {JobId} expired before delivery; processing continues without retained history", jobId); + } } using var processing = CancellationTokenSource.CreateLinkedTokenSource(delivery.CancellationToken, cancellationToken); var token = processing.Token; - Task? poll = null; long started = Stopwatch.GetTimestamp(); + Task? poll = attempt is null ? null : PollCancellationAsync(attempt, processing); var context = new MessageProcessingContext { Body = delivery.Body, @@ -70,142 +83,107 @@ public async Task ProcessAsync(IMessageContext delivery, Func { await delivery.RenewLockAsync(_options.VisibilityTimeout, ct).AnyContext(); - if (jobId is not null) - await UpdateAsync(t => _store!.HeartbeatAsync(jobId, t, delivery.Attempts, _options.StateRetention), ct).AnyContext(); + if (attempt is not null) + await RecordAsync(t => _store!.HeartbeatJobAsync(attempt.JobId, attempt.ClaimToken!, t), ct).AnyContext(); }, - OnReportDetailedProgress = jobId is null ? null : async (percent, message, ct) => + OnReportDetailedProgress = attempt is null ? null : async (percent, message, ct) => { - if (await IsCancelledAsync(jobId, ct).AnyContext()) + if (await IsCancelledAsync(attempt.JobId, ct).AnyContext()) throw new OperationCanceledException("Job cancellation was requested."); - await UpdateAsync(t => _store!.UpdateJobProgressAsync(jobId, Math.Clamp(percent, 0, 100), message, _options.StateRetention, t, delivery.Attempts), ct).AnyContext(); - await UpdateAsync(t => _store!.HeartbeatAsync(jobId, t, delivery.Attempts, _options.StateRetention), ct).AnyContext(); + await RecordAsync(t => _store!.ReportJobProgressAsync(attempt.JobId, attempt.ClaimToken!, Math.Clamp(percent, 0, 100), message, t), ct).AnyContext(); } }; + JobCompletion completion = new() { Kind = JobCompletionKind.Interrupted }; try { - if (jobId is not null) + if (attempt?.CancellationRequested == true) { - if (await IsCancelledAsync(jobId, token).AnyContext()) - { - if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) - await StatusAsync(jobId, MessageExecutionStatus.Cancelled, delivery.Attempts).AnyContext(); - return; - } - poll = PollCancellationAsync(jobId, delivery.Attempts, processing); - bool accepted = await RunAsync(ct => _store!.UpdateJobStatusAsync(jobId, MessageExecutionStatus.Processing, - startedUtc: _time.GetUtcNow(), attempt: delivery.Attempts, expiry: _options.StateRetention, cancellationToken: ct, workerId: _options.WorkerId), token).AnyContext(); - if (!accepted) - { - var state = await RunAsync(ct => _store!.GetJobStateAsync(jobId, ct), token).AnyContext(); - if (state?.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) - await SettleAsync(delivery.CompleteAsync, delivery).AnyContext(); - else if (state is not null) - await RetryAsync(delivery).AnyContext(); - // Tracking retention must not become a second delivery scheduler. Expired history - // does not prevent broker-owned work from running; state updates become no-ops. - if (state is not null) return; - _logger.LogWarning("Execution history {JobId} expired before delivery; processing continues without retained history", jobId); - } + if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) + completion = new() { Kind = JobCompletionKind.Cancelled }; } - - var outcome = await handler(context, token).AnyContext(); - if (!context.IsCompleted && !context.IsAbandoned) + else if (_options.MaxAttempts > 0 && delivery.Attempts > _options.MaxAttempts) + completion = await DeadLetterAsync(delivery, $"Exceeded max attempts ({_options.MaxAttempts})").AnyContext(); + else { - if (outcome.Kind == MessageOutcomeKind.Retry) + var outcome = await handler(context, token).AnyContext(); + if (!context.IsCompleted && !context.IsAbandoned) { - await FailureAsync(delivery, jobId, outcome.Reason ?? "Processing failed", Stopwatch.GetElapsedTime(started)).AnyContext(); - return; - } - if (outcome.Kind == MessageOutcomeKind.DeadLetter) - { - await DeadLetterAsync(delivery, jobId, outcome.Reason ?? "Processing rejected").AnyContext(); - return; + if (outcome.Kind == MessageOutcomeKind.Retry) + completion = await FailureAsync(delivery, outcome.Reason ?? "Processing failed").AnyContext(); + else if (outcome.Kind == MessageOutcomeKind.DeadLetter) + completion = await DeadLetterAsync(delivery, outcome.Reason ?? "Processing rejected").AnyContext(); + else if (_options.AutoComplete && outcome.Kind != MessageOutcomeKind.Unsettled) + { + token.ThrowIfCancellationRequested(); + await SettleAsync(context.CompleteAsync, delivery).AnyContext(); + } } } - if (context.IsAbandoned) return; - token.ThrowIfCancellationRequested(); - if (_options.AutoComplete && !context.IsCompleted && outcome.Kind != MessageOutcomeKind.Unsettled) - await SettleAsync(context.CompleteAsync, delivery).AnyContext(); - if (!context.IsCompleted) - await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, "Handler finished without confirmed acknowledgment; delivery may recur.").AnyContext(); } catch (OperationCanceledException) when (delivery.IsLeaseLost) { } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { if (!context.IsCompleted && !context.IsAbandoned) - { await SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.Zero }, ct), delivery).AnyContext(); - await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts).AnyContext(); - } } catch (OperationCanceledException) { - if (context.IsCompleted || context.IsAbandoned) return; - if (jobId is not null && await IsCancelledAsync(jobId, CancellationToken.None).AnyContext()) + if (!context.IsCompleted && !context.IsAbandoned) { - if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) - await StatusAsync(jobId, MessageExecutionStatus.Cancelled, delivery.Attempts).AnyContext(); + if (attempt is not null && await IsCancelledAsync(attempt.JobId, CancellationToken.None).AnyContext()) + { + if (await SettleAsync(delivery.CompleteAsync, delivery).AnyContext()) + completion = new() { Kind = JobCompletionKind.Cancelled }; + } + else completion = await FailureAsync(delivery, "Processing was cancelled").AnyContext(); } - else - await FailureAsync(delivery, jobId, "Processing was cancelled", Stopwatch.GetElapsedTime(started)).AnyContext(); } catch (Exception exception) { _logger.LogError(exception, "Processing failed for {MessageId} at {Queue} on attempt {Attempt}", delivery.Id, _options.QueueName, delivery.Attempts); if (!delivery.IsLeaseLost && !context.IsCompleted && !context.IsAbandoned) - await FailureAsync(delivery, jobId, exception.Message, Stopwatch.GetElapsedTime(started)).AnyContext(); + completion = await FailureAsync(delivery, exception.Message).AnyContext(); } finally { await processing.CancelAsync().AnyContext(); if (poll is not null) await poll.AnyContext(); - if (context.IsCompleted) + if (context.IsCompleted) completion = new() { Kind = JobCompletionKind.Succeeded }; + if (attempt is not null) + await RecordAsync(ct => _store!.CompleteJobAsync(attempt.JobId, attempt.ClaimToken!, completion, ct)).AnyContext(); + var outcome = completion.Kind switch + { + JobCompletionKind.Succeeded => MessageOutcomeKind.Success, + JobCompletionKind.Failed when !completion.Retryable => MessageOutcomeKind.DeadLetter, + JobCompletionKind.Failed => MessageOutcomeKind.Retry, + _ => (MessageOutcomeKind?)null + }; + if (outcome is { } kind) { - _options.OnProcessed?.Invoke(MessageOutcomeKind.Success, Stopwatch.GetElapsedTime(started)); - if (jobId is not null) - await UpdateAsync(ct => _store!.UpdateJobStatusAsync(jobId, MessageExecutionStatus.Completed, attempt: delivery.Attempts, - completedUtc: _time.GetUtcNow(), progress: 100, expiry: _options.StateRetention, cancellationToken: ct)).AnyContext(); - await CounterAsync("processed").AnyContext(); + _options.OnProcessed?.Invoke(kind, Stopwatch.GetElapsedTime(started)); + if (_store is not null) + await RecordAsync(ct => _store.IncrementCounterAsync(_options.QueueName, kind == MessageOutcomeKind.Success ? "processed" : kind == MessageOutcomeKind.DeadLetter ? "dead_lettered" : "failed", 1, ct)).AnyContext(); } - else if (context.IsAbandoned) - await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts).AnyContext(); } } - private async Task FailureAsync(IMessageContext delivery, string? jobId, string reason, TimeSpan elapsed) + private async Task FailureAsync(IMessageContext delivery, string reason) { if (_options.AutoComplete && _options.MaxAttempts > 0 && delivery.Attempts >= _options.MaxAttempts) - await DeadLetterAsync(delivery, jobId, reason).AnyContext(); - else - { - if (_options.AutoComplete) await RetryAsync(delivery).AnyContext(); - await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, reason).AnyContext(); - } - _options.OnProcessed?.Invoke(MessageOutcomeKind.Retry, elapsed); - await CounterAsync("failed").AnyContext(); + return await DeadLetterAsync(delivery, reason).AnyContext(); + if (_options.AutoComplete) + await SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = _options.RetryBackoff(delivery.Attempts) }, ct), delivery).AnyContext(); + return new() { Kind = JobCompletionKind.Failed, Error = reason }; } - private async Task DeadLetterAsync(IMessageContext delivery, string? jobId, string reason) - { - if (!await SettleAsync(ct => MessageOutcome.DeadLetter(reason).SettleFailureAsync(delivery, _options.MaxAttempts, _options.RetryBackoff, ct), delivery).AnyContext()) - { - await StatusAsync(jobId, MessageExecutionStatus.RetryPending, delivery.Attempts, reason).AnyContext(); - return; - } - _options.OnProcessed?.Invoke(MessageOutcomeKind.DeadLetter, TimeSpan.Zero); - await StatusAsync(jobId, MessageExecutionStatus.Failed, delivery.Attempts, reason).AnyContext(); - await CounterAsync("dead_lettered").AnyContext(); - } + private async Task DeadLetterAsync(IMessageContext delivery, string reason) + => await SettleAsync(ct => MessageOutcome.DeadLetter(reason).SettleFailureAsync(delivery, _options.MaxAttempts, _options.RetryBackoff, ct), delivery).AnyContext() + ? new() { Kind = JobCompletionKind.Failed, Retryable = false, Error = reason } + : new() { Kind = JobCompletionKind.Interrupted, Error = reason }; - private Task RetryAsync(IMessageContext delivery) => SettleAsync(ct => delivery.RejectAsync(new RejectOptions { RedeliveryDelay = _options.RetryBackoff(delivery.Attempts) }, ct), delivery); private Task IsCancelledAsync(string jobId, CancellationToken token) => RunAsync(ct => _store!.IsCancellationRequestedAsync(jobId, ct), token); - private Task CounterAsync(string name) => _store is null ? Task.CompletedTask : UpdateAsync(ct => _store.IncrementCounterAsync(_options.QueueName, name, 1, ct)); - private Task StatusAsync(string? jobId, MessageExecutionStatus status, int attempt, string? error = null) - => jobId is null ? Task.CompletedTask : UpdateAsync(ct => _store!.UpdateJobStatusAsync(jobId, status, attempt: attempt, - completedUtc: status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled ? _time.GetUtcNow() : null, - errorMessage: error, expiry: _options.StateRetention, cancellationToken: ct)); - private async Task PollCancellationAsync(string jobId, int attempt, CancellationTokenSource processing) + private async Task PollCancellationAsync(JobState attempt, CancellationTokenSource processing) { var token = processing.Token; while (!token.IsCancellationRequested) @@ -213,15 +191,15 @@ private async Task PollCancellationAsync(string jobId, int attempt, Cancellation try { await Task.Delay(_options.CancellationPollInterval, _time, token).AnyContext(); - if (await IsCancelledAsync(jobId, token).AnyContext()) + if (await IsCancelledAsync(attempt.JobId, token).AnyContext()) { await processing.CancelAsync().AnyContext(); return; } - await UpdateAsync(ct => _store!.HeartbeatAsync(jobId, ct, attempt, _options.StateRetention), token).AnyContext(); + await RecordAsync(ct => _store!.HeartbeatJobAsync(attempt.JobId, attempt.ClaimToken!, ct), token).AnyContext(); } catch (OperationCanceledException) when (token.IsCancellationRequested) { return; } - catch (Exception exception) { _logger.LogWarning(exception, "Unable to poll execution {JobId}; retrying", jobId); } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to poll job {JobId}; retrying", attempt.JobId); } } } @@ -234,24 +212,28 @@ private async Task SettleAsync(Func operation, IM return false; } } - private Task UpdateAsync(Func> operation, CancellationToken cancellationToken = default) - => UpdateAsync(async ct => { _ = await operation(ct).AnyContext(); }, cancellationToken); - private async Task UpdateAsync(Func operation, CancellationToken cancellationToken = default) + private Task RecordAsync(Func> operation, CancellationToken cancellationToken = default) + => RecordAsync(async ct => { _ = await operation(ct).AnyContext(); }, cancellationToken); + + private async Task RecordAsync(Func operation, CancellationToken cancellationToken = default) { try { await RunAsync(operation, cancellationToken).AnyContext(); } - catch (Exception exception) { _logger.LogWarning(exception, "Unable to update execution state at {Queue}", _options.QueueName); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception exception) { _logger.LogWarning(exception, "Unable to persist job history for {Queue}", _options.QueueName); } } + private async Task RunAsync(Func operation, CancellationToken cancellationToken = default) { - using var timeout = new CancellationTokenSource(OperationTimeout, _time); - using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + using var deadline = new CancellationTokenSource(OperationTimeout, _time); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, deadline.Token); await operation(linked.Token).WaitAsync(linked.Token).AnyContext(); } + private async Task RunAsync(Func> operation, CancellationToken cancellationToken = default) { - using var timeout = new CancellationTokenSource(OperationTimeout, _time); - using var linked = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, cancellationToken); + using var deadline = new CancellationTokenSource(OperationTimeout, _time); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, deadline.Token); return await operation(linked.Token).WaitAsync(linked.Token).AnyContext(); } } diff --git a/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs b/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs index 55de4457b..375fb1b74 100644 --- a/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs +++ b/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs @@ -78,6 +78,15 @@ private sealed class LeaseFailingStore : IJobRuntimeStore public bool DenyRenewals { get; set; } public bool ThrowOnRenewals { get; set; } + public bool IsShared => _inner.IsShared; + public Task CountAsync(JobQuery query, CancellationToken cancellationToken = default) => _inner.CountAsync(query, cancellationToken); + public Task BeginBrokerAttemptAsync(string jobId, int attempt, string nodeId, CancellationToken cancellationToken = default) => _inner.BeginBrokerAttemptAsync(jobId, attempt, nodeId, cancellationToken); + public Task MarkEnqueueUnknownAsync(string jobId, string error, CancellationToken cancellationToken = default) => _inner.MarkEnqueueUnknownAsync(jobId, error, cancellationToken); + public Task HeartbeatJobAsync(string jobId, string claimToken, CancellationToken cancellationToken = default) => _inner.HeartbeatJobAsync(jobId, claimToken, cancellationToken); + public Task RemoveAsync(string jobId, CancellationToken cancellationToken = default) => _inner.RemoveAsync(jobId, cancellationToken); + public Task IncrementCounterAsync(string name, string counterName, long value = 1, CancellationToken cancellationToken = default) => _inner.IncrementCounterAsync(name, counterName, value, cancellationToken); + public Task GetCounterStatsAsync(string name, TimeSpan? window = null, CancellationToken cancellationToken = default) => _inner.GetCounterStatsAsync(name, window, cancellationToken); + public Task ScheduleAsync(ScheduledJobDefinition definition, CancellationToken ct = default) => _inner.ScheduleAsync(definition, ct); public Task ReconcileAsync(ScheduledJobDefinition definition, CancellationToken ct = default) => _inner.ReconcileAsync(definition, ct); public Task GetScheduleAsync(string name, CancellationToken ct = default) => _inner.GetScheduleAsync(name, ct); diff --git a/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs b/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs index 1053358b0..77618c6c7 100644 --- a/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs +++ b/tests/Foundatio.Tests/Messaging/MessageEndpointPolicyTests.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Messaging; +using Foundatio.Jobs; using Moq; using Microsoft.Extensions.Time.Testing; using Xunit; @@ -14,7 +15,7 @@ public class MessageEndpointPolicyTests [Fact] public async Task ExpiredTrackingHistory_DoesNotPreventBrokerWorkFromRunning() { - var store = new InMemoryMessageExecutionStore(); + var store = new InMemoryJobRuntimeStore(); var delivery = new Mock(); delivery.SetupGet(value => value.Headers).Returns(MessageHeaders.Create(new Dictionary { [ExecutionHeaders.ExecutionId] = "expired" })); delivery.SetupGet(value => value.Attempts).Returns(1); @@ -28,7 +29,7 @@ await pipeline.ProcessAsync(delivery.Object, (_, _) => }, TestContext.Current.CancellationToken); Assert.True(invoked); delivery.Verify(value => value.CompleteAsync(It.IsAny()), Times.Once); - Assert.Null(await store.GetJobStateAsync("expired", TestContext.Current.CancellationToken)); + Assert.Null(await store.GetAsync("expired", TestContext.Current.CancellationToken)); } [Fact] @@ -57,16 +58,21 @@ public async Task ManualRenewal_ExtendsTheSupervisedDeadlineWhenAutoRenewIsDisab Assert.True(delivery.IsLeaseLost); } - [Fact] - public async Task FailedAcknowledgment_DoesNotPersistCompletion() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task FailedAcknowledgment_DoesNotPersistCompletion(bool cancellationRequested) { - var store = new InMemoryMessageExecutionStore(); - await store.SetJobStateAsync(new MessageExecutionState + var store = new InMemoryJobRuntimeStore(); + await store.CreateIfAbsentAsync(new JobState { JobId = "ack-failure", + Name = "exports", + ExecutionOwner = JobExecutionOwner.Broker, QueueName = "exports", - MessageType = "Export", - Status = MessageExecutionStatus.Queued, + PayloadType = "Export", + CancellationRequested = cancellationRequested, + Status = JobStatus.Queued, CreatedUtc = DateTimeOffset.UtcNow, LastUpdatedUtc = DateTimeOffset.UtcNow }, cancellationToken: TestContext.Current.CancellationToken); @@ -76,7 +82,7 @@ await store.SetJobStateAsync(new MessageExecutionState delivery.Setup(value => value.CompleteAsync(It.IsAny())).ThrowsAsync(new TimeoutException("Unknown broker acknowledgment")); var pipeline = new MessageExecutionPipeline(new MessageExecutionOptions { QueueName = "exports", TrackProgress = true }, store); await pipeline.ProcessAsync(delivery.Object, (_, _) => ValueTask.FromResult(MessageOutcome.Success), TestContext.Current.CancellationToken); - Assert.Equal(MessageExecutionStatus.RetryPending, (await store.GetJobStateAsync("ack-failure", TestContext.Current.CancellationToken))!.Status); + Assert.Equal(JobStatus.RetryPending, (await store.GetAsync("ack-failure", TestContext.Current.CancellationToken))!.Status); } [Fact] From 823972fc817deb527a49f0fa02ffa9c22d05ca3f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 7 Sep 2026 21:10:03 -0500 Subject: [PATCH 3/4] Avoid redundant locks in in-memory job history accounting --- src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs | 2 +- src/Foundatio/Jobs/JobRuntime.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs index 922dbce15..fb65bbf6b 100644 --- a/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs +++ b/src/Foundatio/Jobs/InMemoryJobRuntimeStore.Broker.cs @@ -72,7 +72,7 @@ public Task RemoveAsync(string jobId, CancellationToken cancellationToken private void ForgetJob(JobState state) { if (state.HistoryExpiresUtc is { } expires) _brokerExpiry.Remove((expires, state.JobId)); - _jobs.TryRemove(state.JobId, out _); + _jobs.Remove(state.JobId); if (IsActive(state)) _activeJobs--; _active.Remove(state.JobId); if (state.ExecutionOwner == JobExecutionOwner.Broker) _deduplication.Remove(state.JobId); diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index f969cc5da..c0915ac06 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -493,7 +493,7 @@ public interface IJobRuntimeStore : IJobMonitor, IScheduledDispatchStore, ISched public sealed partial class InMemoryJobRuntimeStore : IJobRuntimeStore { - private readonly ConcurrentDictionary _jobs = new(StringComparer.Ordinal); + private readonly Dictionary _jobs = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _dispatches = new(StringComparer.Ordinal); private readonly TimeProvider _timeProvider; private readonly object _lock = new(); @@ -691,7 +691,7 @@ public Task CleanupAsync(int limit = 1000, CancellationToken cancellationTo PurgeBrokerHistory(); PurgeDeduplication(); var now = _timeProvider.GetUtcNow(); - foreach (var state in _jobs.Values.Where(s => s.ExecutionOwner == JobExecutionOwner.Runtime && s.RequiredNodeId is not null && s.Attempt == 0 && s.Status is JobStatus.Queued or JobStatus.Scheduled && s.ExpiresUtc <= now).Take(limit)) + foreach (var state in _jobs.Values.Where(s => s.ExecutionOwner == JobExecutionOwner.Runtime && s.RequiredNodeId is not null && s.Attempt == 0 && s.Status is JobStatus.Queued or JobStatus.Scheduled && s.ExpiresUtc <= now).Take(limit).ToArray()) StoreJob(state with { Status = JobStatus.Cancelled, CompletedUtc = now, LastUpdatedUtc = now, ResultMessage = "Unclaimed per-node occurrence expired." }); return Task.FromResult(TrimHistory(limit)); } From 9288e40bfc7fb41a27697132a550c36080189a01 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 7 Sep 2026 21:31:17 -0500 Subject: [PATCH 4/4] Remove superseded broker execution stores and registrations --- .../Messaging/RedisMessageExecutionScripts.cs | 120 ------- .../Messaging/RedisMessageExecutionStore.cs | 339 ------------------ .../RedisMessageExecutionStoreOptions.cs | 46 --- .../RedisFoundatioBuilderExtensions.cs | 15 - .../MessageExecutionStoreConformanceTests.cs | 90 ----- src/Foundatio/FoundatioServicesExtensions.cs | 7 - .../Tracking/IMessageExecutionStore.cs | 63 ---- .../Tracking/InMemoryMessageExecutionStore.cs | 265 -------------- .../Tracking/MessageExecutionState.cs | 52 --- .../MessageExecutionStoreExtensions.cs | 24 -- .../RedisMessageExecutionStoreTests.cs | 12 - .../Messaging/MessageExecutionStoreTests.cs | 8 - 12 files changed, 1041 deletions(-) delete mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs delete mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs delete mode 100644 src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs delete mode 100644 src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs delete mode 100644 src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs delete mode 100644 src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs delete mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionState.cs delete mode 100644 src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs delete mode 100644 tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs delete mode 100644 tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs deleted file mode 100644 index ba72bef4d..000000000 --- a/src/Foundatio.Redis/Messaging/RedisMessageExecutionScripts.cs +++ /dev/null @@ -1,120 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -namespace Foundatio.Messaging; - -// A job hash, its cancellation flag, and its indexes change in one server operation. Index scores -// remain creation times for pagination; a separate index records actual server expiration deadlines. -internal static class RedisMessageExecutionScripts -{ - public const string IncrementCounter = """ - local value = redis.call('HINCRBY', KEYS[1], ARGV[1], ARGV[2]) - redis.call('PEXPIRE', KEYS[1], ARGV[3]) - return value - """; - - public const string Mutate = """ - local op, prefix, id = ARGV[1], ARGV[2], ARGV[3] - local key, ttl, expected, now = KEYS[1], tonumber(ARGV[4]), ARGV[5], ARGV[6] - local clock = redis.call('TIME') - local serverNow = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) - local function queueKey(queue) return prefix .. ':queues:' .. queue end - local function statusKey(queue, status) return queueKey(queue) .. ':status:' .. status end - local function terminal(status) return status == '2' or status == '3' or status == '4' end - local function removeIndexes(queue, job) - redis.call('ZREM', queueKey(queue), job) - redis.call('ZREM', queueKey(queue) .. ':expires', job) - for status = 0, 6 do redis.call('ZREM', statusKey(queue, status), job) end - end - local function cleanup(queue) - local expires = queueKey(queue) .. ':expires' - local members = redis.call('ZRANGEBYSCORE', expires, '-inf', serverNow, 'LIMIT', 0, 128) - for _, job in ipairs(members) do - local remaining = redis.call('PTTL', prefix .. ':' .. job) - if remaining == -2 then - removeIndexes(queue, job) - else - redis.call('ZADD', expires, remaining == -1 and 9007199254740991 or serverNow + math.max(1, remaining), job) - end - end - return #members - end - local function touchIndex(index, score, retention) - local existed = redis.call('EXISTS', index) == 1 - redis.call('ZADD', index, score, id) - if retention < 0 then - redis.call('PERSIST', index) - else - local remaining = redis.call('PTTL', index) - if not existed or (remaining >= 0 and remaining < retention) then - redis.call('PEXPIRE', index, retention) - end - end - end - local function refresh(queue, status) - local retention = ttl - if retention == -2 then retention = tonumber(redis.call('HGET', key, 'RetentionMs') or '-1') end - redis.call('HSET', key, 'RetentionMs', retention) - if retention < 0 then - redis.call('PERSIST', key) - redis.call('PERSIST', key .. ':cancel') - else - redis.call('PEXPIRE', key, retention) - redis.call('PEXPIRE', key .. ':cancel', retention) - end - local created = tonumber(redis.call('HGET', key, 'CreatedUtc') or '0') - touchIndex(queueKey(queue), created, retention) - touchIndex(statusKey(queue, status), created, retention) - touchIndex(queueKey(queue) .. ':expires', retention < 0 and 9007199254740991 or serverNow + retention, retention) - cleanup(queue) - end - if op == 'clean' then return cleanup(id) end - if op == 'prune' then - if redis.call('EXISTS', key) == 0 then removeIndexes(ARGV[7], id) end - return 1 - end - local oldQueue = redis.call('HGET', key, 'QueueName') - local oldStatus = redis.call('HGET', key, 'Status') - if op == 'remove' then - if oldQueue then removeIndexes(oldQueue, id) end - redis.call('DEL', key, key .. ':cancel') - return 1 - end - if op == 'set' then - if oldQueue then removeIndexes(oldQueue, id) end - redis.call('DEL', key, key .. ':cancel') - for i = 7, #ARGV, 2 do redis.call('HSET', key, ARGV[i], ARGV[i + 1]) end - refresh(redis.call('HGET', key, 'QueueName'), redis.call('HGET', key, 'Status')) - return 1 - end - if not oldQueue or terminal(oldStatus) then return 0 end - local attempt = tonumber(redis.call('HGET', key, 'Attempt') or '0') - if expected ~= '' and tonumber(expected) ~= attempt then return 0 end - if op == 'cancel' then - local remaining = redis.call('PTTL', key) - redis.call('SET', key .. ':cancel', '1') - if remaining >= 0 then redis.call('PEXPIRE', key .. ':cancel', math.max(1, remaining)) end - return 1 - end - if (op == 'progress' or op == 'heartbeat') and oldStatus ~= '1' then return 0 end - if op == 'status' then - local status, nextAttempt = oldStatus, nil - for i = 7, #ARGV, 2 do - if ARGV[i] == 'Status' then status = ARGV[i + 1] end - if ARGV[i] == 'Attempt' then nextAttempt = tonumber(ARGV[i + 1]) end - end - if nextAttempt and nextAttempt < attempt then return 0 end - if status == '1' and nextAttempt == attempt and (oldStatus == '1' or oldStatus == '5') then return 0 end - if status == '6' and oldStatus ~= '0' then return 0 end - if status ~= oldStatus then redis.call('ZREM', statusKey(oldQueue, oldStatus), id) end - if not terminal(status) then redis.call('HDEL', key, 'CompletedUtc') end - if status == '1' then redis.call('HDEL', key, 'ErrorMessage') end - end - for i = 7, #ARGV, 2 do redis.call('HSET', key, ARGV[i], ARGV[i + 1]) end - redis.call('HSET', key, 'LastUpdatedUtc', now) - refresh(oldQueue, redis.call('HGET', key, 'Status')) - return 1 - """; -} diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs deleted file mode 100644 index 01f8545b8..000000000 --- a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStore.cs +++ /dev/null @@ -1,339 +0,0 @@ -using Foundatio.Jobs; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Globalization; -using StackExchange.Redis; - -namespace Foundatio.Messaging; - -/// -/// Redis-backed implementation of . -/// Each job is stored as a Redis Hash. Per-queue and per-status sorted sets scored by creation time -/// index jobs for pagination. Cancellation uses a separate key that shares the job's TTL. -/// -/// -/// Key layout ({prefix} is , optionally preceded by -/// ): -/// -/// {prefix}:{jobId} — hash of job fields; metadata is stored as meta:{name} fields -/// {prefix}:{jobId}:cancel — cancellation flag -/// {prefix}:queues:{queueName} — sorted set of every job in the queue -/// {prefix}:queues:{queueName}:status:{status} — sorted set per value -/// {prefix}:counters:{queueName}:{yyyy-MM-ddTHH} — hourly counter hash -/// -/// Writes update hashes, cancellation flags, and indexes atomically using server-side scripts. -/// A separate expiration index uses Redis server deadlines; creation time never implies expiration. -/// For Redis Cluster, configure a common hash tag in KeyPrefix so a store's keys share a slot. -/// -public sealed class RedisMessageExecutionStore : IMessageExecutionStore -{ - /// - public bool IsShared => true; - - private const string MetadataFieldPrefix = "meta:"; - private static readonly TimeSpan JobCounterBucketRetention = TimeSpan.FromHours(48); - - private readonly IConnectionMultiplexer _redis; - private readonly RedisMessageExecutionStoreOptions _options; - private readonly TimeProvider _timeProvider; - private readonly string _keyPrefix; - - public RedisMessageExecutionStore(IConnectionMultiplexer redis, RedisMessageExecutionStoreOptions? options = null, TimeProvider? timeProvider = null) - { - _redis = redis; - _options = options ?? new RedisMessageExecutionStoreOptions(); - _timeProvider = timeProvider ?? TimeProvider.System; - _keyPrefix = string.IsNullOrEmpty(_options.ResourcePrefix) - ? _options.KeyPrefix - : $"{_options.ResourcePrefix}:{_options.KeyPrefix}"; - } - - public Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default) - => MutateAsync("set", state.JobId, ResolveTtl(expiry, state.Status), - BuildEntries(state, state.CreatedUtc.ToUnixTimeMilliseconds()), cancellationToken); - - public async Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default) - { - var entries = await _redis.GetDatabase().HashGetAllAsync(JobKey(jobId)).WaitAsync(cancellationToken).ConfigureAwait(false); - return entries.Length == 0 ? null : ParseJobState(entries); - } - - public Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null) - { - var updates = new List { new("Status", FormatStatus(status)) }; - if (startedUtc.HasValue) updates.Add(new("StartedUtc", FormatTimestamp(startedUtc.Value))); - if (IsTerminal(status)) updates.Add(new("CompletedUtc", FormatTimestamp(completedUtc ?? _timeProvider.GetUtcNow()))); - if (errorMessage is not null) updates.Add(new("ErrorMessage", errorMessage)); - if (progress.HasValue) updates.Add(new("Progress", FormatInt(progress.Value))); - if (attempt.HasValue) updates.Add(new("Attempt", FormatInt(attempt.Value))); - if (workerId is not null) updates.Add(new("WorkerId", workerId)); - if (status == MessageExecutionStatus.Processing) - { - updates.Add(new("Progress", FormatInt(progress ?? 0))); - updates.Add(new("ProgressMessage", string.Empty)); - updates.Add(new("LastHeartbeatUtc", FormatTimestamp(startedUtc ?? _timeProvider.GetUtcNow()))); - } - return MutateAsync("status", jobId, ResolveTtl(expiry, status), updates, cancellationToken); - } - - public Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null) - => MutateAsync("progress", jobId, ResolveTtl(expiry, MessageExecutionStatus.Processing), - [new("Progress", FormatInt(progress)), new("ProgressMessage", progressMessage ?? string.Empty)], cancellationToken, expectedAttempt); - - public Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null) - => MutateAsync("heartbeat", jobId, expiry is null ? null : ResolveTtl(expiry, MessageExecutionStatus.Processing), - [new("LastHeartbeatUtc", FormatTimestamp(_timeProvider.GetUtcNow()))], cancellationToken, expectedAttempt, preserveExpiry: expiry is null); - - public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) - => MutateAsync("cancel", jobId, null, [], cancellationToken); - - public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) - { - // The flag has the same expiration as its job and is created atomically with the status check. - return _redis.GetDatabase().KeyExistsAsync(CancelKey(jobId)).WaitAsync(cancellationToken); - } - - public Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default) - => MutateAsync("remove", jobId, null, [], cancellationToken); - - private async Task MutateAsync(string operation, string jobId, TimeSpan? expiry, IReadOnlyList fields, - CancellationToken cancellationToken, int? expectedAttempt = null, bool preserveExpiry = false) - => await EvaluateAsync(operation, jobId, expiry, fields, cancellationToken, expectedAttempt, preserveExpiry).ConfigureAwait(false) != 0; - - private async Task EvaluateAsync(string operation, string jobId, TimeSpan? expiry, IReadOnlyList fields, - CancellationToken cancellationToken, int? expectedAttempt = null, bool preserveExpiry = false) - { - cancellationToken.ThrowIfCancellationRequested(); - RedisValue[] args = new RedisValue[6 + fields.Count * 2]; - args[0] = operation; - args[1] = _keyPrefix; - args[2] = jobId; - args[3] = preserveExpiry ? -2L : expiry is { } ttl ? Math.Max(1L, (long)ttl.TotalMilliseconds) : -1L; - args[4] = expectedAttempt.HasValue ? FormatInt(expectedAttempt.Value) : string.Empty; - args[5] = FormatTimestamp(_timeProvider.GetUtcNow()); - for (int i = 0; i < fields.Count; i++) - { - args[6 + i * 2] = fields[i].Name; - args[7 + i * 2] = fields[i].Value; - } - return (long)await _redis.GetDatabase().ScriptEvaluateAsync(RedisMessageExecutionScripts.Mutate, [JobKey(jobId)], args) - .WaitAsync(cancellationToken).ConfigureAwait(false); - } - - private async Task CleanupAsync(string queueName, CancellationToken cancellationToken) - { - while (await EvaluateAsync("clean", queueName, null, [], cancellationToken).ConfigureAwait(false) == 128) - cancellationToken.ThrowIfCancellationRequested(); - } - - public Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default) - { - var db = _redis.GetDatabase(); - var bucketKey = JobCounterBucketKey(queueName, _timeProvider.GetUtcNow()); - - // Increment and retention refresh form one atomic server operation, without MULTI/EXEC overhead. - return db.ScriptEvaluateAsync(RedisMessageExecutionScripts.IncrementCounter, [bucketKey], - [counterName, value, (long)JobCounterBucketRetention.TotalMilliseconds]).WaitAsync(cancellationToken); - } - - public async Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) - { - var db = _redis.GetDatabase(); - var now = _timeProvider.GetUtcNow(); - var effectiveWindow = window ?? TimeSpan.FromHours(24); - var startHour = TruncateToHour(now - effectiveWindow); - var endHour = TruncateToHour(now); - - var hours = new List(); - for (var hour = startHour; hour <= endHour; hour = hour.AddHours(1)) - hours.Add(hour); - - var batch = db.CreateBatch(); - var tasks = new Task[hours.Count]; - for (int i = 0; i < hours.Count; i++) - tasks[i] = batch.HashGetAllAsync(JobCounterBucketKey(queueName, hours[i])); - batch.Execute(); - - var totals = new Dictionary(); - var buckets = new List(hours.Count); - - for (int i = 0; i < hours.Count; i++) - { - var entries = await tasks[i].WaitAsync(cancellationToken).ConfigureAwait(false); - var counters = new Dictionary(entries.Length); - - foreach (var entry in entries) - { - if (entry.Value.TryParse(out long val)) - { - var name = entry.Name.ToString(); - counters[name] = val; - totals[name] = totals.GetValueOrDefault(name) + val; - } - } - - buckets.Add(new JobCounterBucket { Hour = hours[i], Counters = counters }); - } - - return new JobCounterStats { Totals = totals, Buckets = buckets }; - } - - public async Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default) - { - if (take <= 0) - return []; - - var db = _redis.GetDatabase(); - await CleanupAsync(queueName, cancellationToken).ConfigureAwait(false); - var setKey = StatusSetKey(queueName, status); - var results = new List(take); - var dangling = new List(); - long cursor = Math.Max(skip, 0); - - // Keep paging past members whose hash has expired so the caller still gets a full page. - while (results.Count < take) - { - int wanted = take - results.Count; - var members = await db.SortedSetRangeByRankAsync(setKey, cursor, cursor + wanted - 1, Order.Descending).WaitAsync(cancellationToken).ConfigureAwait(false); - if (members.Length == 0) - break; - - cursor += members.Length; - - var batch = db.CreateBatch(); - var tasks = new Task[members.Length]; - for (int i = 0; i < members.Length; i++) - tasks[i] = batch.HashGetAllAsync(JobKey(members[i].ToString())); - batch.Execute(); - - for (int i = 0; i < tasks.Length; i++) - { - var entries = await tasks[i].WaitAsync(cancellationToken).ConfigureAwait(false); - if (entries.Length > 0) - { - var state = ParseJobState(entries); - if (state.Status == status && state.QueueName == queueName) - results.Add(state); - } - else - dangling.Add(members[i]); - } - - if (members.Length < wanted) - break; - } - - // Removal is deferred until after paging so ranks stay stable while reading. - foreach (var id in dangling) - await MutateAsync("prune", id.ToString(), null, [new(queueName, string.Empty)], cancellationToken).ConfigureAwait(false); - - return results; - } - - /// Counts current status-index entries after removing expired jobs using server deadlines. - public async Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default) - { - await CleanupAsync(queueName, cancellationToken).ConfigureAwait(false); - return await _redis.GetDatabase().SortedSetLengthAsync(StatusSetKey(queueName, status)).WaitAsync(cancellationToken).ConfigureAwait(false); - } - - private TimeSpan? ResolveTtl(TimeSpan? expiry, MessageExecutionStatus status) - { - var ttl = expiry ?? _options.DefaultExpiry; - if (ttl is null || IsTerminal(status)) - return ttl; - - return ttl.Value > _options.NonTerminalExpiry ? ttl : _options.NonTerminalExpiry; - } - - private static bool IsTerminal(MessageExecutionStatus status) - => status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled; - - private static HashEntry[] BuildEntries(MessageExecutionState state, long createdScore) - { - var entries = new List(12 + (state.Metadata?.Count ?? 0)) - { - new("JobId", state.JobId), - new("QueueName", state.QueueName), - new("MessageType", state.MessageType), - new("Status", FormatStatus(state.Status)), - new("Progress", FormatInt(state.Progress)), - new("ProgressMessage", state.ProgressMessage ?? string.Empty), - new("CreatedUtc", createdScore.ToString(CultureInfo.InvariantCulture)), - new("StartedUtc", state.StartedUtc is { } started ? FormatTimestamp(started) : string.Empty), - new("CompletedUtc", state.CompletedUtc is { } completed ? FormatTimestamp(completed) : string.Empty), - new("ErrorMessage", state.ErrorMessage ?? string.Empty), - new("Attempt", FormatInt(state.Attempt)), - new("WorkerId", state.WorkerId ?? string.Empty), - new("LastUpdatedUtc", FormatTimestamp(state.LastUpdatedUtc)), - new("LastHeartbeatUtc", state.LastHeartbeatUtc is { } heartbeat ? FormatTimestamp(heartbeat) : string.Empty) - }; - - if (state.Metadata is not null) - { - foreach (var kvp in state.Metadata) - entries.Add(new(MetadataFieldPrefix + kvp.Key, kvp.Value)); - } - - return entries.ToArray(); - } - - private static string FormatStatus(MessageExecutionStatus status) => ((int)status).ToString(CultureInfo.InvariantCulture); - private static string FormatInt(int value) => value.ToString(CultureInfo.InvariantCulture); - private static string FormatTimestamp(DateTimeOffset value) => value.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); - - private string JobKey(string jobId) => $"{_keyPrefix}:{jobId}"; - private string CancelKey(string jobId) => $"{_keyPrefix}:{jobId}:cancel"; - private string QueueSetKey(string queueName) => $"{_keyPrefix}:queues:{queueName}"; - private string StatusSetKey(string queueName, MessageExecutionStatus status) => $"{_keyPrefix}:queues:{queueName}:status:{(int)status}"; - private string JobCounterBucketKey(string queueName, DateTimeOffset timestamp) => $"{_keyPrefix}:counters:{queueName}:{TruncateToHour(timestamp):yyyy-MM-ddTHH}"; - - private static DateTimeOffset TruncateToHour(DateTimeOffset timestamp) - => new(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0, TimeSpan.Zero); - - private static MessageExecutionState ParseJobState(HashEntry[] entries) - { - var dict = new Dictionary(entries.Length); - Dictionary? metadata = null; - - foreach (var entry in entries) - { - var name = entry.Name.ToString(); - if (name.StartsWith(MetadataFieldPrefix, StringComparison.Ordinal)) - (metadata ??= new Dictionary())[name[MetadataFieldPrefix.Length..]] = entry.Value.ToString(); - else - dict[name] = entry.Value.ToString(); - } - - return new MessageExecutionState - { - JobId = dict.GetValueOrDefault("JobId") ?? string.Empty, - QueueName = dict.GetValueOrDefault("QueueName") ?? string.Empty, - MessageType = dict.GetValueOrDefault("MessageType") ?? string.Empty, - Status = int.TryParse(dict.GetValueOrDefault("Status"), out var s) ? (MessageExecutionStatus)s : MessageExecutionStatus.Queued, - Progress = int.TryParse(dict.GetValueOrDefault("Progress"), out var p) ? p : 0, - ProgressMessage = NullIfEmpty(dict.GetValueOrDefault("ProgressMessage")), - CreatedUtc = ParseDateTimeOffset(dict.GetValueOrDefault("CreatedUtc")), - StartedUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("StartedUtc")), - CompletedUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("CompletedUtc")), - ErrorMessage = NullIfEmpty(dict.GetValueOrDefault("ErrorMessage")), - Attempt = int.TryParse(dict.GetValueOrDefault("Attempt"), out var a) ? a : 0, - WorkerId = string.IsNullOrEmpty(dict.GetValueOrDefault("WorkerId")) ? null : dict["WorkerId"], - LastUpdatedUtc = ParseDateTimeOffset(dict.GetValueOrDefault("LastUpdatedUtc")), - LastHeartbeatUtc = ParseNullableDateTimeOffset(dict.GetValueOrDefault("LastHeartbeatUtc")), - Metadata = metadata - }; - } - - private static DateTimeOffset ParseDateTimeOffset(string? value) - => long.TryParse(value, out var ms) ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : DateTimeOffset.MinValue; - - private static DateTimeOffset? ParseNullableDateTimeOffset(string? value) - => string.IsNullOrEmpty(value) ? null : long.TryParse(value, out var ms) ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null; - - private static string? NullIfEmpty(string? value) - => string.IsNullOrEmpty(value) ? null : value; -} diff --git a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs b/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs deleted file mode 100644 index 34a31838a..000000000 --- a/src/Foundatio.Redis/Messaging/RedisMessageExecutionStoreOptions.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -namespace Foundatio.Messaging; - -/// -/// Options for configuring . -/// -public class RedisMessageExecutionStoreOptions -{ - /// - /// Key prefix for all Redis keys. Default is "fnd:executions". - /// - public string KeyPrefix { get; set; } = "fnd:executions"; - - /// - /// Optional prefix applied before for app-level scoping. - /// When set, all Redis keys become "{ResourcePrefix}:{KeyPrefix}:...". - /// When null or empty (default), only is used. - /// - /// - /// Use this to isolate multiple applications sharing the same Redis instance - /// (e.g., "myapp" produces keys like "myapp:fnd:executions:..."). - /// - public string? ResourcePrefix { get; set; } - - /// - /// TTL applied to a job's keys by any write whose caller passes no expiry. - /// Default is 24 hours. Set to null to disable auto-expiry for such writes. - /// - /// - /// The queue worker passes MessageExecutionOptions.StateRetention on every write, so this only - /// takes effect for direct callers of the store. Writes that leave a job - /// or are raised to at least . - /// - public TimeSpan? DefaultExpiry { get; set; } = TimeSpan.FromHours(24); - - /// - /// Minimum TTL for a job while it is or , - /// so a live job does not vanish before it reaches a terminal state. Default is 7 days. A longer caller-supplied - /// expiry still wins; a null effective expiry (no TTL) is left as is. - /// - public TimeSpan NonTerminalExpiry { get; set; } = TimeSpan.FromDays(7); -} diff --git a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs index a250768bc..fa8023614 100644 --- a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs +++ b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs @@ -64,21 +64,6 @@ public static FoundatioBuilder.MessagingBuilder UseRedis(this FoundatioBuilder.M return builder.UseTransport(sp => new RedisStreamsMessageTransport(sp.GetRequiredService())); } - /// Shares progress, cancellation and history for broker-delivered executions through Redis. - public static FoundatioBuilder.MessagingBuilder UseRedisExecutionTracking(this FoundatioBuilder.MessagingBuilder builder, - Action? configure = null, string? connectionString = null) - { - var services = ((IFoundatioBuilder)builder).Services; - EnsureConnection(services, connectionString); - services.AddSingleton(sp => - { - var options = new RedisMessageExecutionStoreOptions(); - configure?.Invoke(options); - return new RedisMessageExecutionStore(sp.GetRequiredService(), options, sp.GetService()); - }); - return builder; - } - /// Coordinates resources across workers with ownership-checked Redis locks. public static FoundatioBuilder UseRedis(this FoundatioBuilder.LockingBuilder builder, string keyPrefix = "fnd:locks:", string? connectionString = null) { diff --git a/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs deleted file mode 100644 index 9e61a546e..000000000 --- a/src/Foundatio.TestHarness/Messaging/MessageExecutionStoreConformanceTests.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Threading.Tasks; -using Foundatio.Messaging; -using Xunit; - -namespace Foundatio.Tests.Messaging; - -/// Atomic transition guarantees for optional broker-driven execution tracking. -public abstract class MessageExecutionStoreConformanceTests -{ - protected abstract IMessageExecutionStore? CreateStore(); - private IMessageExecutionStore Store() { var store = CreateStore(); Assert.SkipWhen(store is null, "Execution store is not configured."); return store!; } - private static MessageExecutionState Queued(string id) => new() - { - JobId = id, - QueueName = "exports", - MessageType = "Export", - Status = MessageExecutionStatus.Queued, - CreatedUtc = DateTimeOffset.UtcNow, - LastUpdatedUtc = DateTimeOffset.UtcNow - }; - - [Fact] - public async Task ADeliveryAttempt_CanStartOnlyOnce() - { - var store = Store(); - string id = Guid.NewGuid().ToString("N"); - await store.SetJobStateAsync(Queued(id)); - Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1, workerId: "first")); - Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1, workerId: "second")); - Assert.Equal("first", (await store.GetJobStateAsync(id))!.WorkerId); - } - - [Fact] - public async Task OlderAttempt_CannotOverwriteANewerAttempt() - { - var store = Store(); - string id = Guid.NewGuid().ToString("N"); - await store.SetJobStateAsync(Queued(id)); - Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1)); - Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.RetryPending, attempt: 1)); - Assert.True(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 2)); - await store.UpdateJobProgressAsync(id, 40, "current", expectedAttempt: 2); - await store.UpdateJobProgressAsync(id, 99, "stale", expectedAttempt: 1); - Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Completed, attempt: 1)); - var state = await store.GetJobStateAsync(id); - Assert.Equal(MessageExecutionStatus.Processing, state!.Status); - Assert.Equal(40, state.Progress); - Assert.Equal("current", state.ProgressMessage); - } - - [Fact] - public async Task RetryPending_IgnoresProgressFromTheFinishedAttempt() - { - var store = Store(); - string id = Guid.NewGuid().ToString("N"); - await store.SetJobStateAsync(Queued(id)); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.RetryPending, attempt: 1); - await store.UpdateJobProgressAsync(id, 80, "too late", expectedAttempt: 1); - Assert.Equal(0, (await store.GetJobStateAsync(id))!.Progress); - } - - [Fact] - public async Task Cancellation_SurvivesProgressAndStatusWrites() - { - var store = Store(); - string id = Guid.NewGuid().ToString("N"); - await store.SetJobStateAsync(Queued(id)); - Assert.True(await store.RequestCancellationAsync(id)); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); - await store.UpdateJobProgressAsync(id, 50, expectedAttempt: 1); - Assert.True(await store.IsCancellationRequestedAsync(id)); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Cancelled, attempt: 1); - Assert.False(await store.RequestCancellationAsync(id)); - } - - [Fact] - public async Task TerminalState_RejectsDelayedSendReconciliation() - { - var store = Store(); - string id = Guid.NewGuid().ToString("N"); - await store.SetJobStateAsync(Queued(id)); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 1); - await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Completed, attempt: 1, progress: 100); - Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.EnqueueUnknown, errorMessage: "Late send timeout")); - Assert.False(await store.UpdateJobStatusAsync(id, MessageExecutionStatus.Processing, attempt: 2)); - Assert.Equal(MessageExecutionStatus.Completed, (await store.GetJobStateAsync(id))!.Status); - } -} diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index a4fb4c75a..00b68b82c 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -362,13 +362,6 @@ public MessagingBuilder UseSchedulingStore(FuncTracks broker-delivered executions in this process. The broker remains the only source of runnable work. - public MessagingBuilder UseInMemoryExecutionTracking() - { - _services.ReplaceSingleton(sp => new InMemoryMessageExecutionStore(sp.GetService())); - return this; - } - public MessagingBuilder UseTransport(IMessageTransport transport) { ArgumentNullException.ThrowIfNull(transport); diff --git a/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs deleted file mode 100644 index e404ae6f3..000000000 --- a/src/Foundatio/Messaging/Tracking/IMessageExecutionStore.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Foundatio.Jobs; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -namespace Foundatio.Messaging; - -/// -/// Stores the state of tracked queue jobs. Implementations must be safe for concurrent use. -/// -public interface IMessageExecutionStore -{ - /// Whether jobs and cancellation requests are shared across processes. Decorators must forward this capability. - bool IsShared { get; } - - /// - /// Creates or replaces a job's state. Called once at enqueue time with . - /// - Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default); - - /// Gets retained execution state, or null when absent or expired. - Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default); - - /// - /// Updates a job's status and optional fields. Implementations should apply the change atomically - /// relative to other status updates for the same job. Returns false for a missing job, a terminal - /// job, an older attempt, or a repeated start of an attempt already waiting for retry. - /// Starting an attempt resets progress and its message, initializes the heartbeat, and records workerId when supplied. - /// Administrative replay creates a new job identity; SetJobStateAsync is an explicit administrative replacement. - /// - Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null); - - /// Updates progress only for a live processing attempt; stale attempts are ignored. - Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null); - - /// - /// Signals that processing is still alive. The execution pipeline polls and heartbeats - /// independently of the broker lease supervisor. Only the current processing attempt may update it. - /// - Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null); - - /// Requests cooperative cancellation; returns false for missing or terminal executions. - Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); - - /// Checks whether cooperative cancellation was requested. - Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default); - - /// Removes retained history and cancellation state; does not remove broker work. - Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default); - - /// Adds an operational counter to the current hourly bucket. - Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default); - - /// Reads hourly operational counters within the requested time window. - Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default); - - /// Reads executions by status in descending creation order. - Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default); - - /// Counts retained executions with the requested status. - Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default); -} diff --git a/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs b/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs deleted file mode 100644 index f39bbb80c..000000000 --- a/src/Foundatio/Messaging/Tracking/InMemoryMessageExecutionStore.cs +++ /dev/null @@ -1,265 +0,0 @@ -using Foundatio.Jobs; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Collections.Concurrent; - -namespace Foundatio.Messaging; - -/// -/// In-memory implementation of . -/// Suitable for development, testing, and single-node deployments. -/// Expired entries are lazily cleaned up on access. -/// -public sealed class InMemoryMessageExecutionStore : IMessageExecutionStore -{ - /// - public bool IsShared => false; - - private readonly object _gate = new(); - private readonly ConcurrentDictionary _jobs = new(); - private readonly ConcurrentDictionary _cancellations = new(); - private readonly ConcurrentDictionary> _counterBuckets = new(); - private readonly TimeProvider _timeProvider; - private int _accessCount; - - public InMemoryMessageExecutionStore(TimeProvider? timeProvider = null) - { - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public Task SetJobStateAsync(MessageExecutionState state, TimeSpan? expiry = null, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - var now = _timeProvider.GetUtcNow(); - var expiresAt = expiry.HasValue ? now + expiry.Value : DateTimeOffset.MaxValue; - - _cancellations.TryRemove(state.JobId, out _); - _jobs[state.JobId] = new JobEntry(state, expiresAt, expiry); - - CleanupIfNeeded(); - - return Task.CompletedTask; - - } - } - - public Task GetJobStateAsync(string jobId, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - if (!_jobs.TryGetValue(jobId, out var entry)) - return Task.FromResult(null); - - if (!IsExpired(entry)) - return Task.FromResult(entry.State); - - // Remove expired entry on access - _jobs.TryRemove(jobId, out _); - _cancellations.TryRemove(jobId, out _); - - return Task.FromResult(null); - - } - } - - public Task UpdateJobStatusAsync(string jobId, MessageExecutionStatus status, DateTimeOffset? startedUtc = null, DateTimeOffset? completedUtc = null, string? errorMessage = null, int? progress = null, int? attempt = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, string? workerId = null) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - if (!_jobs.TryGetValue(jobId, out var entry) || IsExpired(entry) - || IsTerminal(entry.State.Status) || attempt < entry.State.Attempt - || (status == MessageExecutionStatus.Processing && attempt == entry.State.Attempt && entry.State.Status is MessageExecutionStatus.Processing or MessageExecutionStatus.RetryPending) - || (status == MessageExecutionStatus.EnqueueUnknown && entry.State.Status != MessageExecutionStatus.Queued)) - return Task.FromResult(false); - - var now = _timeProvider.GetUtcNow(); - var updated = entry.State with - { - Status = status, - StartedUtc = startedUtc ?? entry.State.StartedUtc, - CompletedUtc = IsTerminal(status) ? completedUtc ?? now : null, - ErrorMessage = errorMessage ?? (status == MessageExecutionStatus.Processing ? null : entry.State.ErrorMessage), - Progress = progress ?? (status == MessageExecutionStatus.Processing ? 0 : entry.State.Progress), - ProgressMessage = status == MessageExecutionStatus.Processing ? null : entry.State.ProgressMessage, - LastHeartbeatUtc = status == MessageExecutionStatus.Processing ? startedUtc ?? now : entry.State.LastHeartbeatUtc, - WorkerId = workerId ?? entry.State.WorkerId, - Attempt = attempt ?? entry.State.Attempt, - LastUpdatedUtc = now - }; - _jobs[jobId] = new JobEntry(updated, expiry.HasValue ? now + expiry.Value : entry.ExpiresAt, expiry ?? entry.Retention); - return Task.FromResult(true); - } - } - - public Task UpdateJobProgressAsync(string jobId, int progress, string? progressMessage = null, TimeSpan? expiry = null, CancellationToken cancellationToken = default, int? expectedAttempt = null) - => UpdateLiveJobAsync(jobId, expectedAttempt, expiry, state => state with - { - Progress = progress, - ProgressMessage = progressMessage, - LastUpdatedUtc = _timeProvider.GetUtcNow() - }, cancellationToken); - - public Task HeartbeatAsync(string jobId, CancellationToken cancellationToken = default, int? expectedAttempt = null, TimeSpan? expiry = null) - => UpdateLiveJobAsync(jobId, expectedAttempt, expiry, state => state with - { - LastHeartbeatUtc = _timeProvider.GetUtcNow(), - LastUpdatedUtc = _timeProvider.GetUtcNow() - }, cancellationToken); - - private Task UpdateLiveJobAsync(string jobId, int? expectedAttempt, TimeSpan? expiry, Func update, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - if (_jobs.TryGetValue(jobId, out var entry) && !IsExpired(entry) && entry.State.Status == MessageExecutionStatus.Processing - && (!expectedAttempt.HasValue || expectedAttempt == entry.State.Attempt)) - _jobs[jobId] = new JobEntry(update(entry.State), (expiry ?? entry.Retention) is { } retention ? _timeProvider.GetUtcNow() + retention : entry.ExpiresAt, expiry ?? entry.Retention); - return Task.CompletedTask; - } - } - - private static bool IsTerminal(MessageExecutionStatus status) => status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled; - - public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - if (!_jobs.TryGetValue(jobId, out var entry) || IsExpired(entry)) - return Task.FromResult(false); - - // Only allow cancellation for non-terminal states - if (entry.State.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) - return Task.FromResult(false); - - _cancellations[jobId] = true; - return Task.FromResult(true); - - } - } - - public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - return Task.FromResult(_cancellations.ContainsKey(jobId)); - - } - } - - public Task RemoveJobStateAsync(string jobId, CancellationToken cancellationToken = default) - { - cancellationToken.ThrowIfCancellationRequested(); - lock (_gate) - { - _jobs.TryRemove(jobId, out _); - _cancellations.TryRemove(jobId, out _); - return Task.CompletedTask; - - } - } - - public Task IncrementCounterAsync(string queueName, string counterName, long value = 1, CancellationToken cancellationToken = default) - { - var bucketKey = GetBucketKey(queueName, _timeProvider.GetUtcNow()); - var bucket = _counterBuckets.GetOrAdd(bucketKey, _ => new ConcurrentDictionary()); - bucket.AddOrUpdate(counterName, value, (_, existing) => existing + value); - return Task.CompletedTask; - } - - public Task GetCounterStatsAsync(string queueName, TimeSpan? window = null, CancellationToken cancellationToken = default) - { - var now = _timeProvider.GetUtcNow(); - var effectiveWindow = window ?? TimeSpan.FromHours(24); - var startHour = TruncateToHour(now - effectiveWindow); - var endHour = TruncateToHour(now); - - var totals = new Dictionary(); - var buckets = new List(); - - for (var hour = startHour; hour <= endHour; hour = hour.AddHours(1)) - { - var bucketKey = GetBucketKey(queueName, hour); - var counters = new Dictionary(); - - if (_counterBuckets.TryGetValue(bucketKey, out var bucket)) - { - foreach (var kvp in bucket) - { - counters[kvp.Key] = kvp.Value; - totals[kvp.Key] = totals.GetValueOrDefault(kvp.Key) + kvp.Value; - } - } - - buckets.Add(new JobCounterBucket { Hour = hour, Counters = counters }); - } - - return Task.FromResult(new JobCounterStats { Totals = totals, Buckets = buckets }); - } - - private static string GetBucketKey(string queueName, DateTimeOffset timestamp) - { - var hour = TruncateToHour(timestamp); - return $"{queueName}:{hour:yyyy-MM-ddTHH}"; - } - - private static DateTimeOffset TruncateToHour(DateTimeOffset timestamp) - => new(timestamp.Year, timestamp.Month, timestamp.Day, timestamp.Hour, 0, 0, TimeSpan.Zero); - - public Task> GetJobsByStatusAsync(string queueName, MessageExecutionStatus status, int skip = 0, int take = 50, CancellationToken cancellationToken = default) - { - var now = _timeProvider.GetUtcNow(); - var results = _jobs.Values - .Where(e => !IsExpired(e, now) - && string.Equals(e.State.QueueName, queueName, StringComparison.OrdinalIgnoreCase) - && e.State.Status == status) - .OrderByDescending(e => e.State.CreatedUtc) - .Skip(skip) - .Take(take) - .Select(e => e.State) - .ToList(); - - return Task.FromResult>(results); - } - - public Task GetJobCountByStatusAsync(string queueName, MessageExecutionStatus status, CancellationToken cancellationToken = default) - { - var now = _timeProvider.GetUtcNow(); - var count = _jobs.Values.Count(e => !IsExpired(e, now) - && string.Equals(e.State.QueueName, queueName, StringComparison.OrdinalIgnoreCase) - && e.State.Status == status); - - return Task.FromResult((long)count); - } - - private bool IsExpired(JobEntry entry) => IsExpired(entry, _timeProvider.GetUtcNow()); - - private static bool IsExpired(JobEntry entry, DateTimeOffset now) => now >= entry.ExpiresAt; - - private void CleanupIfNeeded() - { - // Run cleanup every 100 writes to avoid accumulating expired entries - if (Interlocked.Increment(ref _accessCount) % 100 != 0) - return; - - var now = _timeProvider.GetUtcNow(); - foreach (var kvp in _jobs) - { - if (now >= kvp.Value.ExpiresAt) - { - _jobs.TryRemove(kvp.Key, out _); - _cancellations.TryRemove(kvp.Key, out _); - } - } - } - - private sealed record JobEntry(MessageExecutionState State, DateTimeOffset ExpiresAt, TimeSpan? Retention); -} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs deleted file mode 100644 index 25b48c679..000000000 --- a/src/Foundatio/Messaging/Tracking/MessageExecutionState.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -namespace Foundatio.Messaging; - -/// -/// Tracked state of a queued job. Immutable; use with expressions to derive updates. -/// -public sealed record MessageExecutionState -{ - public required string JobId { get; init; } - public required string QueueName { get; init; } - public string MessageType { get; init; } = string.Empty; - public MessageExecutionStatus Status { get; init; } = MessageExecutionStatus.Queued; - public int Progress { get; init; } - public string? ProgressMessage { get; init; } - public DateTimeOffset CreatedUtc { get; init; } - public DateTimeOffset? StartedUtc { get; init; } - public DateTimeOffset? CompletedUtc { get; init; } - public int Attempt { get; init; } - /// Identity of the worker process that started the current or most recent attempt. - public string? WorkerId { get; init; } - public string? ErrorMessage { get; init; } - public DateTimeOffset LastUpdatedUtc { get; init; } - - /// - /// When the worker last signalled that the job is alive, through a visibility renewal or a progress - /// report. A processing job whose heartbeat is stale has most likely lost its worker. - /// - public DateTimeOffset? LastHeartbeatUtc { get; init; } - - /// - /// Caller-supplied metadata captured by the producer at enqueue time, - /// such as a tenant or user id, so stores can index and display jobs by them. - /// - public IReadOnlyDictionary? Metadata { get; init; } -} - -public enum MessageExecutionStatus -{ - Queued = 0, - Processing = 1, - Completed = 2, - Failed = 3, - Cancelled = 4, - /// The current attempt did not settle successfully and may be delivered again. - RetryPending = 5, - /// The transport call failed without confirming whether the message was accepted. - EnqueueUnknown = 6 -} diff --git a/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs b/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs deleted file mode 100644 index f37e5beee..000000000 --- a/src/Foundatio/Messaging/Tracking/MessageExecutionStoreExtensions.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace Foundatio.Messaging; - -public static class MessageExecutionStoreExtensions -{ - /// Observes a tracked execution until it reaches a terminal state. Cancellation stops observation, not the queued work. - public static async Task WaitForCompletionAsync(this IMessageExecutionStore store, string id, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(store); - ArgumentException.ThrowIfNullOrWhiteSpace(id); - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - var state = await store.GetJobStateAsync(id, cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException($"Execution '{id}' was not found or has expired."); - if (state.Status is MessageExecutionStatus.Completed or MessageExecutionStatus.Failed or MessageExecutionStatus.Cancelled) - return state; - await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs b/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs deleted file mode 100644 index e481e27b6..000000000 --- a/tests/Foundatio.Redis.Tests/RedisMessageExecutionStoreTests.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using Foundatio.Messaging; -using Foundatio.Tests.Messaging; - -namespace Foundatio.Redis.Tests; - -public sealed class RedisMessageExecutionStoreTests : MessageExecutionStoreConformanceTests -{ - protected override IMessageExecutionStore? CreateStore() => RedisTestConnection.Multiplexer is { } connection - ? new RedisMessageExecutionStore(connection, new RedisMessageExecutionStoreOptions { KeyPrefix = $"native-execution-test:{Guid.NewGuid():N}" }) - : null; -} diff --git a/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs b/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs deleted file mode 100644 index ba2206506..000000000 --- a/tests/Foundatio.Tests/Messaging/MessageExecutionStoreTests.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Foundatio.Messaging; - -namespace Foundatio.Tests.Messaging; - -public sealed class MessageExecutionStoreTests : MessageExecutionStoreConformanceTests -{ - protected override IMessageExecutionStore CreateStore() => new InMemoryMessageExecutionStore(); -}