From fc6213b1b506aa66588976f0a390e7f120543bb2 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 16:32:01 -0500 Subject: [PATCH 01/57] feat: add message transport foundation --- .../MessageTransportConformanceTests.cs | 333 ++++++++++ .../Messaging/InMemoryMessageTransport.cs | 606 ++++++++++++++++++ src/Foundatio/Messaging/KnownHeaders.cs | 14 + src/Foundatio/Messaging/MessageHeaders.cs | 118 ++++ src/Foundatio/Messaging/MessageTransport.cs | 178 +++++ .../InMemoryMessageTransportTests.cs | 95 +++ 6 files changed, 1344 insertions(+) create mode 100644 src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs create mode 100644 src/Foundatio/Messaging/InMemoryMessageTransport.cs create mode 100644 src/Foundatio/Messaging/KnownHeaders.cs create mode 100644 src/Foundatio/Messaging/MessageHeaders.cs create mode 100644 src/Foundatio/Messaging/MessageTransport.cs create mode 100644 tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs new file mode 100644 index 000000000..933937762 --- /dev/null +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -0,0 +1,333 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Xunit; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public abstract class MessageTransportConformanceTests : TestWithLoggingBase +{ + protected MessageTransportConformanceTests(ITestOutputHelper output) : base(output) { } + + protected virtual IMessageTransport? CreateTransport() + { + return null; + } + + protected virtual ValueTask CleanupTransportAsync(IMessageTransport transport) + { + return transport.DisposeAsync(); + } + + public virtual async Task CanSendAndReceiveBatchAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "orders", Role = DestinationRole.Queue }); + + var result = await transport.SendAsync("orders", [ + CreateMessage("one", ("tenant", "acme")), + CreateMessage("two", ("tenant", "acme")) + ], new TransportSendOptions(), TestCancellationToken); + + Assert.True(result.AllSucceeded); + Assert.Equal(2, result.Items.Count); + Assert.All(result.Items, item => Assert.True(item.Success)); + + var entries = await pull.ReceiveAsync("orders", new ReceiveRequest + { + MaxMessages = 2, + MaxWaitTime = TimeSpan.FromSeconds(1) + }, TestCancellationToken); + + Assert.Equal(2, entries.Count); + Assert.Equal("one", ReadBody(entries[0])); + Assert.Equal("two", ReadBody(entries[1])); + Assert.Equal("acme", entries[0].Headers["tenant"]); + Assert.Equal(1, entries[0].DeliveryCount); + + await transport.CompleteAsync(entries[0], TestCancellationToken); + await transport.CompleteAsync(entries[1], TestCancellationToken); + + if (transport is ISupportsStats stats) + { + QueueStats queueStats = await stats.GetStatsAsync("orders", TestCancellationToken); + Assert.Equal(0, queueStats.Queued); + Assert.Equal(0, queueStats.Working); + Assert.Equal(2, queueStats.Completed); + } + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "retry", Role = DestinationRole.Queue }); + await transport.SendAsync("retry", [CreateMessage("retry-me")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await pull.ReceiveAsync("retry", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + Assert.Equal(1, first.DeliveryCount); + + await transport.AbandonAsync(first, TestCancellationToken); + + var second = Assert.Single(await pull.ReceiveAsync("retry", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + Assert.Equal(first.Id, second.Id); + Assert.Equal(2, second.DeliveryCount); + Assert.Equal("retry-me", ReadBody(second)); + + await transport.CompleteAsync(second, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "receipts", Role = DestinationRole.Queue }); + await transport.SendAsync("receipts", [CreateMessage("done")], new TransportSendOptions(), TestCancellationToken); + + var entry = Assert.Single(await pull.ReceiveAsync("receipts", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + await transport.CompleteAsync(entry, TestCancellationToken); + + await Assert.ThrowsAsync(async () => + await transport.CompleteAsync(entry, TestCancellationToken)); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task SubscribeAsync_DeliversPushMessagesAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPush push) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "push", Role = DestinationRole.Queue }); + + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var subscription = await push.SubscribeAsync("push", async (entry, ct) => + { + await transport.CompleteAsync(entry, ct); + received.TrySetResult(entry); + }, new PushOptions(), TestCancellationToken); + + await transport.SendAsync("push", [CreateMessage("pushed")], new TransportSendOptions(), TestCancellationToken); + + var completed = await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(3), TestCancellationToken)); + Assert.Equal(received.Task, completed); + Assert.Equal("pushed", ReadBody(await received.Task)); + Assert.Equal("push", subscription.Source); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsProvisioning) + return; + + try + { + await EnsureAsync(transport, + new DestinationDeclaration { Name = "orders-topic", Role = DestinationRole.Topic }, + new DestinationDeclaration { Name = "orders-subscription-a", Role = DestinationRole.Subscription, Source = "orders-topic" }, + new DestinationDeclaration { Name = "orders-subscription-b", Role = DestinationRole.Subscription, Source = "orders-topic" }); + + await transport.SendAsync("orders-topic", [CreateMessage("fanout")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await pull.ReceiveAsync("orders-subscription-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var second = Assert.Single(await pull.ReceiveAsync("orders-subscription-b", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + + Assert.Equal("fanout", ReadBody(first)); + Assert.Equal("fanout", ReadBody(second)); + + await transport.CompleteAsync(first, TestCancellationToken); + await transport.CompleteAsync(second, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task ReceiveAsync_RespectsPriorityAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsPriority) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "priority", Role = DestinationRole.Queue }); + await transport.SendAsync("priority", [CreateMessage("low")], new TransportSendOptions { Priority = MessagePriority.Low }, TestCancellationToken); + await transport.SendAsync("priority", [CreateMessage("high")], new TransportSendOptions { Priority = MessagePriority.High }, TestCancellationToken); + await transport.SendAsync("priority", [CreateMessage("normal")], new TransportSendOptions { Priority = MessagePriority.Normal }, TestCancellationToken); + + var entries = await pull.ReceiveAsync("priority", new ReceiveRequest + { + MaxMessages = 3, + MaxWaitTime = TimeSpan.FromSeconds(1) + }, TestCancellationToken); + + Assert.Equal(3, entries.Count); + Assert.Equal("high", ReadBody(entries[0])); + Assert.Equal("normal", ReadBody(entries[1])); + Assert.Equal("low", ReadBody(entries[2])); + + foreach (var entry in entries) + await transport.CompleteAsync(entry, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsDelayedDelivery) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "delayed", Role = DestinationRole.Queue }); + await transport.SendAsync("delayed", [CreateMessage("later")], new TransportSendOptions + { + DeliverAt = DateTimeOffset.UtcNow.AddMilliseconds(250) + }, TestCancellationToken); + + var immediate = await pull.ReceiveAsync("delayed", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + Assert.Empty(immediate); + + var delayed = Assert.Single(await pull.ReceiveAsync("delayed", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + Assert.Equal("later", ReadBody(delayed)); + await transport.CompleteAsync(delayed, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsDeadLetter || transport is not ISupportsStats stats) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "deadletter", Role = DestinationRole.Queue }); + await transport.SendAsync("deadletter", [CreateMessage("poison")], new TransportSendOptions(), TestCancellationToken); + + var entry = Assert.Single(await pull.ReceiveAsync("deadletter", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + await ((ISupportsDeadLetter)transport).DeadLetterAsync(entry, "bad-payload", TestCancellationToken); + + QueueStats queueStats = await stats.GetStatsAsync("deadletter", TestCancellationToken); + Assert.Equal(0, queueStats.Working); + Assert.Equal(1, queueStats.Deadletter); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsExpiration || transport is not ISupportsStats stats) + return; + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "expiration", Role = DestinationRole.Queue }); + var expired = new TransportMessage + { + Body = Encoding.UTF8.GetBytes("expired"), + Headers = MessageHeaders.Create([ + new KeyValuePair(KnownHeaders.Expiration, DateTimeOffset.UtcNow.AddMinutes(-1).ToString("O")) + ]) + }; + + await transport.SendAsync("expiration", [expired], new TransportSendOptions(), TestCancellationToken); + + var entries = await pull.ReceiveAsync("expiration", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + Assert.Empty(entries); + + QueueStats queueStats = await stats.GetStatsAsync("expiration", TestCancellationToken); + Assert.Equal(0, queueStats.Queued); + Assert.Equal(1, queueStats.Deadletter); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + private async ValueTask CleanupTransportIfNotNullAsync(IMessageTransport? transport) + { + if (transport is not null) + await CleanupTransportAsync(transport); + } + + private static async Task EnsureAsync(IMessageTransport transport, params DestinationDeclaration[] declarations) + { + if (transport is ISupportsProvisioning provisioning) + await provisioning.EnsureAsync(declarations, CancellationToken.None); + } + + private static TransportMessage CreateMessage(string body, params (string Key, string Value)[] headers) + { + return new TransportMessage + { + Body = Encoding.UTF8.GetBytes(body), + Headers = MessageHeaders.Create(ToKeyValuePairs(headers)) + }; + } + + private static IEnumerable> ToKeyValuePairs((string Key, string Value)[] headers) + { + foreach (var header in headers) + yield return new KeyValuePair(header.Key, header.Value); + } + + private static string ReadBody(TransportEntry entry) + { + return Encoding.UTF8.GetString(entry.Body.Span); + } +} diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs new file mode 100644 index 000000000..600ab61fa --- /dev/null +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -0,0 +1,606 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Queues; +using Foundatio.Utility; + +namespace Foundatio.Messaging; + +public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsRedeliveryDelay, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsDelayedDelivery, ISupportsExpiration, ISupportsProvisioning, ITransportInfo +{ + private static readonly IReadOnlySet _supportedRoles = new HashSet + { + DestinationRole.Queue, + DestinationRole.Topic, + DestinationRole.Subscription, + DestinationRole.Binding + }; + + private readonly ConcurrentDictionary _destinations = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _roles = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary> _topicSubscriptions = new(StringComparer.OrdinalIgnoreCase); + private readonly TimeProvider _timeProvider; + private readonly CancellationTokenSource _disposeCancellationTokenSource = new(); + private int _isDisposed; + + public InMemoryMessageTransport(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; + public IReadOnlySet SupportedRoles => _supportedRoles; + public int? MaxBatchSize => null; + public long? MaxMessageBytes => null; + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(messages); + + var results = new SendItemResult[messages.Count]; + for (int index = 0; index < messages.Count; index++) + { + var message = messages[index]; + string messageId = message.MessageId ?? options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + var stored = CreateStoredMessage(destination, messageId, message, options); + EnqueueOrSchedule(destination, stored, options); + + results[index] = new SendItemResult + { + MessageId = messageId, + Success = true + }; + } + + return Task.FromResult(new SendResult { Items = results }); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + { + return ReceiveAsync(source, request, visibility: null, ct); + } + + public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + { + return await ReceiveAsync(source, request, (TimeSpan?)visibility, ct).AnyContext(); + } + + private async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan? visibility, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(source); + + int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; + var state = GetOrAddDestination(source, DestinationRole.Queue); + var entries = new List(maxMessages); + DateTimeOffset? waitUntil = request.MaxWaitTime is { } waitTime && waitTime > TimeSpan.Zero + ? _timeProvider.GetUtcNow().Add(waitTime) + : null; + + while (entries.Count < maxMessages) + { + if (TryReceive(source, state, out var entry)) + { + entries.Add(entry); + continue; + } + + if (entries.Count > 0 || waitUntil is null) + break; + + TimeSpan remaining = waitUntil.Value - _timeProvider.GetUtcNow(); + if (remaining <= TimeSpan.Zero) + break; + + using var waitCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, _disposeCancellationTokenSource.Token); + waitCancellationTokenSource.CancelAfter(remaining); + + try + { + await state.AvailableSignal.WaitAsync(waitCancellationTokenSource.Token).AnyContext(); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested && !_disposeCancellationTokenSource.IsCancellationRequested) + { + break; + } + } + + return entries; + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(entry); + + var receipt = GetReceipt(entry); + var state = GetExistingDestination(receipt.Destination); + + if (!state.InFlight.TryRemove(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) + throw new ReceiptExpiredException(); + + Interlocked.Increment(ref state.Completed); + return Task.CompletedTask; + } + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + return AbandonAsync(entry, TimeSpan.Zero, ct); + } + + public Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(entry); + + var receipt = GetReceipt(entry); + var state = GetExistingDestination(receipt.Destination); + + if (!state.InFlight.TryRemove(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) + throw new ReceiptExpiredException(); + + Interlocked.Increment(ref state.Abandoned); + var redelivered = inFlight.Message with { DeliveryCount = entry.DeliveryCount + 1 }; + + if (redeliveryDelay > TimeSpan.Zero) + _ = Run.DelayedAsync(redeliveryDelay, () => EnqueueStoredMessageAsync(receipt.Destination, redelivered), _timeProvider, _disposeCancellationTokenSource.Token); + else + EnqueueStoredMessage(receipt.Destination, redelivered); + + return Task.CompletedTask; + } + + public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(entry); + + var receipt = GetReceipt(entry); + var state = GetExistingDestination(receipt.Destination); + + if (!state.InFlight.TryRemove(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) + throw new ReceiptExpiredException(); + + DeadLetter(state, inFlight.Message, reason); + return Task.CompletedTask; + } + + public Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(onMessage); + ArgumentNullException.ThrowIfNull(options); + + var subscription = new PushSubscription(source); + subscription.Start(RunPushSubscriptionAsync(source, onMessage, options, subscription.CancellationToken)); + return Task.FromResult(subscription); + } + + public Task GetStatsAsync(string destination, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(destination); + + if (!_destinations.TryGetValue(destination, out var state)) + return Task.FromResult(new QueueStats()); + + return Task.FromResult(new QueueStats + { + Queued = state.QueuedCount, + Working = state.InFlight.Count, + Deadletter = state.DeadletterCount, + Enqueued = Volatile.Read(ref state.Enqueued), + Dequeued = Volatile.Read(ref state.Dequeued), + Completed = Volatile.Read(ref state.Completed), + Abandoned = Volatile.Read(ref state.Abandoned) + }); + } + + public Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(declarations); + + foreach (var declaration in declarations) + { + ArgumentException.ThrowIfNullOrEmpty(declaration.Name); + + switch (declaration.Role) + { + case DestinationRole.Queue: + GetOrAddDestination(declaration.Name, DestinationRole.Queue); + break; + case DestinationRole.Topic: + SetRole(declaration.Name, DestinationRole.Topic); + _topicSubscriptions.GetOrAdd(declaration.Name, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + break; + case DestinationRole.Subscription: + GetOrAddDestination(declaration.Name, DestinationRole.Subscription); + if (!String.IsNullOrEmpty(declaration.Source)) + AddTopicSubscription(declaration.Source, declaration.Name); + break; + case DestinationRole.Binding: + if (String.IsNullOrEmpty(declaration.Source)) + throw new ArgumentException("A binding declaration must specify a source topic.", nameof(declarations)); + + GetOrAddDestination(declaration.Name, DestinationRole.Subscription); + AddTopicSubscription(declaration.Source, declaration.Name); + break; + default: + throw new ArgumentOutOfRangeException(nameof(declarations), declaration.Role, "Unsupported destination role."); + } + } + + return Task.CompletedTask; + } + + public Task DeleteAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(name); + + _roles.TryRemove(name, out _); + _destinations.TryRemove(name, out _); + _topicSubscriptions.TryRemove(name, out _); + + foreach (var subscriptions in _topicSubscriptions.Values) + subscriptions.TryRemove(name, out _); + + return Task.CompletedTask; + } + + public Task ExistsAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(name); + + return Task.FromResult(_roles.ContainsKey(name)); + } + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return ValueTask.CompletedTask; + + _disposeCancellationTokenSource.Cancel(); + _disposeCancellationTokenSource.Dispose(); + _destinations.Clear(); + _roles.Clear(); + _topicSubscriptions.Clear(); + return ValueTask.CompletedTask; + } + + private async Task RunPushSubscriptionAsync(string source, Func onMessage, PushOptions options, CancellationToken subscriptionCancellationToken) + { + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(subscriptionCancellationToken, _disposeCancellationTokenSource.Token); + var token = linkedCancellationTokenSource.Token; + int maxMessages = Math.Max(1, options.MaxConcurrentMessages); + + while (!token.IsCancellationRequested) + { + IReadOnlyList entries; + try + { + entries = await ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = maxMessages, + MaxWaitTime = options.PollInterval + }, token).AnyContext(); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + break; + } + + foreach (var entry in entries) + { + try + { + await onMessage(entry, token).AnyContext(); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + break; + } + catch + { + await AbandonAsync(entry, token).AnyContext(); + } + } + } + } + + private void EnqueueOrSchedule(string destination, StoredMessage message, TransportSendOptions options) + { + if (options.DeliverAt is { } deliverAt) + { + TimeSpan delay = deliverAt - _timeProvider.GetUtcNow(); + if (delay > TimeSpan.Zero) + { + _ = Run.DelayedAsync(delay, () => EnqueueForDestinationAsync(destination, message), _timeProvider, _disposeCancellationTokenSource.Token); + return; + } + } + + EnqueueForDestination(destination, message); + } + + private Task EnqueueForDestinationAsync(string destination, StoredMessage message) + { + EnqueueForDestination(destination, message); + return Task.CompletedTask; + } + + private void EnqueueForDestination(string destination, StoredMessage message) + { + var role = _roles.GetOrAdd(destination, DestinationRole.Queue); + if (role == DestinationRole.Topic) + { + if (!_topicSubscriptions.TryGetValue(destination, out var subscriptions)) + return; + + foreach (string subscription in subscriptions.Keys) + EnqueueStoredMessage(subscription, message with { Destination = subscription }); + + return; + } + + if (role is DestinationRole.Binding) + throw new InvalidOperationException($"Cannot send directly to binding destination \"{destination}\"."); + + EnqueueStoredMessage(destination, message); + } + + private Task EnqueueStoredMessageAsync(string destination, StoredMessage message) + { + EnqueueStoredMessage(destination, message); + return Task.CompletedTask; + } + + private void EnqueueStoredMessage(string destination, StoredMessage message) + { + var role = _roles.GetOrAdd(destination, DestinationRole.Queue); + var state = GetOrAddDestination(destination, role == DestinationRole.Topic ? DestinationRole.Queue : role); + state.Enqueue(message with { Destination = destination }); + } + + private bool TryReceive(string source, DestinationState state, out TransportEntry entry) + { + while (state.TryDequeue(out var message)) + { + if (IsExpired(message)) + { + DeadLetter(state, message, "expired"); + continue; + } + + var receipt = new InMemoryReceipt(source, Guid.NewGuid().ToString("N")); + state.InFlight[receipt.LockToken] = new InFlightMessage(message, receipt); + Interlocked.Increment(ref state.Dequeued); + + entry = new TransportEntry + { + Id = message.Id, + Destination = source, + Body = message.Body, + Headers = message.Headers, + DeliveryCount = message.DeliveryCount, + EnqueuedUtc = message.EnqueuedUtc, + Receipt = new Receipt { TransportState = receipt } + }; + return true; + } + + entry = null!; + return false; + } + + private bool IsExpired(StoredMessage message) + { + string? expiration = message.Headers.GetValueOrDefault(KnownHeaders.Expiration); + if (expiration is null) + return false; + + return DateTimeOffset.TryParse(expiration, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var expiresAt) + && expiresAt <= _timeProvider.GetUtcNow(); + } + + private void DeadLetter(DestinationState state, StoredMessage message, string? reason) + { + if (!String.IsNullOrEmpty(reason)) + message = message with { Headers = message.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build() }; + + state.Deadletter(message); + } + + private StoredMessage CreateStoredMessage(string destination, string messageId, TransportMessage message, TransportSendOptions options) + { + var headers = message.Headers.ToBuilder() + .SetIfMissing(KnownHeaders.Priority, options.Priority.ToString()) + .Build(); + + return new StoredMessage( + messageId, + destination, + message.Body.ToArray(), + headers, + NormalizePriority(options.Priority), + DeliveryCount: 1, + EnqueuedUtc: _timeProvider.GetUtcNow()); + } + + private DestinationState GetOrAddDestination(string name, DestinationRole role) + { + SetRole(name, role); + return _destinations.GetOrAdd(name, static _ => new DestinationState()); + } + + private DestinationState GetExistingDestination(string name) + { + if (_destinations.TryGetValue(name, out var destination)) + return destination; + + throw new ReceiptExpiredException($"The destination \"{name}\" no longer exists."); + } + + private void SetRole(string name, DestinationRole role) + { + _roles.AddOrUpdate(name, role, (_, existing) => + { + if (existing == role) + return existing; + + if (existing == DestinationRole.Queue && role == DestinationRole.Subscription) + return role; + + if (existing == DestinationRole.Subscription && role == DestinationRole.Queue) + return existing; + + throw new InvalidOperationException($"Destination \"{name}\" is already declared as {existing}."); + }); + } + + private void AddTopicSubscription(string topic, string subscription) + { + SetRole(topic, DestinationRole.Topic); + GetOrAddDestination(subscription, DestinationRole.Subscription); + var subscriptions = _topicSubscriptions.GetOrAdd(topic, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + subscriptions[subscription] = 0; + } + + private static MessagePriority NormalizePriority(MessagePriority priority) + { + return priority switch + { + MessagePriority.Low => MessagePriority.Low, + MessagePriority.Normal => MessagePriority.Normal, + MessagePriority.High => MessagePriority.High, + _ => MessagePriority.Normal + }; + } + + private static InMemoryReceipt GetReceipt(TransportEntry entry) + { + return entry.Receipt.TransportState as InMemoryReceipt ?? throw new ReceiptExpiredException(); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + } + + private sealed record StoredMessage( + string Id, + string Destination, + ReadOnlyMemory Body, + MessageHeaders Headers, + MessagePriority Priority, + int DeliveryCount, + DateTimeOffset EnqueuedUtc); + + private sealed record InFlightMessage(StoredMessage Message, InMemoryReceipt Receipt); + + private sealed record InMemoryReceipt(string Destination, string LockToken); + + private sealed class DestinationState + { + private readonly ConcurrentQueue[] _queues = + [ + new ConcurrentQueue(), + new ConcurrentQueue(), + new ConcurrentQueue() + ]; + + private readonly ConcurrentQueue _deadletterQueue = new(); + + public ConcurrentDictionary InFlight { get; } = new(StringComparer.Ordinal); + public AsyncAutoResetEvent AvailableSignal { get; } = new(); + public long Enqueued; + public long Dequeued; + public long Completed; + public long Abandoned; + public long Deadlettered; + + public long QueuedCount => _queues.Sum(q => q.Count); + public long DeadletterCount => _deadletterQueue.Count; + + public void Enqueue(StoredMessage message) + { + _queues[(int)message.Priority].Enqueue(message); + Interlocked.Increment(ref Enqueued); + AvailableSignal.Set(); + } + + public bool TryDequeue(out StoredMessage message) + { + for (int index = (int)MessagePriority.High; index >= (int)MessagePriority.Low; index--) + { + if (_queues[index].TryDequeue(out message!)) + return true; + } + + message = null!; + return false; + } + + public void Deadletter(StoredMessage message) + { + _deadletterQueue.Enqueue(message); + Interlocked.Increment(ref Deadlettered); + } + } + + private sealed class PushSubscription : IPushSubscription + { + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private Task? _worker; + + public PushSubscription(string source) + { + Source = source; + } + + public string Source { get; } + public CancellationToken CancellationToken => _cancellationTokenSource.Token; + + public void Start(Task worker) + { + _worker = worker; + } + + public async ValueTask DisposeAsync() + { + await _cancellationTokenSource.CancelAsync().AnyContext(); + + if (_worker is not null) + { + try + { + await _worker.AnyContext(); + } + catch (OperationCanceledException) { } + } + + _cancellationTokenSource.Dispose(); + } + } +} diff --git a/src/Foundatio/Messaging/KnownHeaders.cs b/src/Foundatio/Messaging/KnownHeaders.cs new file mode 100644 index 000000000..d460072ee --- /dev/null +++ b/src/Foundatio/Messaging/KnownHeaders.cs @@ -0,0 +1,14 @@ +namespace Foundatio.Messaging; + +public static class KnownHeaders +{ + public const string MessageType = "message.type"; + public const string ContentType = "message.content_type"; + public const string CorrelationId = "message.correlation_id"; + public const string TraceParent = "traceparent"; + public const string TraceState = "tracestate"; + public const string Priority = "message.priority"; + public const string Expiration = "message.expiration"; + public const string Attempts = "message.attempts"; + public const string DeadLetterReason = "message.dead_letter.reason"; +} diff --git a/src/Foundatio/Messaging/MessageHeaders.cs b/src/Foundatio/Messaging/MessageHeaders.cs new file mode 100644 index 000000000..1ba9686b6 --- /dev/null +++ b/src/Foundatio/Messaging/MessageHeaders.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections; +using System.Collections.Frozen; +using System.Collections.Generic; + +namespace Foundatio.Messaging; + +public sealed class MessageHeaders : IReadOnlyDictionary +{ + public static MessageHeaders Empty { get; } = new(FrozenDictionary.Empty); + + private readonly FrozenDictionary _headers; + + private MessageHeaders(FrozenDictionary headers) + { + _headers = headers; + } + + public string this[string key] => _headers[key]; + public IEnumerable Keys => _headers.Keys; + public IEnumerable Values => _headers.Values; + public int Count => _headers.Count; + + public static MessageHeaders Create(IEnumerable> headers) + { + ArgumentNullException.ThrowIfNull(headers); + + if (headers is MessageHeaders messageHeaders) + return messageHeaders; + + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in headers) + { + ArgumentException.ThrowIfNullOrEmpty(header.Key); + ArgumentNullException.ThrowIfNull(header.Value); + values[header.Key] = header.Value; + } + + return values.Count == 0 + ? Empty + : new MessageHeaders(values.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase)); + } + + public bool ContainsKey(string key) + { + return _headers.ContainsKey(key); + } + + public bool TryGetValue(string key, out string value) + { + return _headers.TryGetValue(key, out value!); + } + + public string? GetValueOrDefault(string key) + { + return _headers.GetValueOrDefault(key); + } + + public Builder ToBuilder() + { + return new Builder(_headers); + } + + public IEnumerator> GetEnumerator() + { + return _headers.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + public sealed class Builder + { + private readonly Dictionary _headers; + + internal Builder(IEnumerable> headers) + { + _headers = new Dictionary(headers, StringComparer.OrdinalIgnoreCase); + } + + public Builder Add(string key, string value) + { + ArgumentException.ThrowIfNullOrEmpty(key); + ArgumentNullException.ThrowIfNull(value); + _headers.Add(key, value); + return this; + } + + public Builder Set(string key, string value) + { + ArgumentException.ThrowIfNullOrEmpty(key); + ArgumentNullException.ThrowIfNull(value); + _headers[key] = value; + return this; + } + + public Builder SetIfMissing(string key, string value) + { + ArgumentException.ThrowIfNullOrEmpty(key); + ArgumentNullException.ThrowIfNull(value); + _headers.TryAdd(key, value); + return this; + } + + public bool Remove(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + return _headers.Remove(key); + } + + public MessageHeaders Build() + { + return Create(_headers); + } + } +} diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs new file mode 100644 index 000000000..3c2d22b72 --- /dev/null +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Queues; + +namespace Foundatio.Messaging; + +public enum MessagePriority +{ + Low = 0, + Normal = 1, + High = 2 +} + +public enum DeliveryGuarantee +{ + AtMostOnce, + AtLeastOnce +} + +public enum OrderingGuarantee +{ + None, + Fifo, + PerPartition +} + +public enum DestinationRole +{ + Queue, + Topic, + Subscription, + Binding +} + +public sealed record TransportMessage +{ + public required ReadOnlyMemory Body { get; init; } + public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; + public string? MessageId { get; init; } +} + +public sealed record TransportSendOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public DateTimeOffset? DeliverAt { get; init; } + public string? DeduplicationId { get; init; } + public string? PartitionKey { get; init; } +} + +public sealed record TransportEntry +{ + public required string Id { get; init; } + public required string Destination { get; init; } + public required ReadOnlyMemory Body { get; init; } + public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; + public int DeliveryCount { get; init; } = 1; + public DateTimeOffset? EnqueuedUtc { get; init; } + public required Receipt Receipt { get; init; } +} + +public readonly struct Receipt +{ + public object? TransportState { get; init; } +} + +public sealed record ReceiveRequest +{ + public int MaxMessages { get; init; } = 1; + public TimeSpan? MaxWaitTime { get; init; } +} + +public sealed record SendItemResult +{ + public string? MessageId { get; init; } + public required bool Success { get; init; } + public string? ErrorCode { get; init; } + public bool IsRetryable { get; init; } +} + +public sealed record SendResult +{ + public required IReadOnlyList Items { get; init; } + public bool AllSucceeded => Items.All(i => i.Success); +} + +public sealed class ReceiptExpiredException : Exception +{ + public ReceiptExpiredException() : base("The transport receipt has expired or has already been settled.") { } + + public ReceiptExpiredException(string message) : base(message) { } + + public ReceiptExpiredException(string message, Exception innerException) : base(message, innerException) { } +} + +public sealed record DestinationDeclaration +{ + public required string Name { get; init; } + public DestinationRole Role { get; init; } = DestinationRole.Queue; + public string? Source { get; init; } +} + +public sealed record PushOptions +{ + public int MaxConcurrentMessages { get; init; } = 1; + public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(1); +} + +public interface ITransportInfo +{ + DeliveryGuarantee DeliveryGuarantee { get; } + OrderingGuarantee Ordering { get; } + IReadOnlySet SupportedRoles { get; } + int? MaxBatchSize { get; } + long? MaxMessageBytes { get; } +} + +public interface IMessageTransport : IAsyncDisposable +{ + Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default); + Task CompleteAsync(TransportEntry entry, CancellationToken ct = default); + Task AbandonAsync(TransportEntry entry, CancellationToken ct = default); +} + +public interface ISupportsPull : IMessageTransport +{ + Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct); +} + +public interface ISupportsPush : IMessageTransport +{ + Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct); +} + +public interface ISupportsRedeliveryDelay : IMessageTransport +{ + Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct); +} + +public interface ISupportsDeadLetter : IMessageTransport +{ + Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct); +} + +public interface ISupportsLockRenewal : IMessageTransport +{ + Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct); +} + +public interface ISupportsVisibilityTimeout : IMessageTransport +{ + Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct); +} + +public interface ISupportsStats : IMessageTransport +{ + Task GetStatsAsync(string destination, CancellationToken ct); +} + +public interface ISupportsPriority : IMessageTransport { } + +public interface ISupportsDelayedDelivery : IMessageTransport { } + +public interface ISupportsExpiration : IMessageTransport { } + +public interface ISupportsProvisioning : IMessageTransport +{ + Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct); + Task DeleteAsync(string name, CancellationToken ct); + Task ExistsAsync(string name, CancellationToken ct); +} + +public interface IPushSubscription : IAsyncDisposable +{ + string Source { get; } +} diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs new file mode 100644 index 000000000..9671f6f08 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class InMemoryMessageTransportTests : MessageTransportConformanceTests +{ + public InMemoryMessageTransportTests(ITestOutputHelper output) : base(output) { } + + protected override IMessageTransport CreateTransport() + { + return new InMemoryMessageTransport(); + } + + [Fact] + public void MessageHeaders_AreImmutableAndCaseInsensitive() + { + var source = new Dictionary(StringComparer.Ordinal) + { + ["message.type"] = "order.created" + }; + + var headers = MessageHeaders.Create(source); + source["message.type"] = "changed"; + + Assert.Equal("order.created", headers["MESSAGE.TYPE"]); + Assert.Equal("order.created", headers.GetValueOrDefault("Message.Type")); + Assert.True(headers.ContainsKey("MESSAGE.TYPE")); + + var updated = headers.ToBuilder() + .Set("TraceParent", "00-123") + .SetIfMissing("traceparent", "ignored") + .Build(); + + Assert.Equal("00-123", updated["traceparent"]); + Assert.False(headers.ContainsKey("traceparent")); + } + + [Fact] + public override Task CanSendAndReceiveBatchAsync() + { + return base.CanSendAndReceiveBatchAsync(); + } + + [Fact] + public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() + { + return base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); + } + + [Fact] + public override Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() + { + return base.CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync(); + } + + [Fact] + public override Task SubscribeAsync_DeliversPushMessagesAsync() + { + return base.SubscribeAsync_DeliversPushMessagesAsync(); + } + + [Fact] + public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() + { + return base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); + } + + [Fact] + public override Task ReceiveAsync_RespectsPriorityAsync() + { + return base.ReceiveAsync_RespectsPriorityAsync(); + } + + [Fact] + public override Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() + { + return base.SendAsync_WithDeliverAt_DelaysVisibilityAsync(); + } + + [Fact] + public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() + { + return base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); + } + + [Fact] + public override Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() + { + return base.ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync(); + } +} From 754e36673a0420a04fb9a038e5245386ff179ffc Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 16:39:30 -0500 Subject: [PATCH 02/57] feat: add transport-backed message queue --- src/Foundatio/Queues/MessageQueue.cs | 399 ++++++++++++++++++ .../Queue/MessageQueueTests.cs | 200 +++++++++ 2 files changed, 599 insertions(+) create mode 100644 src/Foundatio/Queues/MessageQueue.cs create mode 100644 tests/Foundatio.Tests/Queue/MessageQueueTests.cs diff --git a/src/Foundatio/Queues/MessageQueue.cs b/src/Foundatio/Queues/MessageQueue.cs new file mode 100644 index 000000000..161dd6265 --- /dev/null +++ b/src/Foundatio/Queues/MessageQueue.cs @@ -0,0 +1,399 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Serializer; +using Foundatio.Utility; + +namespace Foundatio.Queues; + +public enum AckMode +{ + Auto, + Manual +} + +public sealed record EnqueueOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public TimeSpan? Delay { get; init; } + public DateTimeOffset? DeliverAt { get; init; } + public TimeSpan? TimeToLive { get; init; } + public string? CorrelationId { get; init; } + public string? DeduplicationId { get; init; } + public string? Destination { get; init; } + public MessageHeaders? Headers { get; init; } +} + +public sealed record ReceiveOptions +{ + public string? Source { get; init; } + public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); +} + +public sealed record WorkerOptions +{ + public AckMode AckMode { get; init; } = AckMode.Auto; + public string? Source { get; init; } + public int MaxConcurrency { get; init; } = 1; + public int MaxAttempts { get; init; } = 5; + public Func? RedeliveryBackoff { get; init; } +} + +public sealed record MessageQueueOptions +{ + public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; + public string ContentType { get; init; } = "application/json"; + public Func? DestinationResolver { get; init; } + public Func? MessageTypeResolver { get; init; } +} + +public interface IMessageQueue : IAsyncDisposable +{ + Task EnqueueAsync(T message, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task?> ReceiveAsync(ReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task StartWorkingAsync(Func, CancellationToken, Task> handler, WorkerOptions? options = null, CancellationToken cancellationToken = default) where T : class; +} + +public interface IReceivedMessage where T : class +{ + T Message { get; } + string Id { get; } + MessageHeaders Headers { get; } + string? CorrelationId { get; } + string? MessageType { get; } + MessagePriority Priority { get; } + int Attempts { get; } + bool IsHandled { get; } + CancellationToken CancellationToken { get; } + Task CompleteAsync(CancellationToken cancellationToken = default); + Task RejectAsync(bool retry = true, string? reason = null, CancellationToken cancellationToken = default); + Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default); + Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); + Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default); +} + +public sealed class MessageQueue : IMessageQueue +{ + private readonly IMessageTransport _transport; + private readonly MessageQueueOptions _options; + private int _isDisposed; + + public MessageQueue(IMessageTransport transport, MessageQueueOptions? options = null) + { + _transport = transport ?? throw new ArgumentNullException(nameof(transport)); + _options = options ?? new MessageQueueOptions(); + } + + public async Task EnqueueAsync(T message, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(message); + ThrowIfDisposed(); + + options ??= new EnqueueOptions(); + string destination = GetDestination(typeof(T), options.Destination); + var result = await _transport.SendAsync(destination, [CreateTransportMessage(message, options)], CreateSendOptions(options), cancellationToken).AnyContext(); + var item = result.Items.Count > 0 ? result.Items[0] : null; + + if (item is null || !item.Success) + throw new QueueException($"Unable to enqueue message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}"); + + return item.MessageId ?? throw new QueueException($"Transport did not return a message id for \"{destination}\"."); + } + + public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(messages); + ThrowIfDisposed(); + + options ??= new EnqueueOptions(); + string destination = GetDestination(typeof(T), options.Destination); + var transportMessages = messages.Select(message => + { + ArgumentNullException.ThrowIfNull(message); + return CreateTransportMessage(message, options); + }).ToArray(); + + if (transportMessages.Length == 0) + return; + + var result = await _transport.SendAsync(destination, transportMessages, CreateSendOptions(options), cancellationToken).AnyContext(); + if (!result.AllSucceeded) + throw new QueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{destination}\"."); + } + + public async Task?> ReceiveAsync(ReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ThrowIfDisposed(); + + if (_transport is not ISupportsPull pull) + throw new QueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); + + options ??= new ReceiveOptions(); + string source = GetDestination(typeof(T), options.Source); + var entries = await pull.ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = 1, + MaxWaitTime = options.MaxWaitTime + }, cancellationToken).AnyContext(); + + if (entries.Count == 0) + return null; + + return await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); + } + + public async Task StartWorkingAsync(Func, CancellationToken, Task> handler, WorkerOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(handler); + ThrowIfDisposed(); + + options ??= new WorkerOptions(); + string source = GetDestination(typeof(T), options.Source); + + if (_transport is ISupportsPush push) + { + await using var subscription = await push.SubscribeAsync(source, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + return; + } + + if (_transport is not ISupportsPull pull) + throw new QueueException($"Transport \"{_transport.GetType().Name}\" does not support receiving messages."); + + while (!cancellationToken.IsCancellationRequested) + { + var entries = await pull.ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = Math.Max(1, options.MaxConcurrency), + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + + foreach (var entry in entries) + { + var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); + await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); + } + } + } + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return ValueTask.CompletedTask; + + return _transport.DisposeAsync(); + } + + private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken ct) where T : class + { + try + { + var message = _options.Serializer.Deserialize(entry.Body); + if (message is null) + throw new QueueException($"Message \"{entry.Id}\" deserialized to null."); + + return new ReceivedMessage(_transport, entry, message, ct); + } + catch (Exception ex) when (ex is not QueueException) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); + throw new QueueException($"Unable to deserialize message \"{entry.Id}\".", ex); + } + catch (QueueException) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); + throw; + } + } + + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, WorkerOptions options, CancellationToken ct) where T : class + { + try + { + await handler(message, ct).AnyContext(); + + if (options.AckMode == AckMode.Auto && !message.IsHandled) + await message.CompleteAsync(ct).AnyContext(); + } + catch + { + if (!message.IsHandled) + await message.RejectAsync(message.Attempts < options.MaxAttempts, "handler-error", ct).AnyContext(); + } + } + + private TransportMessage CreateTransportMessage(T message, EnqueueOptions options) where T : class + { + var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() + .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) + .Set(KnownHeaders.ContentType, _options.ContentType) + .Set(KnownHeaders.Priority, options.Priority.ToString()); + + if (!String.IsNullOrEmpty(options.CorrelationId)) + headers.Set(KnownHeaders.CorrelationId, options.CorrelationId); + + if (Activity.Current is { } activity) + { + if (!String.IsNullOrEmpty(activity.Id)) + headers.SetIfMissing(KnownHeaders.TraceParent, activity.Id); + + if (!String.IsNullOrEmpty(activity.TraceStateString)) + headers.SetIfMissing(KnownHeaders.TraceState, activity.TraceStateString); + } + + if (options.TimeToLive is { } ttl) + headers.Set(KnownHeaders.Expiration, DateTimeOffset.UtcNow.Add(ttl).ToString("O", CultureInfo.InvariantCulture)); + + return new TransportMessage + { + Body = _options.Serializer.SerializeToBytes(message), + Headers = headers.Build() + }; + } + + private static TransportSendOptions CreateSendOptions(EnqueueOptions options) + { + return new TransportSendOptions + { + Priority = options.Priority, + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? DateTimeOffset.UtcNow.Add(delay) : null), + DeduplicationId = options.DeduplicationId + }; + } + + private string GetDestination(Type messageType, string? destination) + { + return !String.IsNullOrEmpty(destination) + ? destination + : (_options.DestinationResolver?.Invoke(messageType) ?? ToKebabCase(messageType.Name)); + } + + private string GetMessageType(Type messageType) + { + return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; + } + + private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken ct) + { + if (_transport is ISupportsDeadLetter deadLetter) + await deadLetter.DeadLetterAsync(entry, reason, ct).AnyContext(); + else + await _transport.AbandonAsync(entry, ct).AnyContext(); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + } + + private static string ToKebabCase(string value) + { + if (String.IsNullOrEmpty(value)) + return value; + + Span buffer = stackalloc char[value.Length * 2]; + int position = 0; + for (int index = 0; index < value.Length; index++) + { + char current = value[index]; + if (Char.IsUpper(current)) + { + if (index > 0) + buffer[position++] = '-'; + + buffer[position++] = Char.ToLowerInvariant(current); + } + else + { + buffer[position++] = current; + } + } + + return new String(buffer[..position]); + } +} + +internal sealed class ReceivedMessage : IReceivedMessage where T : class +{ + private readonly IMessageTransport _transport; + private readonly TransportEntry _entry; + private int _isHandled; + + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken) + { + _transport = transport; + _entry = entry; + Message = message; + CancellationToken = cancellationToken; + } + + public T Message { get; } + public string Id => _entry.Id; + public MessageHeaders Headers => _entry.Headers; + public string? CorrelationId => Headers.GetValueOrDefault(KnownHeaders.CorrelationId); + public string? MessageType => Headers.GetValueOrDefault(KnownHeaders.MessageType); + public MessagePriority Priority => Enum.TryParse(Headers.GetValueOrDefault(KnownHeaders.Priority), ignoreCase: true, out MessagePriority priority) ? priority : MessagePriority.Normal; + public int Attempts => _entry.DeliveryCount; + public bool IsHandled => Volatile.Read(ref _isHandled) == 1; + public CancellationToken CancellationToken { get; } + + public Task CompleteAsync(CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return Task.CompletedTask; + + return _transport.CompleteAsync(_entry, cancellationToken); + } + + public Task RejectAsync(bool retry = true, string? reason = null, CancellationToken cancellationToken = default) + { + return retry ? AbandonAsync(cancellationToken) : DeadLetterAsync(reason, cancellationToken); + } + + public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return; + + if (_transport is ISupportsDeadLetter deadLetter) + await deadLetter.DeadLetterAsync(_entry, reason, cancellationToken).AnyContext(); + else + await _transport.AbandonAsync(_entry, cancellationToken).AnyContext(); + } + + public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) + { + return _transport is ISupportsLockRenewal lockRenewal + ? lockRenewal.RenewLockAsync(_entry, duration, cancellationToken) + : Task.CompletedTask; + } + + public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + private Task AbandonAsync(CancellationToken ct) + { + if (!TryMarkHandled()) + return Task.CompletedTask; + + return _transport.AbandonAsync(_entry, ct); + } + + private bool TryMarkHandled() + { + return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; + } +} diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs new file mode 100644 index 000000000..514167955 --- /dev/null +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -0,0 +1,200 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Tests.Extensions; +using Xunit; + +namespace Foundatio.Tests.Queue; + +public class MessageQueueTests +{ + [Fact] + public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + + string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new EnqueueOptions + { + CorrelationId = "corr-123", + Priority = MessagePriority.High, + Headers = MessageHeaders.Create([ + new KeyValuePair("tenant", "acme") + ]) + }, cancellationToken); + + var received = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + + Assert.NotNull(received); + Assert.Equal(id, received.Id); + Assert.Equal("hello", received.Message.Data); + Assert.Equal("corr-123", received.CorrelationId); + Assert.Equal(MessagePriority.High, received.Priority); + Assert.Equal(1, received.Attempts); + Assert.Equal("acme", received.Headers["tenant"]); + Assert.Equal(typeof(PreviewWorkItem).FullName, received.MessageType); + + await received.CompleteAsync(cancellationToken); + + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(1, stats.Completed); + Assert.Equal(0, stats.Working); + } + + [Fact] + public async Task EnqueueBatchAsync_UsesDestinationOverrideAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + + await queue.EnqueueBatchAsync([ + new PreviewWorkItem { Data = "one" }, + new PreviewWorkItem { Data = "two" } + ], new EnqueueOptions { Destination = "custom-work" }, cancellationToken); + + var first = await queue.ReceiveAsync(new ReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new ReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + + Assert.NotNull(first); + Assert.NotNull(second); + Assert.Equal("one", first.Message.Data); + Assert.Equal("two", second.Message.Data); + + await first.CompleteAsync(cancellationToken); + await second.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task RejectAsync_WithRetry_RedeliversAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cancellationToken); + + var first = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(first); + + await first.RejectAsync(cancellationToken: cancellationToken); + + var second = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(second); + Assert.Equal(first.Id, second.Id); + Assert.Equal(2, second.Attempts); + Assert.Equal("retry", second.Message.Data); + + await second.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task RejectAsync_WithoutRetry_DeadLettersAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(message); + + await message.RejectAsync(retry: false, reason: "validation", cancellationToken: cancellationToken); + + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(1, stats.Deadletter); + Assert.Equal(0, stats.Working); + } + + [Fact] + public async Task StartWorkingAsync_WithAutoAck_CompletesMessageAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var handled = new AsyncCountdownEvent(1); + + var worker = queue.StartWorkingAsync((message, _) => + { + Assert.Equal("work", message.Message.Data); + handled.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "work" }, cancellationToken: cts.Token); + await handled.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await worker); + + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(1, stats.Completed); + } + + [Fact] + public async Task EnqueueAsync_WithDelay_DelaysVisibilityAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMilliseconds(250) }, cancellationToken); + + var immediate = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + Assert.Null(immediate); + + var delayed = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + Assert.NotNull(delayed); + Assert.Equal("later", delayed.Message.Data); + await delayed.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new EnqueueOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); + + var received = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + Assert.Null(received); + + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(1, stats.Deadletter); + } + + [Fact] + public async Task ReceiveAsync_WithPoisonPayload_DeadLettersAndThrowsQueueExceptionAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + + await transport.SendAsync("preview-work-item", [ + new TransportMessage + { + Body = "not-json"u8.ToArray(), + Headers = MessageHeaders.Create([ + new KeyValuePair(KnownHeaders.MessageType, typeof(PreviewWorkItem).FullName!) + ]) + } + ], new TransportSendOptions(), cancellationToken); + + await Assert.ThrowsAsync(async () => + await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); + + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(1, stats.Deadletter); + Assert.Equal(0, stats.Working); + } + + private sealed class PreviewWorkItem + { + public string? Data { get; set; } + } +} \ No newline at end of file From 36e8016a6d6099678105a0c50abff473f74af960 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 16:43:22 -0500 Subject: [PATCH 03/57] feat: add transport-backed pubsub --- src/Foundatio/Messaging/PubSub.cs | 301 ++++++++++++++++++ .../Foundatio.Tests/Messaging/PubSubTests.cs | 185 +++++++++++ 2 files changed, 486 insertions(+) create mode 100644 src/Foundatio/Messaging/PubSub.cs create mode 100644 tests/Foundatio.Tests/Messaging/PubSubTests.cs diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs new file mode 100644 index 000000000..ce1b6ee43 --- /dev/null +++ b/src/Foundatio/Messaging/PubSub.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Queues; +using Foundatio.Serializer; +using Foundatio.Utility; + +namespace Foundatio.Messaging; + +public sealed record PublishOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public TimeSpan? Delay { get; init; } + public DateTimeOffset? DeliverAt { get; init; } + public TimeSpan? TimeToLive { get; init; } + public string? CorrelationId { get; init; } + public string? DeduplicationId { get; init; } + public string? Topic { get; init; } + public MessageHeaders? Headers { get; init; } +} + +public sealed record SubscriptionOptions +{ + public string? Topic { get; init; } + public string? Subscription { get; init; } + public AckMode AckMode { get; init; } = AckMode.Auto; + public int MaxConcurrency { get; init; } = 1; + public int MaxAttempts { get; init; } = 5; +} + +public sealed record PubSubOptions +{ + public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; + public string ContentType { get; init; } = "application/json"; + public Func? TopicResolver { get; init; } + public Func? MessageTypeResolver { get; init; } + public Func? SubscriptionResolver { get; init; } +} + +public interface IPubSub : IAsyncDisposable +{ + Task PublishAsync(T message, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SubscribeAsync(Func, CancellationToken, Task> handler, SubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; +} + +public sealed class PubSub : IPubSub +{ + private readonly IMessageTransport _transport; + private readonly PubSubOptions _options; + private int _isDisposed; + + public PubSub(IMessageTransport transport, PubSubOptions? options = null) + { + _transport = transport ?? throw new ArgumentNullException(nameof(transport)); + _options = options ?? new PubSubOptions(); + } + + public async Task PublishAsync(T message, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(message); + ThrowIfDisposed(); + + options ??= new PublishOptions(); + string topic = GetTopic(typeof(T), options.Topic); + await EnsureTopicAsync(topic, cancellationToken).AnyContext(); + + var result = await _transport.SendAsync(topic, [CreateTransportMessage(message, options)], CreateSendOptions(options), cancellationToken).AnyContext(); + var item = result.Items.Count > 0 ? result.Items[0] : null; + if (item is null || !item.Success) + throw new MessageBusException($"Unable to publish message to \"{topic}\": {item?.ErrorCode ?? "unknown error"}"); + } + + public async Task PublishBatchAsync(IEnumerable messages, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(messages); + ThrowIfDisposed(); + + options ??= new PublishOptions(); + string topic = GetTopic(typeof(T), options.Topic); + await EnsureTopicAsync(topic, cancellationToken).AnyContext(); + + var transportMessages = messages.Select(message => + { + ArgumentNullException.ThrowIfNull(message); + return CreateTransportMessage(message, options); + }).ToArray(); + + if (transportMessages.Length == 0) + return; + + var result = await _transport.SendAsync(topic, transportMessages, CreateSendOptions(options), cancellationToken).AnyContext(); + if (!result.AllSucceeded) + throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{topic}\"."); + } + + public async Task SubscribeAsync(Func, CancellationToken, Task> handler, SubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(handler); + ThrowIfDisposed(); + + options ??= new SubscriptionOptions(); + string topic = GetTopic(typeof(T), options.Topic); + string subscription = GetSubscription(typeof(T), topic, options.Subscription); + await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); + + if (_transport is ISupportsPush push) + { + await using var pushSubscription = await push.SubscribeAsync(subscription, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + return; + } + + if (_transport is not ISupportsPull pull) + throw new MessageBusException($"Transport \"{_transport.GetType().Name}\" does not support subscriptions."); + + while (!cancellationToken.IsCancellationRequested) + { + var entries = await pull.ReceiveAsync(subscription, new ReceiveRequest + { + MaxMessages = Math.Max(1, options.MaxConcurrency), + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + + foreach (var entry in entries) + { + var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); + await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); + } + } + } + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return ValueTask.CompletedTask; + + return _transport.DisposeAsync(); + } + + private async Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) + { + if (_transport is ISupportsProvisioning provisioning) + await provisioning.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken).AnyContext(); + } + + private async Task EnsureSubscriptionAsync(string topic, string subscription, CancellationToken cancellationToken) + { + if (_transport is ISupportsProvisioning provisioning) + { + await provisioning.EnsureAsync([ + new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }, + new DestinationDeclaration { Name = subscription, Role = DestinationRole.Subscription, Source = topic } + ], cancellationToken).AnyContext(); + } + } + + private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class + { + try + { + var message = _options.Serializer.Deserialize(entry.Body); + if (message is null) + throw new MessageBusException($"Message \"{entry.Id}\" deserialized to null."); + + return new ReceivedMessage(_transport, entry, message, cancellationToken); + } + catch (Exception ex) when (ex is not MessageBusException) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + throw new MessageBusException($"Unable to deserialize message \"{entry.Id}\".", ex); + } + catch (MessageBusException) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + throw; + } + } + + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, SubscriptionOptions options, CancellationToken cancellationToken) where T : class + { + try + { + await handler(message, cancellationToken).AnyContext(); + + if (options.AckMode == AckMode.Auto && !message.IsHandled) + await message.CompleteAsync(cancellationToken).AnyContext(); + } + catch + { + if (!message.IsHandled) + await message.RejectAsync(message.Attempts < options.MaxAttempts, "handler-error", cancellationToken).AnyContext(); + } + } + + private TransportMessage CreateTransportMessage(T message, PublishOptions options) where T : class + { + var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() + .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) + .Set(KnownHeaders.ContentType, _options.ContentType) + .Set(KnownHeaders.Priority, options.Priority.ToString()); + + if (!String.IsNullOrEmpty(options.CorrelationId)) + headers.Set(KnownHeaders.CorrelationId, options.CorrelationId); + + if (Activity.Current is { } activity) + { + if (!String.IsNullOrEmpty(activity.Id)) + headers.SetIfMissing(KnownHeaders.TraceParent, activity.Id); + + if (!String.IsNullOrEmpty(activity.TraceStateString)) + headers.SetIfMissing(KnownHeaders.TraceState, activity.TraceStateString); + } + + if (options.TimeToLive is { } ttl) + headers.Set(KnownHeaders.Expiration, DateTimeOffset.UtcNow.Add(ttl).ToString("O", CultureInfo.InvariantCulture)); + + return new TransportMessage + { + Body = _options.Serializer.SerializeToBytes(message), + Headers = headers.Build() + }; + } + + private static TransportSendOptions CreateSendOptions(PublishOptions options) + { + return new TransportSendOptions + { + Priority = options.Priority, + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? DateTimeOffset.UtcNow.Add(delay) : null), + DeduplicationId = options.DeduplicationId + }; + } + + private string GetTopic(Type messageType, string? topic) + { + return !String.IsNullOrEmpty(topic) + ? topic + : (_options.TopicResolver?.Invoke(messageType) ?? ToKebabCase(messageType.Name)); + } + + private string GetSubscription(Type messageType, string topic, string? subscription) + { + return !String.IsNullOrEmpty(subscription) + ? subscription + : (_options.SubscriptionResolver?.Invoke(messageType, topic) ?? $"{topic}.{ToKebabCase(messageType.Name)}"); + } + + private string GetMessageType(Type messageType) + { + return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; + } + + private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) + { + if (_transport is ISupportsDeadLetter deadLetter) + await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + else + await _transport.AbandonAsync(entry, cancellationToken).AnyContext(); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + } + + private static string ToKebabCase(string value) + { + if (String.IsNullOrEmpty(value)) + return value; + + Span buffer = stackalloc char[value.Length * 2]; + int position = 0; + for (int index = 0; index < value.Length; index++) + { + char current = value[index]; + if (Char.IsUpper(current)) + { + if (index > 0) + buffer[position++] = '-'; + + buffer[position++] = Char.ToLowerInvariant(current); + } + else + { + buffer[position++] = current; + } + } + + return new String(buffer[..position]); + } +} diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs new file mode 100644 index 000000000..d4a5063da --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Tests.Extensions; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class PubSubTests +{ + [Fact] + public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var firstReceived = new AsyncCountdownEvent(1); + var secondReceived = new AsyncCountdownEvent(1); + + var first = pubSub.SubscribeAsync((message, _) => + { + Assert.Equal("published", message.Message.Data); + firstReceived.Signal(); + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "subscriber-a" }, cts.Token); + + var second = pubSub.SubscribeAsync((message, _) => + { + Assert.Equal("published", message.Message.Data); + secondReceived.Signal(); + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "subscriber-b" }, cts.Token); + + await pubSub.PublishAsync(new PreviewEvent { Data = "published" }, cancellationToken: cancellationToken); + + await firstReceived.WaitAsync(TimeSpan.FromSeconds(2)); + await secondReceived.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await first); + await Assert.ThrowsAnyAsync(async () => await second); + + var firstStats = await transport.GetStatsAsync("subscriber-a", cancellationToken); + var secondStats = await transport.GetStatsAsync("subscriber-b", cancellationToken); + Assert.Equal(1, firstStats.Completed); + Assert.Equal(1, secondStats.Completed); + } + + [Fact] + public async Task PublishBatchAsync_DeliversAllMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new AsyncCountdownEvent(2); + + var subscription = pubSub.SubscribeAsync((message, _) => + { + Assert.StartsWith("batch-", message.Message.Data); + received.Signal(); + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "batch-subscription" }, cts.Token); + + await pubSub.PublishBatchAsync([ + new PreviewEvent { Data = "batch-one" }, + new PreviewEvent { Data = "batch-two" } + ], cancellationToken: cancellationToken); + + await received.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await subscription); + + var stats = await transport.GetStatsAsync("batch-subscription", cancellationToken); + Assert.Equal(2, stats.Completed); + } + + [Fact] + public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var pubSub = new PubSub(new InMemoryMessageTransport()); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + + var subscription = pubSub.SubscribeAsync((message, _) => + { + received.TrySetResult(message); + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); + + await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new PublishOptions + { + CorrelationId = "corr-456", + Priority = MessagePriority.High, + Headers = MessageHeaders.Create([ + new KeyValuePair("tenant", "acme") + ]) + }, cancellationToken); + + var completed = await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken)); + Assert.Equal(received.Task, completed); + + var message = await received.Task; + Assert.Equal("metadata", message.Message.Data); + Assert.Equal("corr-456", message.CorrelationId); + Assert.Equal(MessagePriority.High, message.Priority); + Assert.Equal("acme", message.Headers["tenant"]); + Assert.Equal(typeof(PreviewEvent).FullName, message.MessageType); + + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await subscription); + } + + [Fact] + public async Task PublishAsync_WithDelay_DelaysDeliveryAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var pubSub = new PubSub(new InMemoryMessageTransport()); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new AsyncCountdownEvent(1); + + var subscription = pubSub.SubscribeAsync((_, _) => + { + received.Signal(); + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); + + await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PublishOptions { Delay = TimeSpan.FromMilliseconds(250) }, cancellationToken); + + await Assert.ThrowsAsync(async () => await received.WaitAsync(TimeSpan.FromMilliseconds(50))); + await received.WaitAsync(TimeSpan.FromSeconds(2)); + + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await subscription); + } + + [Fact] + public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new AsyncCountdownEvent(2); + int attempts = 0; + + var subscription = pubSub.SubscribeAsync((message, _) => + { + attempts++; + Assert.Equal(attempts, message.Attempts); + received.Signal(); + + if (attempts == 1) + throw new InvalidOperationException("try again"); + + return Task.CompletedTask; + }, new SubscriptionOptions { Subscription = "retry-subscription", MaxAttempts = 2 }, cts.Token); + + await pubSub.PublishAsync(new PreviewEvent { Data = "retry" }, cancellationToken: cancellationToken); + + await received.WaitAsync(TimeSpan.FromSeconds(2)); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await subscription); + + var stats = await transport.GetStatsAsync("retry-subscription", cancellationToken); + Assert.Equal(1, stats.Completed); + Assert.Equal(1, stats.Abandoned); + } + + private sealed class PreviewEvent + { + public string? Data { get; set; } + } +} From 1d78bf42272d5016a1ab872fb7082c41fd2a1e44 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 16:50:08 -0500 Subject: [PATCH 04/57] feat: add in-memory job runtime --- src/Foundatio/Jobs/JobRuntime.cs | 517 ++++++++++++++++++ tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 224 ++++++++ 2 files changed, 741 insertions(+) create mode 100644 src/Foundatio/Jobs/JobRuntime.cs create mode 100644 tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs new file mode 100644 index 000000000..60496669e --- /dev/null +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -0,0 +1,517 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace Foundatio.Jobs; + +public enum JobStatus +{ + Queued, + Scheduled, + Processing, + Completed, + Failed, + Cancelled, + DeadLettered +} + +public enum ScheduledDispatchKind +{ + QueueMessage, + PubSubMessage, + JobOccurrence +} + +public sealed record JobState +{ + public required string JobId { get; init; } + public required string Name { get; init; } + public JobStatus Status { get; init; } = JobStatus.Queued; + public int? Progress { get; init; } + public string? ProgressMessage { get; init; } + public int Attempt { get; init; } + public string? NodeId { get; init; } + public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow; + public DateTimeOffset LastUpdatedUtc { get; init; } = DateTimeOffset.UtcNow; + public DateTimeOffset? StartedUtc { get; init; } + public DateTimeOffset? CompletedUtc { get; init; } + public DateTimeOffset? LeaseExpiresUtc { get; init; } + public string? Error { get; init; } + public bool CancellationRequested { get; init; } + public DateTimeOffset? ScheduledForUtc { get; init; } +} + +public sealed record JobStatePatch +{ + public JobStatus? Status { get; init; } + public int? Progress { get; init; } + public string? ProgressMessage { get; init; } + public string? Error { get; init; } + public int AttemptDelta { get; init; } + public string? NodeId { get; init; } + public DateTimeOffset? LeaseExpiresUtc { get; init; } + public DateTimeOffset? LastUpdatedUtc { get; init; } + public DateTimeOffset? StartedUtc { get; init; } + public DateTimeOffset? CompletedUtc { get; init; } + public bool? CancellationRequested { get; init; } +} + +public sealed record JobQuery +{ + public string? Name { get; init; } + public JobStatus? Status { get; init; } + public int Limit { get; init; } = 100; +} + +public sealed record ScheduledDispatchState +{ + public required string DispatchId { get; init; } + public ScheduledDispatchKind Kind { get; init; } + public required string Destination { get; init; } + public required ReadOnlyMemory Body { get; init; } + public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; + public TransportSendOptions Options { get; init; } = new(); + public DateTimeOffset DueUtc { get; init; } + public string? ClaimOwner { get; init; } + public DateTimeOffset? ClaimExpiresUtc { get; init; } + public int Attempts { get; init; } + public string? JobId { get; init; } +} + +public sealed record RunJobOptions +{ + public string? JobId { get; init; } + public string? Name { get; init; } + public string? NodeId { get; init; } +} + +public interface IJobMonitor +{ + Task GetAsync(string jobId, CancellationToken cancellationToken = default); + Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default); +} + +public interface IJobClient : IJobMonitor +{ + Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob; + Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); +} + +public interface IJobRuntimeStore : IJobMonitor +{ + Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default); + Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default); + Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); + Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); + Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken cancellationToken = default); + Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken cancellationToken = default); + Task IncrementAttemptAsync(string jobId, CancellationToken cancellationToken = default); + Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); + Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default); + Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default); + Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); + Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken cancellationToken = default); + Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken cancellationToken = default); +} + +public sealed class InMemoryJobRuntimeStore : IJobRuntimeStore +{ + private readonly ConcurrentDictionary _jobs = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _dispatches = new(StringComparer.Ordinal); + private readonly TimeProvider _timeProvider; + private readonly object _lock = new(); + + public InMemoryJobRuntimeStore(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(initial); + cancellationToken.ThrowIfCancellationRequested(); + + var now = _timeProvider.GetUtcNow(); + _jobs.TryAdd(initial.JobId, initial with + { + CreatedUtc = initial.CreatedUtc == default ? now : initial.CreatedUtc, + LastUpdatedUtc = initial.LastUpdatedUtc == default ? now : initial.LastUpdatedUtc + }); + + return Task.CompletedTask; + } + + public Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _jobs.TryGetValue(jobId, out var state); + return Task.FromResult(state); + } + + public Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + cancellationToken.ThrowIfCancellationRequested(); + + IEnumerable results = _jobs.Values; + if (!String.IsNullOrEmpty(query.Name)) + results = results.Where(s => String.Equals(s.Name, query.Name, StringComparison.Ordinal)); + + if (query.Status is { } status) + results = results.Where(s => s.Status == status); + + return Task.FromResult>(results + .OrderByDescending(s => s.LastUpdatedUtc) + .Take(Math.Max(1, query.Limit)) + .ToArray()); + } + + public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current) || current.Status != expectedStatus) + return Task.FromResult(false); + + _jobs[jobId] = ApplyPatch(current, patch) with + { + Status = newStatus, + LastUpdatedUtc = patch?.LastUpdatedUtc ?? _timeProvider.GetUtcNow() + }; + return Task.FromResult(true); + } + } + + public Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(nodeId); + + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current)) + return Task.FromResult(false); + + var now = _timeProvider.GetUtcNow(); + if (!String.IsNullOrEmpty(current.NodeId) && current.LeaseExpiresUtc is { } leaseExpires && leaseExpires > now && current.NodeId != nodeId) + return Task.FromResult(false); + + _jobs[jobId] = current with + { + NodeId = nodeId, + LeaseExpiresUtc = now.Add(lease), + LastUpdatedUtc = now + }; + return Task.FromResult(true); + } + } + + public Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current) || current.NodeId != nodeId) + return Task.FromResult(false); + + var now = _timeProvider.GetUtcNow(); + _jobs[jobId] = current with + { + LeaseExpiresUtc = now.Add(lease), + LastUpdatedUtc = now + }; + return Task.FromResult(true); + } + } + + public Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current) || current.NodeId != nodeId) + return Task.FromResult(false); + + _jobs[jobId] = current with + { + NodeId = null, + LeaseExpiresUtc = null, + LastUpdatedUtc = _timeProvider.GetUtcNow() + }; + return Task.FromResult(true); + } + } + + public Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + UpdateJob(jobId, state => state with + { + Progress = percent ?? state.Progress, + ProgressMessage = message ?? state.ProgressMessage, + LastUpdatedUtc = _timeProvider.GetUtcNow() + }); + + return Task.CompletedTask; + } + + public Task IncrementAttemptAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + UpdateJob(jobId, state => state with { Attempt = state.Attempt + 1, LastUpdatedUtc = _timeProvider.GetUtcNow() }); + return Task.CompletedTask; + } + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(UpdateJob(jobId, state => state with + { + CancellationRequested = true, + LastUpdatedUtc = _timeProvider.GetUtcNow() + })); + } + + public Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(_jobs.TryGetValue(jobId, out var state) && state.CancellationRequested); + } + + public Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dispatch); + cancellationToken.ThrowIfCancellationRequested(); + _dispatches.TryAdd(dispatch.DispatchId, dispatch); + return Task.CompletedTask; + } + + public Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(nodeId); + + lock (_lock) + { + var due = _dispatches.Values + .Where(d => d.DueUtc <= now && (String.IsNullOrEmpty(d.ClaimOwner) || d.ClaimExpiresUtc <= now)) + .OrderBy(d => d.DueUtc) + .Take(Math.Max(1, limit)) + .ToArray(); + + for (int index = 0; index < due.Length; index++) + { + var claimed = due[index] with + { + ClaimOwner = nodeId, + ClaimExpiresUtc = now.Add(lease), + Attempts = due[index].Attempts + 1 + }; + _dispatches[claimed.DispatchId] = claimed; + due[index] = claimed; + } + + return Task.FromResult>(due); + } + } + + public Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (_dispatches.TryGetValue(dispatchId, out var dispatch) && dispatch.ClaimOwner == nodeId) + _dispatches.TryRemove(dispatchId, out _); + } + + return Task.CompletedTask; + } + + public Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + if (_dispatches.TryGetValue(dispatchId, out var dispatch) && dispatch.ClaimOwner == nodeId) + { + _dispatches[dispatchId] = dispatch with + { + DueUtc = nextDueUtc, + ClaimOwner = null, + ClaimExpiresUtc = null + }; + } + } + + return Task.CompletedTask; + } + + private bool UpdateJob(string jobId, Func update) + { + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current)) + return false; + + _jobs[jobId] = update(current); + return true; + } + } + + private JobState ApplyPatch(JobState state, JobStatePatch? patch) + { + if (patch is null) + return state; + + return state with + { + Status = patch.Status ?? state.Status, + Progress = patch.Progress ?? state.Progress, + ProgressMessage = patch.ProgressMessage ?? state.ProgressMessage, + Error = patch.Error ?? state.Error, + Attempt = state.Attempt + patch.AttemptDelta, + NodeId = patch.NodeId ?? state.NodeId, + LeaseExpiresUtc = patch.LeaseExpiresUtc ?? state.LeaseExpiresUtc, + LastUpdatedUtc = patch.LastUpdatedUtc ?? state.LastUpdatedUtc, + StartedUtc = patch.StartedUtc ?? state.StartedUtc, + CompletedUtc = patch.CompletedUtc ?? state.CompletedUtc, + CancellationRequested = patch.CancellationRequested ?? state.CancellationRequested + }; + } +} + +public sealed class JobClient : IJobClient +{ + private readonly IJobRuntimeStore _store; + private readonly IServiceProvider _serviceProvider; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + + public JobClient(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _timeProvider = timeProvider ?? TimeProvider.System; + _nodeId = !String.IsNullOrEmpty(nodeId) + ? nodeId + : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + } + + public Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + return _store.GetAsync(jobId, cancellationToken); + } + + public Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default) + { + return _store.QueryAsync(query, cancellationToken); + } + + public async Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob + { + options ??= new RunJobOptions(); + string jobId = options.JobId ?? Guid.NewGuid().ToString("N"); + string name = options.Name ?? typeof(TJob).Name; + string nodeId = options.NodeId ?? _nodeId; + var now = _timeProvider.GetUtcNow(); + + await _store.CreateIfAbsentAsync(new JobState + { + JobId = jobId, + Name = name, + Status = JobStatus.Queued, + CreatedUtc = now, + LastUpdatedUtc = now + }, cancellationToken).ConfigureAwait(false); + + if (!await _store.TryTransitionAsync(jobId, JobStatus.Queued, JobStatus.Processing, new JobStatePatch + { + NodeId = nodeId, + StartedUtc = now, + LeaseExpiresUtc = now.AddMinutes(5), + AttemptDelta = 1 + }, cancellationToken).ConfigureAwait(false)) + { + return jobId; + } + + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var cancellationWatcher = WatchCancellation(jobId, linkedCancellationTokenSource); + var job = ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider); + + try + { + var result = await job.TryRunAsync(linkedCancellationTokenSource.Token).ConfigureAwait(false); + var completedAt = _timeProvider.GetUtcNow(); + if (result.IsCancelled) + { + await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Cancelled, new JobStatePatch + { + Error = result.Message, + CompletedUtc = completedAt, + LeaseExpiresUtc = null + }, CancellationToken.None).ConfigureAwait(false); + } + else if (result.IsSuccess) + { + await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Completed, new JobStatePatch + { + CompletedUtc = completedAt, + LeaseExpiresUtc = null, + Progress = 100 + }, CancellationToken.None).ConfigureAwait(false); + } + else + { + await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Failed, new JobStatePatch + { + Error = result.Message, + CompletedUtc = completedAt, + LeaseExpiresUtc = null + }, CancellationToken.None).ConfigureAwait(false); + } + } + finally + { + await _store.ReleaseClaimAsync(jobId, nodeId, CancellationToken.None).ConfigureAwait(false); + } + + return jobId; + } + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + { + return _store.RequestCancellationAsync(jobId, cancellationToken); + } + + private IDisposable WatchCancellation(string jobId, CancellationTokenSource cancellationTokenSource) + { + return new Timer(_ => _ = PollCancellationAsync(jobId, cancellationTokenSource), null, TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50)); + } + + private async Task PollCancellationAsync(string jobId, CancellationTokenSource cancellationTokenSource) + { + if (cancellationTokenSource.IsCancellationRequested) + return; + + try + { + if (await _store.IsCancellationRequestedAsync(jobId, CancellationToken.None).ConfigureAwait(false)) + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + } +} diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs new file mode 100644 index 000000000..522d389bf --- /dev/null +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -0,0 +1,224 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Foundatio.Tests.Jobs; + +public class JobRuntimeTests +{ + [Fact] + public async Task CreateIfAbsentAsync_WithExistingJob_DoesNotOverwriteStateAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + + await store.CreateIfAbsentAsync(new JobState + { + JobId = "job-1", + Name = "first", + Status = JobStatus.Queued + }, cancellationToken); + + await store.CreateIfAbsentAsync(new JobState + { + JobId = "job-1", + Name = "second", + Status = JobStatus.Failed, + Error = "should not overwrite" + }, cancellationToken); + + var state = await store.GetAsync("job-1", cancellationToken); + + Assert.NotNull(state); + Assert.Equal("first", state.Name); + Assert.Equal(JobStatus.Queued, state.Status); + Assert.Null(state.Error); + } + + [Fact] + public async Task TryClaimAsync_WhenLeaseIsHeldByAnotherNode_ReturnsFalseUntilLeaseExpiresAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + + await store.CreateIfAbsentAsync(new JobState + { + JobId = "job-1", + Name = "test", + Status = JobStatus.Queued + }, cancellationToken); + + Assert.True(await store.TryClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(1), cancellationToken)); + Assert.False(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(1), cancellationToken)); + Assert.True(await store.ReleaseClaimAsync("job-1", "node-a", cancellationToken)); + Assert.True(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(1), cancellationToken)); + + var state = await store.GetAsync("job-1", cancellationToken); + Assert.NotNull(state); + Assert.Equal("node-b", state.NodeId); + Assert.NotNull(state.LeaseExpiresUtc); + } + + [Fact] + public async Task ClaimDueDispatchesAsync_ClaimsReleasesAndCompletesDueDispatchesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var now = DateTimeOffset.UtcNow; + + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = "dispatch-1", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = "work", + Body = "hello"u8.ToArray(), + DueUtc = now.AddSeconds(-1) + }, cancellationToken); + + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = "dispatch-2", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = "work", + Body = "later"u8.ToArray(), + DueUtc = now.AddHours(1) + }, cancellationToken); + + var claimed = await store.ClaimDueDispatchesAsync(now, 10, "node-a", TimeSpan.FromMinutes(1), cancellationToken); + + var dispatch = Assert.Single(claimed); + Assert.Equal("dispatch-1", dispatch.DispatchId); + Assert.Equal("node-a", dispatch.ClaimOwner); + Assert.Equal(1, dispatch.Attempts); + + var claimedAgain = await store.ClaimDueDispatchesAsync(now, 10, "node-b", TimeSpan.FromMinutes(1), cancellationToken); + Assert.Empty(claimedAgain); + + await store.ReleaseDispatchAsync("dispatch-1", "node-a", now.AddSeconds(-1), cancellationToken); + + var reclaimed = await store.ClaimDueDispatchesAsync(now, 10, "node-b", TimeSpan.FromMinutes(1), cancellationToken); + dispatch = Assert.Single(reclaimed); + Assert.Equal("node-b", dispatch.ClaimOwner); + Assert.Equal(2, dispatch.Attempts); + + await store.CompleteDispatchAsync("dispatch-1", "node-b", cancellationToken); + + var afterComplete = await store.ClaimDueDispatchesAsync(now, 10, "node-c", TimeSpan.FromMinutes(1), cancellationToken); + Assert.Empty(afterComplete); + } + + [Fact] + public async Task RunAsync_WhenJobSucceeds_TracksCompletedStateAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobRuntimeProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + + string jobId = await client.RunAsync(new RunJobOptions { JobId = "job-1" }, cancellationToken); + + var state = await client.GetAsync(jobId, cancellationToken); + Assert.NotNull(state); + Assert.Equal(1, probe.RunCount); + Assert.Equal(JobStatus.Completed, state.Status); + Assert.Equal(1, state.Attempt); + Assert.Equal(100, state.Progress); + Assert.NotNull(state.StartedUtc); + Assert.NotNull(state.CompletedUtc); + Assert.Null(state.NodeId); + Assert.Null(state.LeaseExpiresUtc); + } + + [Fact] + public async Task RequestCancellationAsync_WhenJobIsRunning_CancelsAndTracksStateAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobRuntimeProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + + var runTask = client.RunAsync(new RunJobOptions { JobId = "job-1" }, cancellationToken); + await probe.Started.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + + Assert.True(await client.RequestCancellationAsync("job-1", cancellationToken)); + + await probe.Cancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + string jobId = await runTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); + var state = await client.GetAsync(jobId, cancellationToken); + + Assert.NotNull(state); + Assert.Equal(JobStatus.Cancelled, state.Status); + Assert.True(state.CancellationRequested); + Assert.NotNull(state.CompletedUtc); + Assert.Null(state.NodeId); + Assert.Null(state.LeaseExpiresUtc); + } + + private sealed class JobRuntimeProbe + { + private int _runCount; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int RunCount => Volatile.Read(ref _runCount); + + public void RecordRun() + { + Interlocked.Increment(ref _runCount); + } + } + + private sealed class SuccessfulTrackedJob : IJob + { + private readonly JobRuntimeProbe _probe; + + public SuccessfulTrackedJob(JobRuntimeProbe probe) + { + _probe = probe; + } + + public Task RunAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _probe.RecordRun(); + return Task.FromResult(JobResult.Success); + } + } + + private sealed class CancellableTrackedJob : IJob + { + private readonly JobRuntimeProbe _probe; + + public CancellableTrackedJob(JobRuntimeProbe probe) + { + _probe = probe; + } + + public async Task RunAsync(CancellationToken cancellationToken = default) + { + _probe.Started.TrySetResult(); + + try + { + await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); + return JobResult.Success; + } + catch (OperationCanceledException) + { + _probe.Cancelled.TrySetResult(); + throw; + } + } + } +} From 2bfb3d5e03e4cb381e0e7471f1b3efd1b28e69fe Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 16:57:27 -0500 Subject: [PATCH 05/57] feat: add durable job scheduler --- src/Foundatio/Jobs/JobRuntime.cs | 16 +- src/Foundatio/Jobs/JobScheduler.cs | 427 ++++++++++++++++++ .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 193 ++++++++ 3 files changed, 633 insertions(+), 3 deletions(-) create mode 100644 src/Foundatio/Jobs/JobScheduler.cs create mode 100644 tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 60496669e..bf534df6b 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -99,6 +99,7 @@ public interface IJobMonitor public interface IJobClient : IJobMonitor { Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob; + Task RunAsync(Type jobType, RunJobOptions? options = null, CancellationToken cancellationToken = default); Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); } @@ -418,11 +419,20 @@ public Task> QueryAsync(JobQuery query, CancellationToke return _store.QueryAsync(query, cancellationToken); } - public async Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob + public Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob { + return RunAsync(typeof(TJob), options, cancellationToken); + } + + public async Task RunAsync(Type jobType, RunJobOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(jobType); + if (!typeof(IJob).IsAssignableFrom(jobType)) + throw new ArgumentException("Job type must implement IJob.", nameof(jobType)); + options ??= new RunJobOptions(); string jobId = options.JobId ?? Guid.NewGuid().ToString("N"); - string name = options.Name ?? typeof(TJob).Name; + string name = options.Name ?? jobType.Name; string nodeId = options.NodeId ?? _nodeId; var now = _timeProvider.GetUtcNow(); @@ -448,7 +458,7 @@ await _store.CreateIfAbsentAsync(new JobState using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); using var cancellationWatcher = WatchCancellation(jobId, linkedCancellationTokenSource); - var job = ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider); + var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); try { diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs new file mode 100644 index 000000000..135d148ce --- /dev/null +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -0,0 +1,427 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; + +namespace Foundatio.Jobs; + +public enum ScheduledJobScope +{ + Global, + PerNode +} + +public enum OverlapPolicy +{ + SkipIfRunning, + AllowConcurrent +} + +public sealed record ScheduledJobDefinition +{ + public required string Name { get; init; } + public required string Cron { get; init; } + public Type? JobType { get; init; } + public TimeZoneInfo? TimeZone { get; init; } + public ScheduledJobScope Scope { get; init; } = ScheduledJobScope.Global; + public OverlapPolicy Overlap { get; init; } = OverlapPolicy.SkipIfRunning; + public TimeSpan? MisfireWindow { get; init; } + public int MaxRetries { get; init; } = 3; + public bool Enabled { get; init; } = true; +} + +public interface IJobScheduler +{ + Task ScheduleAsync(ScheduledJobDefinition definition, CancellationToken cancellationToken = default); + Task UnscheduleAsync(string name, CancellationToken cancellationToken = default); + Task> GetSchedulesAsync(CancellationToken cancellationToken = default); +} + +public sealed class InMemoryJobScheduler : IJobScheduler +{ + private readonly ConcurrentDictionary _definitions = new(StringComparer.Ordinal); + + public Task ScheduleAsync(ScheduledJobDefinition definition, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(definition); + ArgumentException.ThrowIfNullOrEmpty(definition.Name); + ArgumentException.ThrowIfNullOrEmpty(definition.Cron); + cancellationToken.ThrowIfCancellationRequested(); + + if (definition.MaxRetries < 0) + throw new ArgumentOutOfRangeException(nameof(definition), definition.MaxRetries, "MaxRetries must be greater than or equal to zero."); + + if (definition.JobType is not null && !typeof(IJob).IsAssignableFrom(definition.JobType)) + throw new ArgumentException("JobType must implement IJob.", nameof(definition)); + + JobScheduleProcessor.ValidateCron(definition.Cron); + _definitions[definition.Name] = definition; + return Task.CompletedTask; + } + + public Task UnscheduleAsync(string name, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(name); + cancellationToken.ThrowIfCancellationRequested(); + _definitions.TryRemove(name, out _); + return Task.CompletedTask; + } + + public Task> GetSchedulesAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult>(_definitions.Values.OrderBy(d => d.Name, StringComparer.Ordinal).ToArray()); + } +} + +public sealed class JobScheduleProcessor +{ + private static readonly TimeSpan DefaultLease = TimeSpan.FromMinutes(5); + private static readonly TimeSpan DefaultMisfireWindow = TimeSpan.FromMinutes(1); + + private readonly IJobScheduler _scheduler; + private readonly IJobRuntimeStore _store; + private readonly IJobClient _jobClient; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + + public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobClient jobClient, TimeProvider? timeProvider = null, string? nodeId = null) + { + _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); + _store = store ?? throw new ArgumentNullException(nameof(store)); + _jobClient = jobClient ?? throw new ArgumentNullException(nameof(jobClient)); + _timeProvider = timeProvider ?? TimeProvider.System; + _nodeId = !String.IsNullOrEmpty(nodeId) + ? nodeId + : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + } + + public Task> EnqueueDueOccurrencesAsync(CancellationToken cancellationToken = default) + { + return EnqueueDueOccurrencesAsync(_timeProvider.GetUtcNow(), cancellationToken); + } + + public async Task> EnqueueDueOccurrencesAsync(DateTimeOffset utcNow, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var scheduled = new List(); + var definitions = await _scheduler.GetSchedulesAsync(cancellationToken).ConfigureAwait(false); + + foreach (var definition in definitions) + { + if (!definition.Enabled) + continue; + + var cron = CronSchedule.Parse(definition.Cron); + var scheduledForUtc = cron.GetLastOccurrence(utcNow, definition.TimeZone ?? TimeZoneInfo.Utc, definition.MisfireWindow ?? DefaultMisfireWindow); + if (scheduledForUtc is null) + continue; + + string scopeKey = GetScopeKey(definition); + string jobId = CreateOccurrenceId(definition.Name, scheduledForUtc.Value, scopeKey); + + if (await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false) is not null) + continue; + + if (definition.Overlap == OverlapPolicy.SkipIfRunning && await HasActiveOccurrenceAsync(definition.Name, scopeKey, cancellationToken).ConfigureAwait(false)) + continue; + + await _store.CreateIfAbsentAsync(new JobState + { + JobId = jobId, + Name = definition.Name, + Status = JobStatus.Scheduled, + CreatedUtc = utcNow, + LastUpdatedUtc = utcNow, + ScheduledForUtc = scheduledForUtc + }, cancellationToken).ConfigureAwait(false); + + var dispatch = new ScheduledDispatchState + { + DispatchId = jobId, + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = definition.Name, + Body = Array.Empty(), + Headers = CreateOccurrenceHeaders(definition, scheduledForUtc.Value, scopeKey), + DueUtc = utcNow, + JobId = jobId + }; + + await _store.ScheduleDispatchAsync(dispatch, cancellationToken).ConfigureAwait(false); + scheduled.Add(dispatch); + } + + return scheduled; + } + + public Task RunDueOccurrencesAsync(CancellationToken cancellationToken = default) + { + return RunDueOccurrencesAsync(_timeProvider.GetUtcNow(), 100, null, cancellationToken); + } + + public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = 100, TimeSpan? lease = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var definitions = (await _scheduler.GetSchedulesAsync(cancellationToken).ConfigureAwait(false)) + .ToDictionary(d => d.Name, StringComparer.Ordinal); + + var dispatches = await _store.ClaimDueDispatchesAsync(utcNow, limit, _nodeId, lease ?? DefaultLease, cancellationToken).ConfigureAwait(false); + int completed = 0; + + foreach (var dispatch in dispatches) + { + if (dispatch.Kind != ScheduledDispatchKind.JobOccurrence) + { + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, dispatch.DueUtc, cancellationToken).ConfigureAwait(false); + continue; + } + + if (!definitions.TryGetValue(dispatch.Destination, out var definition) || !definition.Enabled || definition.JobType is null) + { + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + continue; + } + + string jobId = dispatch.JobId ?? dispatch.DispatchId; + + try + { + await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false); + await _jobClient.RunAsync(definition.JobType, new RunJobOptions + { + JobId = jobId, + Name = definition.Name, + NodeId = _nodeId + }, cancellationToken).ConfigureAwait(false); + + await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); + completed++; + } + catch + { + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + return completed; + } + + private async Task HasActiveOccurrenceAsync(string name, string scopeKey, CancellationToken cancellationToken) + { + var states = await _store.QueryAsync(new JobQuery { Name = name, Limit = 1000 }, cancellationToken).ConfigureAwait(false); + return states.Any(s => s.JobId.EndsWith($":{scopeKey}", StringComparison.Ordinal) && s.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing); + } + + private string GetScopeKey(ScheduledJobDefinition definition) + { + return definition.Scope == ScheduledJobScope.PerNode ? _nodeId : "global"; + } + + private static string CreateOccurrenceId(string name, DateTimeOffset scheduledForUtc, string scopeKey) + { + return $"{name}:{scheduledForUtc.UtcDateTime:yyyyMMddHHmmss}:{scopeKey}"; + } + + private static MessageHeaders CreateOccurrenceHeaders(ScheduledJobDefinition definition, DateTimeOffset scheduledForUtc, string scopeKey) + { + return MessageHeaders.Create([ + new KeyValuePair("job.name", definition.Name), + new KeyValuePair("job.scheduled_for", scheduledForUtc.UtcDateTime.ToString("O")), + new KeyValuePair("job.scope", scopeKey) + ]); + } + + internal static void ValidateCron(string expression) + { + CronSchedule.Parse(expression); + } + + private sealed class CronSchedule + { + private readonly CronFieldSet _second; + private readonly CronFieldSet _minute; + private readonly CronFieldSet _hour; + private readonly CronFieldSet _dayOfMonth; + private readonly CronFieldSet _month; + private readonly CronFieldSet _dayOfWeek; + + private CronSchedule(CronFieldSet second, CronFieldSet minute, CronFieldSet hour, CronFieldSet dayOfMonth, CronFieldSet month, CronFieldSet dayOfWeek) + { + _second = second; + _minute = minute; + _hour = hour; + _dayOfMonth = dayOfMonth; + _month = month; + _dayOfWeek = dayOfWeek; + } + + public static CronSchedule Parse(string expression) + { + ArgumentException.ThrowIfNullOrWhiteSpace(expression); + + var parts = expression.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 5 && parts.Length != 6) + throw new FormatException("Cron expressions must contain five fields, or six fields when seconds are included."); + + int offset = parts.Length == 6 ? 0 : -1; + return new CronSchedule( + offset == 0 ? CronFieldSet.Parse(parts[0], 0, 59) : CronFieldSet.Single(0, 0, 59), + CronFieldSet.Parse(parts[1 + offset], 0, 59), + CronFieldSet.Parse(parts[2 + offset], 0, 23), + CronFieldSet.Parse(parts[3 + offset], 1, 31, allowQuestion: true), + CronFieldSet.Parse(parts[4 + offset], 1, 12), + CronFieldSet.Parse(parts[5 + offset], 0, 6, allowQuestion: true, normalizeDayOfWeek: true)); + } + + public DateTimeOffset? GetLastOccurrence(DateTimeOffset utcNow, TimeZoneInfo timeZone, TimeSpan misfireWindow) + { + if (misfireWindow < TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(misfireWindow), misfireWindow, "MisfireWindow must be greater than or equal to zero."); + + var localNow = TimeZoneInfo.ConvertTime(utcNow, timeZone); + var candidate = new DateTimeOffset(localNow.Year, localNow.Month, localNow.Day, localNow.Hour, localNow.Minute, localNow.Second, localNow.Offset); + int secondsToSearch = Math.Max(1, (int)Math.Ceiling(misfireWindow.TotalSeconds)) + 1; + + for (int i = 0; i <= secondsToSearch; i++) + { + if (Matches(candidate.DateTime)) + return TimeZoneInfo.ConvertTime(candidate, TimeZoneInfo.Utc); + + candidate = candidate.AddSeconds(-1); + } + + return null; + } + + private bool Matches(DateTime local) + { + if (!_second.Contains(local.Second) || !_minute.Contains(local.Minute) || !_hour.Contains(local.Hour) || !_month.Contains(local.Month)) + return false; + + bool dayOfMonthMatches = _dayOfMonth.Contains(local.Day); + bool dayOfWeekMatches = _dayOfWeek.Contains((int)local.DayOfWeek); + return _dayOfMonth.IsAny || _dayOfWeek.IsAny + ? dayOfMonthMatches && dayOfWeekMatches + : dayOfMonthMatches || dayOfWeekMatches; + } + } + + private sealed class CronFieldSet + { + private readonly bool[] _values; + private readonly int _min; + + private CronFieldSet(bool[] values, int min, bool isAny) + { + _values = values; + _min = min; + IsAny = isAny; + } + + public bool IsAny { get; } + + public static CronFieldSet Single(int value, int min, int max) + { + var values = new bool[max - min + 1]; + values[value - min] = true; + return new CronFieldSet(values, min, false); + } + + public static CronFieldSet Parse(string expression, int min, int max, bool allowQuestion = false, bool normalizeDayOfWeek = false) + { + if (expression == "*" || (allowQuestion && expression == "?")) + return Any(min, max); + + var values = new bool[max - min + 1]; + foreach (string segment in expression.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + AddSegment(values, segment, min, max, allowQuestion, normalizeDayOfWeek); + + return new CronFieldSet(values, min, false); + } + + public bool Contains(int value) + { + int index = value - _min; + return index >= 0 && index < _values.Length && _values[index]; + } + + private static CronFieldSet Any(int min, int max) + { + var values = new bool[max - min + 1]; + Array.Fill(values, true); + return new CronFieldSet(values, min, true); + } + + private static void AddSegment(bool[] values, string segment, int min, int max, bool allowQuestion, bool normalizeDayOfWeek) + { + string[] stepParts = segment.Split('/', StringSplitOptions.TrimEntries); + if (stepParts.Length > 2) + throw new FormatException($"Invalid cron field segment '{segment}'."); + + int step = stepParts.Length == 2 ? ParseNumber(stepParts[1], 1, max) : 1; + string range = stepParts[0]; + + if (range == "*" || (allowQuestion && range == "?")) + { + AddRange(values, min, max, step, min, normalizeDayOfWeek); + return; + } + + string[] rangeParts = range.Split('-', StringSplitOptions.TrimEntries); + if (rangeParts.Length == 1) + { + int value = Normalize(ParseNumber(rangeParts[0], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); + EnsureInRange(value, min, max); + values[value - min] = true; + return; + } + + if (rangeParts.Length != 2) + throw new FormatException($"Invalid cron field segment '{segment}'."); + + int start = Normalize(ParseNumber(rangeParts[0], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); + int end = Normalize(ParseNumber(rangeParts[1], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); + EnsureInRange(start, min, max); + EnsureInRange(end, min, max); + + if (end < start) + throw new FormatException($"Invalid cron range '{segment}'."); + + AddRange(values, start, end, step, min, normalizeDayOfWeek); + } + + private static void AddRange(bool[] values, int start, int end, int step, int min, bool normalizeDayOfWeek) + { + for (int value = start; value <= end; value += step) + { + int normalized = Normalize(value, normalizeDayOfWeek); + values[normalized - min] = true; + } + } + + private static int ParseNumber(string value, int min, int max) + { + if (!Int32.TryParse(value, out int result) || result < min || result > max) + throw new FormatException($"Cron value '{value}' must be between {min} and {max}."); + + return result; + } + + private static int Normalize(int value, bool normalizeDayOfWeek) + { + return normalizeDayOfWeek && value == 7 ? 0 : value; + } + + private static void EnsureInRange(int value, int min, int max) + { + if (value < min || value > max) + throw new FormatException($"Cron value '{value}' must be between {min} and {max}."); + } + } +} diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs new file mode 100644 index 000000000..b886f1114 --- /dev/null +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -0,0 +1,193 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Foundatio.Tests.Jobs; + +public class JobSchedulerTests +{ + [Fact] + public async Task EnqueueDueOccurrencesAsync_WhenOccurrenceIsDue_CreatesSingleGlobalOccurrenceAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var processor = CreateProcessor(scheduler, store, "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(ScheduledProbeJob) + }, cancellationToken); + + var first = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + var second = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + + var dispatch = Assert.Single(first); + Assert.Empty(second); + Assert.Equal("nightly:20260101000000:global", dispatch.DispatchId); + Assert.Equal(ScheduledDispatchKind.JobOccurrence, dispatch.Kind); + Assert.Equal("nightly", dispatch.Headers["job.name"]); + + var state = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(state); + Assert.Equal(JobStatus.Scheduled, state.Status); + Assert.Equal(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), state.ScheduledForUtc); + } + + [Fact] + public async Task RunDueOccurrencesAsync_WhenOccurrenceIsDue_RunsConfiguredJobAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var probe = new JobSchedulerProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(ScheduledProbeJob) + }, cancellationToken); + var scheduled = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + + int completed = await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken); + + var dispatch = Assert.Single(scheduled); + var state = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.Equal(1, completed); + Assert.Equal(1, probe.RunCount); + Assert.NotNull(state); + Assert.Equal(JobStatus.Completed, state.Status); + Assert.Equal(1, state.Attempt); + Assert.Equal(100, state.Progress); + } + + [Fact] + public async Task EnqueueDueOccurrencesAsync_WithPerNodeScope_CreatesOccurrencePerNodeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var nodeA = CreateProcessor(scheduler, store, "node-a"); + var nodeB = CreateProcessor(scheduler, store, "node-b"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "per-node", + Cron = "* * * * *", + JobType = typeof(ScheduledProbeJob), + Scope = ScheduledJobScope.PerNode + }, cancellationToken); + + var first = await nodeA.EnqueueDueOccurrencesAsync(now, cancellationToken); + var second = await nodeB.EnqueueDueOccurrencesAsync(now, cancellationToken); + + Assert.Equal("per-node:20260101000000:node-a", Assert.Single(first).DispatchId); + Assert.Equal("per-node:20260101000000:node-b", Assert.Single(second).DispatchId); + + var states = await store.QueryAsync(new JobQuery { Name = "per-node" }, cancellationToken); + Assert.Equal(2, states.Count); + } + + [Fact] + public async Task EnqueueDueOccurrencesAsync_WithMisfireWindow_CatchesRecentMissedOccurrenceAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var processor = CreateProcessor(scheduler, store, "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 5, 0, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "daily", + Cron = "0 0 * * *", + JobType = typeof(ScheduledProbeJob), + MisfireWindow = TimeSpan.FromMinutes(10) + }, cancellationToken); + + var scheduled = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + + var dispatch = Assert.Single(scheduled); + var state = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(state); + Assert.Equal(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), state.ScheduledForUtc); + } + + [Fact] + public async Task RunDueOccurrencesAsync_WhenDispatchIsNotJobOccurrence_ReleasesItAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var processor = CreateProcessor(scheduler, store, "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = "delayed-message", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = "work", + Body = "hello"u8.ToArray(), + DueUtc = now + }, cancellationToken); + + int completed = await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken); + + Assert.Equal(0, completed); + var claimed = await store.ClaimDueDispatchesAsync(now, 10, "node-b", TimeSpan.FromMinutes(1), cancellationToken); + var dispatch = Assert.Single(claimed); + Assert.Equal("delayed-message", dispatch.DispatchId); + Assert.Equal("node-b", dispatch.ClaimOwner); + } + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId) + { + var serviceProvider = new ServiceCollection() + .AddSingleton(new JobSchedulerProbe()) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: nodeId); + return new JobScheduleProcessor(scheduler, store, client, nodeId: nodeId); + } + + private sealed class JobSchedulerProbe + { + private int _runCount; + + public int RunCount => Volatile.Read(ref _runCount); + + public void RecordRun() + { + Interlocked.Increment(ref _runCount); + } + } + + private sealed class ScheduledProbeJob : IJob + { + private readonly JobSchedulerProbe _probe; + + public ScheduledProbeJob(JobSchedulerProbe probe) + { + _probe = probe; + } + + public Task RunAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _probe.RecordRun(); + return Task.FromResult(JobResult.Success); + } + } +} From 6ef25db152e0d0aec0f1538ad971c5caa0b5ecae Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 25 Jun 2026 17:20:13 -0500 Subject: [PATCH 06/57] refactor: use channels in in-memory transport --- .../Messaging/InMemoryMessageTransport.cs | 75 +++++++++++++++---- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 600ab61fa..97b40adc9 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -2,9 +2,9 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; -using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Threading.Channels; using Foundatio.AsyncEx; using Foundatio.Queues; using Foundatio.Utility; @@ -107,7 +107,8 @@ private async Task> ReceiveAsync(string source, Re try { - await state.AvailableSignal.WaitAsync(waitCancellationTokenSource.Token).AnyContext(); + if (!await state.WaitToReadAsync(waitCancellationTokenSource.Token).ConfigureAwait(false)) + break; } catch (OperationCanceledException) when (!ct.IsCancellationRequested && !_disposeCancellationTokenSource.IsCancellationRequested) { @@ -258,7 +259,8 @@ public Task DeleteAsync(string name, CancellationToken ct) ArgumentException.ThrowIfNullOrEmpty(name); _roles.TryRemove(name, out _); - _destinations.TryRemove(name, out _); + if (_destinations.TryRemove(name, out var removed)) + removed.Complete(); _topicSubscriptions.TryRemove(name, out _); foreach (var subscriptions in _topicSubscriptions.Values) @@ -523,50 +525,93 @@ private sealed record InMemoryReceipt(string Destination, string LockToken); private sealed class DestinationState { - private readonly ConcurrentQueue[] _queues = + private readonly Channel[] _channels = [ - new ConcurrentQueue(), - new ConcurrentQueue(), - new ConcurrentQueue() + Channel.CreateUnbounded(CreateChannelOptions()), + Channel.CreateUnbounded(CreateChannelOptions()), + Channel.CreateUnbounded(CreateChannelOptions()) ]; - private readonly ConcurrentQueue _deadletterQueue = new(); + private readonly Channel _deadletterChannel = Channel.CreateUnbounded(CreateChannelOptions()); + private readonly SemaphoreSlim _availableMessages = new(0); + private long _queuedCount; + private long _deadletterCount; + private int _isCompleted; public ConcurrentDictionary InFlight { get; } = new(StringComparer.Ordinal); - public AsyncAutoResetEvent AvailableSignal { get; } = new(); public long Enqueued; public long Dequeued; public long Completed; public long Abandoned; public long Deadlettered; - public long QueuedCount => _queues.Sum(q => q.Count); - public long DeadletterCount => _deadletterQueue.Count; + public long QueuedCount => Volatile.Read(ref _queuedCount); + public long DeadletterCount => Volatile.Read(ref _deadletterCount); public void Enqueue(StoredMessage message) { - _queues[(int)message.Priority].Enqueue(message); + if (!_channels[(int)message.Priority].Writer.TryWrite(message)) + throw new InvalidOperationException("The destination is no longer accepting messages."); + + Interlocked.Increment(ref _queuedCount); Interlocked.Increment(ref Enqueued); - AvailableSignal.Set(); + _availableMessages.Release(); } public bool TryDequeue(out StoredMessage message) { for (int index = (int)MessagePriority.High; index >= (int)MessagePriority.Low; index--) { - if (_queues[index].TryDequeue(out message!)) + if (_channels[index].Reader.TryRead(out message!)) + { + _availableMessages.Wait(0); + Interlocked.Decrement(ref _queuedCount); return true; + } } message = null!; return false; } + public async ValueTask WaitToReadAsync(CancellationToken cancellationToken) + { + if (QueuedCount > 0) + return true; + + await _availableMessages.WaitAsync(cancellationToken).ConfigureAwait(false); + return true; + } + public void Deadletter(StoredMessage message) { - _deadletterQueue.Enqueue(message); + if (!_deadletterChannel.Writer.TryWrite(message)) + throw new InvalidOperationException("The destination is no longer accepting dead-letter messages."); + + Interlocked.Increment(ref _deadletterCount); Interlocked.Increment(ref Deadlettered); } + + public void Complete() + { + if (Interlocked.Exchange(ref _isCompleted, 1) == 1) + return; + + foreach (var channel in _channels) + channel.Writer.TryComplete(); + + _deadletterChannel.Writer.TryComplete(); + } + + private static UnboundedChannelOptions CreateChannelOptions() + { + return new UnboundedChannelOptions + { + AllowSynchronousContinuations = false, + SingleReader = false, + SingleWriter = false + }; + } } private sealed class PushSubscription : IPushSubscription From 9effb46891a1a790833abed3013cb09f9227164c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 27 Jun 2026 16:53:57 -0500 Subject: [PATCH 07/57] fix: route delays through job runtime --- src/Foundatio/Jobs/JobRuntime.cs | 6 +- src/Foundatio/Jobs/JobScheduler.cs | 91 ++++++++- .../Messaging/InMemoryMessageTransport.cs | 50 +---- src/Foundatio/Messaging/PubSub.cs | 90 +++++++-- src/Foundatio/Queues/MessageQueue.cs | 189 +++++++++++++++--- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 128 +++++++++++- .../Foundatio.Tests/Messaging/PubSubTests.cs | 20 +- .../Queue/MessageQueueTests.cs | 100 ++++++++- 8 files changed, 578 insertions(+), 96 deletions(-) diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index bf534df6b..22096eb14 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -54,7 +54,9 @@ public sealed record JobStatePatch public string? Error { get; init; } public int AttemptDelta { get; init; } public string? NodeId { get; init; } + public bool ClearNodeId { get; init; } public DateTimeOffset? LeaseExpiresUtc { get; init; } + public bool ClearLeaseExpiresUtc { get; init; } public DateTimeOffset? LastUpdatedUtc { get; init; } public DateTimeOffset? StartedUtc { get; init; } public DateTimeOffset? CompletedUtc { get; init; } @@ -382,8 +384,8 @@ private JobState ApplyPatch(JobState state, JobStatePatch? patch) ProgressMessage = patch.ProgressMessage ?? state.ProgressMessage, Error = patch.Error ?? state.Error, Attempt = state.Attempt + patch.AttemptDelta, - NodeId = patch.NodeId ?? state.NodeId, - LeaseExpiresUtc = patch.LeaseExpiresUtc ?? state.LeaseExpiresUtc, + NodeId = patch.ClearNodeId ? null : patch.NodeId ?? state.NodeId, + LeaseExpiresUtc = patch.ClearLeaseExpiresUtc ? null : patch.LeaseExpiresUtc ?? state.LeaseExpiresUtc, LastUpdatedUtc = patch.LastUpdatedUtc ?? state.LastUpdatedUtc, StartedUtc = patch.StartedUtc ?? state.StartedUtc, CompletedUtc = patch.CompletedUtc ?? state.CompletedUtc, diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 135d148ce..506517303 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -87,8 +87,9 @@ public sealed class JobScheduleProcessor private readonly IJobClient _jobClient; private readonly TimeProvider _timeProvider; private readonly string _nodeId; + private readonly IMessageTransport? _transport; - public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobClient jobClient, TimeProvider? timeProvider = null, string? nodeId = null) + public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobClient jobClient, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null) { _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _store = store ?? throw new ArgumentNullException(nameof(store)); @@ -97,6 +98,7 @@ public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJo _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + _transport = transport; } public Task> EnqueueDueOccurrencesAsync(CancellationToken cancellationToken = default) @@ -175,9 +177,16 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = foreach (var dispatch in dispatches) { + if (dispatch.Kind is ScheduledDispatchKind.QueueMessage or ScheduledDispatchKind.PubSubMessage) + { + await MaterializeMessageDispatchAsync(dispatch, cancellationToken).ConfigureAwait(false); + completed++; + continue; + } + if (dispatch.Kind != ScheduledDispatchKind.JobOccurrence) { - await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, dispatch.DueUtc, cancellationToken).ConfigureAwait(false); + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); continue; } @@ -191,7 +200,12 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = try { - await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false); + if (!await TryPrepareOccurrenceForRunAsync(jobId, definition, utcNow, cancellationToken).ConfigureAwait(false)) + { + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + continue; + } + await _jobClient.RunAsync(definition.JobType, new RunJobOptions { JobId = jobId, @@ -199,6 +213,29 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = NodeId = _nodeId }, cancellationToken).ConfigureAwait(false); + var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (state?.Status == JobStatus.Failed) + { + if (state.Attempt <= definition.MaxRetries) + { + await _store.TryTransitionAsync(jobId, JobStatus.Failed, JobStatus.Scheduled, new JobStatePatch + { + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + LastUpdatedUtc = utcNow + }, cancellationToken).ConfigureAwait(false); + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + continue; + } + + await _store.TryTransitionAsync(jobId, JobStatus.Failed, JobStatus.DeadLettered, new JobStatePatch + { + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + LastUpdatedUtc = utcNow + }, cancellationToken).ConfigureAwait(false); + } + await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); completed++; } @@ -212,6 +249,54 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = return completed; } + private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken) + { + if (_transport is null) + throw new InvalidOperationException("A message transport is required to materialize scheduled queue and pub/sub dispatches."); + + var result = await _transport.SendAsync(dispatch.Destination, [ + new TransportMessage + { + MessageId = dispatch.DispatchId, + Body = dispatch.Body, + Headers = dispatch.Headers + } + ], dispatch.Options with { DeliverAt = null }, cancellationToken).ConfigureAwait(false); + + if (!result.AllSucceeded) + throw new MessageBusException($"Unable to materialize scheduled dispatch \"{dispatch.DispatchId}\" to \"{dispatch.Destination}\"."); + + await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); + } + + private async Task TryPrepareOccurrenceForRunAsync(string jobId, ScheduledJobDefinition definition, DateTimeOffset utcNow, CancellationToken cancellationToken) + { + if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) + return true; + + var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (state?.Status != JobStatus.Processing || state.LeaseExpiresUtc is null || state.LeaseExpiresUtc > utcNow) + return false; + + if (state.Attempt > definition.MaxRetries) + { + await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.DeadLettered, new JobStatePatch + { + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + LastUpdatedUtc = utcNow + }, cancellationToken).ConfigureAwait(false); + return false; + } + + return await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Queued, new JobStatePatch + { + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + LastUpdatedUtc = utcNow + }, cancellationToken).ConfigureAwait(false); + } + private async Task HasActiveOccurrenceAsync(string name, string scopeKey, CancellationToken cancellationToken) { var states = await _store.QueryAsync(new JobQuery { Name = name, Limit = 1000 }, cancellationToken).ConfigureAwait(false); diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 97b40adc9..a89723aa2 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -11,7 +11,7 @@ namespace Foundatio.Messaging; -public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsRedeliveryDelay, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsDelayedDelivery, ISupportsExpiration, ISupportsProvisioning, ITransportInfo +public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo { private static readonly IReadOnlySet _supportedRoles = new HashSet { @@ -46,13 +46,16 @@ public Task SendAsync(string destination, IReadOnlyList _timeProvider.GetUtcNow()) + throw new NotSupportedException($"Transport \"{GetType().Name}\" does not support native delayed delivery. Use the runtime-store scheduled dispatch fallback."); + var results = new SendItemResult[messages.Count]; for (int index = 0; index < messages.Count; index++) { var message = messages[index]; string messageId = message.MessageId ?? options.DeduplicationId ?? Guid.NewGuid().ToString("N"); var stored = CreateStoredMessage(destination, messageId, message, options); - EnqueueOrSchedule(destination, stored, options); + EnqueueForDestination(destination, stored); results[index] = new SendItemResult { @@ -136,11 +139,6 @@ public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) } public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) - { - return AbandonAsync(entry, TimeSpan.Zero, ct); - } - - public Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); @@ -154,11 +152,7 @@ public Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, Cancell Interlocked.Increment(ref state.Abandoned); var redelivered = inFlight.Message with { DeliveryCount = entry.DeliveryCount + 1 }; - - if (redeliveryDelay > TimeSpan.Zero) - _ = Run.DelayedAsync(redeliveryDelay, () => EnqueueStoredMessageAsync(receipt.Destination, redelivered), _timeProvider, _disposeCancellationTokenSource.Token); - else - EnqueueStoredMessage(receipt.Destination, redelivered); + EnqueueStoredMessage(receipt.Destination, redelivered); return Task.CompletedTask; } @@ -331,27 +325,6 @@ private async Task RunPushSubscriptionAsync(string source, Func TimeSpan.Zero) - { - _ = Run.DelayedAsync(delay, () => EnqueueForDestinationAsync(destination, message), _timeProvider, _disposeCancellationTokenSource.Token); - return; - } - } - - EnqueueForDestination(destination, message); - } - - private Task EnqueueForDestinationAsync(string destination, StoredMessage message) - { - EnqueueForDestination(destination, message); - return Task.CompletedTask; - } - private void EnqueueForDestination(string destination, StoredMessage message) { var role = _roles.GetOrAdd(destination, DestinationRole.Queue); @@ -372,12 +345,6 @@ private void EnqueueForDestination(string destination, StoredMessage message) EnqueueStoredMessage(destination, message); } - private Task EnqueueStoredMessageAsync(string destination, StoredMessage message) - { - EnqueueStoredMessage(destination, message); - return Task.CompletedTask; - } - private void EnqueueStoredMessage(string destination, StoredMessage message) { var role = _roles.GetOrAdd(destination, DestinationRole.Queue); @@ -439,6 +406,9 @@ private StoredMessage CreateStoredMessage(string destination, string messageId, var headers = message.Headers.ToBuilder() .SetIfMissing(KnownHeaders.Priority, options.Priority.ToString()) .Build(); + int deliveryCount = Int32.TryParse(headers.GetValueOrDefault(KnownHeaders.Attempts), NumberStyles.Integer, CultureInfo.InvariantCulture, out int attempts) && attempts > 0 + ? attempts + : 1; return new StoredMessage( messageId, @@ -446,7 +416,7 @@ private StoredMessage CreateStoredMessage(string destination, string messageId, message.Body.ToArray(), headers, NormalizePriority(options.Priority), - DeliveryCount: 1, + DeliveryCount: deliveryCount, EnqueuedUtc: _timeProvider.GetUtcNow()); } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index ce1b6ee43..6e52f9cd1 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Serializer; using Foundatio.Utility; @@ -39,6 +40,8 @@ public sealed record PubSubOptions public Func? TopicResolver { get; init; } public Func? MessageTypeResolver { get; init; } public Func? SubscriptionResolver { get; init; } + public IJobRuntimeStore? RuntimeStore { get; init; } + public TimeProvider TimeProvider { get; init; } = TimeProvider.System; } public interface IPubSub : IAsyncDisposable @@ -69,7 +72,14 @@ public async Task PublishAsync(T message, PublishOptions? options = null, Can string topic = GetTopic(typeof(T), options.Topic); await EnsureTopicAsync(topic, cancellationToken).AnyContext(); - var result = await _transport.SendAsync(topic, [CreateTransportMessage(message, options)], CreateSendOptions(options), cancellationToken).AnyContext(); + var sendOptions = CreateSendOptions(options); + string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + var transportMessage = CreateTransportMessage(message, options, messageId); + + if (await TryScheduleDispatchAsync(topic, transportMessage, sendOptions, cancellationToken).AnyContext()) + return; + + var result = await _transport.SendAsync(topic, [transportMessage], sendOptions, cancellationToken).AnyContext(); var item = result.Items.Count > 0 ? result.Items[0] : null; if (item is null || !item.Success) throw new MessageBusException($"Unable to publish message to \"{topic}\": {item?.ErrorCode ?? "unknown error"}"); @@ -84,16 +94,23 @@ public async Task PublishBatchAsync(IEnumerable messages, PublishOptions? string topic = GetTopic(typeof(T), options.Topic); await EnsureTopicAsync(topic, cancellationToken).AnyContext(); + var sendOptions = CreateSendOptions(options); + int index = 0; var transportMessages = messages.Select(message => { ArgumentNullException.ThrowIfNull(message); - return CreateTransportMessage(message, options); + string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; + index++; + return CreateTransportMessage(message, options, messageId); }).ToArray(); if (transportMessages.Length == 0) return; - var result = await _transport.SendAsync(topic, transportMessages, CreateSendOptions(options), cancellationToken).AnyContext(); + if (await TryScheduleDispatchesAsync(topic, transportMessages, sendOptions, cancellationToken).AnyContext()) + return; + + var result = await _transport.SendAsync(topic, transportMessages, sendOptions, cancellationToken).AnyContext(); if (!result.AllSucceeded) throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{topic}\"."); } @@ -172,7 +189,7 @@ private async Task> CreateReceivedMessageAsync(TransportE if (message is null) throw new MessageBusException($"Message \"{entry.Id}\" deserialized to null."); - return new ReceivedMessage(_transport, entry, message, cancellationToken); + return new ReceivedMessage(_transport, entry, message, cancellationToken, _options.RuntimeStore, _options.TimeProvider); } catch (Exception ex) when (ex is not MessageBusException) { @@ -202,7 +219,7 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func(T message, PublishOptions options) where T : class + private TransportMessage CreateTransportMessage(T message, PublishOptions options, string? messageId = null) where T : class { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) @@ -222,25 +239,75 @@ private TransportMessage CreateTransportMessage(T message, PublishOptions opt } if (options.TimeToLive is { } ttl) - headers.Set(KnownHeaders.Expiration, DateTimeOffset.UtcNow.Add(ttl).ToString("O", CultureInfo.InvariantCulture)); + headers.Set(KnownHeaders.Expiration, _options.TimeProvider.GetUtcNow().Add(ttl).ToString("O", CultureInfo.InvariantCulture)); return new TransportMessage { Body = _options.Serializer.SerializeToBytes(message), - Headers = headers.Build() + Headers = headers.Build(), + MessageId = messageId }; } - private static TransportSendOptions CreateSendOptions(PublishOptions options) + private TransportSendOptions CreateSendOptions(PublishOptions options) { return new TransportSendOptions { Priority = options.Priority, - DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? DateTimeOffset.UtcNow.Add(delay) : null), + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _options.TimeProvider.GetUtcNow().Add(delay) : null), DeduplicationId = options.DeduplicationId }; } + private async Task TryScheduleDispatchesAsync(string topic, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + { + if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) + return false; + + for (int index = 0; index < messages.Count; index++) + { + var message = messages[index]; + string messageId = message.MessageId ?? Guid.NewGuid().ToString("N"); + await ScheduleDispatchAsync(topic, message with { MessageId = messageId }, options, dueUtc, cancellationToken).AnyContext(); + } + + return true; + } + + private Task TryScheduleDispatchAsync(string topic, TransportMessage message, TransportSendOptions options, CancellationToken cancellationToken) + { + return TryScheduleDispatchesAsync(topic, [message], options, cancellationToken); + } + + private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) + { + dueUtc = options.DeliverAt.GetValueOrDefault(); + if (options.DeliverAt is null || dueUtc <= _options.TimeProvider.GetUtcNow()) + return false; + + if (_transport is ISupportsDelayedDelivery) + return false; + + if (_options.RuntimeStore is null) + throw new MessageBusException($"Delayed publish requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(PubSubOptions)}.{nameof(PubSubOptions.RuntimeStore)}."); + + return true; + } + + private Task ScheduleDispatchAsync(string topic, TransportMessage message, TransportSendOptions options, DateTimeOffset dueUtc, CancellationToken cancellationToken) + { + return _options.RuntimeStore!.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = message.MessageId!, + Kind = ScheduledDispatchKind.PubSubMessage, + Destination = topic, + Body = message.Body, + Headers = message.Headers, + Options = options with { DeliverAt = null }, + DueUtc = dueUtc + }, cancellationToken); + } + private string GetTopic(Type messageType, string? topic) { return !String.IsNullOrEmpty(topic) @@ -262,10 +329,7 @@ private string GetMessageType(Type messageType) private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) { - if (_transport is ISupportsDeadLetter deadLetter) - await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); - else - await _transport.AbandonAsync(entry, cancellationToken).AnyContext(); + await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken).AnyContext(); } private void ThrowIfDisposed() diff --git a/src/Foundatio/Queues/MessageQueue.cs b/src/Foundatio/Queues/MessageQueue.cs index 161dd6265..ac6bddbb9 100644 --- a/src/Foundatio/Queues/MessageQueue.cs +++ b/src/Foundatio/Queues/MessageQueue.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Foundatio.Jobs; using Foundatio.Messaging; using Foundatio.Serializer; using Foundatio.Utility; @@ -50,6 +51,8 @@ public sealed record MessageQueueOptions public string ContentType { get; init; } = "application/json"; public Func? DestinationResolver { get; init; } public Func? MessageTypeResolver { get; init; } + public IJobRuntimeStore? RuntimeStore { get; init; } + public TimeProvider TimeProvider { get; init; } = TimeProvider.System; } public interface IMessageQueue : IAsyncDisposable @@ -97,13 +100,20 @@ public async Task EnqueueAsync(T message, EnqueueOptions? options = n options ??= new EnqueueOptions(); string destination = GetDestination(typeof(T), options.Destination); - var result = await _transport.SendAsync(destination, [CreateTransportMessage(message, options)], CreateSendOptions(options), cancellationToken).AnyContext(); + var sendOptions = CreateSendOptions(options); + string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + var transportMessage = CreateTransportMessage(message, options, messageId); + + if (await TryScheduleDispatchAsync(ScheduledDispatchKind.QueueMessage, destination, transportMessage, sendOptions, cancellationToken).AnyContext()) + return messageId; + + var result = await _transport.SendAsync(destination, [transportMessage], sendOptions, cancellationToken).AnyContext(); var item = result.Items.Count > 0 ? result.Items[0] : null; if (item is null || !item.Success) throw new QueueException($"Unable to enqueue message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}"); - return item.MessageId ?? throw new QueueException($"Transport did not return a message id for \"{destination}\"."); + return item.MessageId ?? messageId; } public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class @@ -113,16 +123,23 @@ public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options ??= new EnqueueOptions(); string destination = GetDestination(typeof(T), options.Destination); + var sendOptions = CreateSendOptions(options); + int index = 0; var transportMessages = messages.Select(message => { ArgumentNullException.ThrowIfNull(message); - return CreateTransportMessage(message, options); + string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; + index++; + return CreateTransportMessage(message, options, messageId); }).ToArray(); if (transportMessages.Length == 0) return; - var result = await _transport.SendAsync(destination, transportMessages, CreateSendOptions(options), cancellationToken).AnyContext(); + if (await TryScheduleDispatchesAsync(ScheduledDispatchKind.QueueMessage, destination, transportMessages, sendOptions, cancellationToken).AnyContext()) + return; + + var result = await _transport.SendAsync(destination, transportMessages, sendOptions, cancellationToken).AnyContext(); if (!result.AllSucceeded) throw new QueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{destination}\"."); } @@ -203,7 +220,7 @@ private async Task> CreateReceivedMessageAsync(TransportE if (message is null) throw new QueueException($"Message \"{entry.Id}\" deserialized to null."); - return new ReceivedMessage(_transport, entry, message, ct); + return new ReceivedMessage(_transport, entry, message, ct, _options.RuntimeStore, _options.TimeProvider); } catch (Exception ex) when (ex is not QueueException) { @@ -229,11 +246,19 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func received) + await received.RejectAsync(retry, "handler-error", redeliveryDelay, ct).AnyContext(); + else + await message.RejectAsync(retry, "handler-error", ct).AnyContext(); + } } } - private TransportMessage CreateTransportMessage(T message, EnqueueOptions options) where T : class + private TransportMessage CreateTransportMessage(T message, EnqueueOptions options, string? messageId = null) where T : class { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) @@ -253,25 +278,75 @@ private TransportMessage CreateTransportMessage(T message, EnqueueOptions opt } if (options.TimeToLive is { } ttl) - headers.Set(KnownHeaders.Expiration, DateTimeOffset.UtcNow.Add(ttl).ToString("O", CultureInfo.InvariantCulture)); + headers.Set(KnownHeaders.Expiration, _options.TimeProvider.GetUtcNow().Add(ttl).ToString("O", CultureInfo.InvariantCulture)); return new TransportMessage { Body = _options.Serializer.SerializeToBytes(message), - Headers = headers.Build() + Headers = headers.Build(), + MessageId = messageId }; } - private static TransportSendOptions CreateSendOptions(EnqueueOptions options) + private TransportSendOptions CreateSendOptions(EnqueueOptions options) { return new TransportSendOptions { Priority = options.Priority, - DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? DateTimeOffset.UtcNow.Add(delay) : null), + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _options.TimeProvider.GetUtcNow().Add(delay) : null), DeduplicationId = options.DeduplicationId }; } + private async Task TryScheduleDispatchesAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + { + if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) + return false; + + for (int index = 0; index < messages.Count; index++) + { + var message = messages[index]; + string messageId = message.MessageId ?? Guid.NewGuid().ToString("N"); + await ScheduleDispatchAsync(kind, destination, message with { MessageId = messageId }, options, dueUtc, cancellationToken).AnyContext(); + } + + return true; + } + + private Task TryScheduleDispatchAsync(ScheduledDispatchKind kind, string destination, TransportMessage message, TransportSendOptions options, CancellationToken cancellationToken) + { + return TryScheduleDispatchesAsync(kind, destination, [message], options, cancellationToken); + } + + private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) + { + dueUtc = options.DeliverAt.GetValueOrDefault(); + if (options.DeliverAt is null || dueUtc <= _options.TimeProvider.GetUtcNow()) + return false; + + if (_transport is ISupportsDelayedDelivery) + return false; + + if (_options.RuntimeStore is null) + throw new QueueException($"Delayed queue delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(MessageQueueOptions)}.{nameof(MessageQueueOptions.RuntimeStore)}."); + + return true; + } + + private Task ScheduleDispatchAsync(ScheduledDispatchKind kind, string destination, TransportMessage message, TransportSendOptions options, DateTimeOffset dueUtc, CancellationToken cancellationToken) + { + return _options.RuntimeStore!.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = message.MessageId!, + Kind = kind, + Destination = destination, + Body = message.Body, + Headers = message.Headers, + Options = options with { DeliverAt = null }, + DueUtc = dueUtc + }, cancellationToken); + } + private string GetDestination(Type messageType, string? destination) { return !String.IsNullOrEmpty(destination) @@ -286,10 +361,7 @@ private string GetMessageType(Type messageType) private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken ct) { - if (_transport is ISupportsDeadLetter deadLetter) - await deadLetter.DeadLetterAsync(entry, reason, ct).AnyContext(); - else - await _transport.AbandonAsync(entry, ct).AnyContext(); + await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, ct).AnyContext(); } private void ThrowIfDisposed() @@ -328,12 +400,16 @@ internal sealed class ReceivedMessage : IReceivedMessage where T : class { private readonly IMessageTransport _transport; private readonly TransportEntry _entry; + private readonly IJobRuntimeStore? _runtimeStore; + private readonly TimeProvider _timeProvider; private int _isHandled; - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken) + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) { _transport = transport; _entry = entry; + _runtimeStore = runtimeStore; + _timeProvider = timeProvider ?? TimeProvider.System; Message = message; CancellationToken = cancellationToken; } @@ -358,7 +434,17 @@ public Task CompleteAsync(CancellationToken cancellationToken = default) public Task RejectAsync(bool retry = true, string? reason = null, CancellationToken cancellationToken = default) { - return retry ? AbandonAsync(cancellationToken) : DeadLetterAsync(reason, cancellationToken); + return RejectAsync(retry, reason, redeliveryDelay: null, cancellationToken); + } + + internal Task RejectAsync(bool retry, string? reason, TimeSpan? redeliveryDelay, CancellationToken cancellationToken = default) + { + if (!retry) + return DeadLetterAsync(reason, cancellationToken); + + return redeliveryDelay is { } delay && delay > TimeSpan.Zero + ? AbandonAsync(delay, cancellationToken) + : AbandonAsync(cancellationToken); } public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) @@ -366,22 +452,19 @@ public async Task DeadLetterAsync(string? reason = null, CancellationToken cance if (!TryMarkHandled()) return; - if (_transport is ISupportsDeadLetter deadLetter) - await deadLetter.DeadLetterAsync(_entry, reason, cancellationToken).AnyContext(); - else - await _transport.AbandonAsync(_entry, cancellationToken).AnyContext(); + await DeadLetterAsync(_transport, _entry, reason, cancellationToken).AnyContext(); } public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) { return _transport is ISupportsLockRenewal lockRenewal ? lockRenewal.RenewLockAsync(_entry, duration, cancellationToken) - : Task.CompletedTask; + : throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support lock renewal."); } public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) { - return Task.CompletedTask; + throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue messages."); } private Task AbandonAsync(CancellationToken ct) @@ -392,6 +475,66 @@ private Task AbandonAsync(CancellationToken ct) return _transport.AbandonAsync(_entry, ct); } + private async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken ct) + { + if (!TryMarkHandled()) + return; + + if (_transport is ISupportsRedeliveryDelay redelivery) + { + await redelivery.AbandonAsync(_entry, redeliveryDelay, ct).AnyContext(); + return; + } + + if (_runtimeStore is null) + throw new QueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or {nameof(MessageQueueOptions)}.{nameof(MessageQueueOptions.RuntimeStore)}."); + + int nextAttempt = _entry.DeliveryCount + 1; + var headers = _entry.Headers.ToBuilder() + .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) + .Build(); + + await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = $"{_entry.Id}:retry:{nextAttempt}", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = _entry.Destination, + Body = _entry.Body, + Headers = headers, + Options = new TransportSendOptions { Priority = Priority }, + DueUtc = _timeProvider.GetUtcNow().Add(redeliveryDelay) + }, ct).AnyContext(); + + await _transport.CompleteAsync(_entry, ct).AnyContext(); + } + + internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) + { + if (transport is ISupportsDeadLetter deadLetter) + { + await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + return; + } + + var headers = entry.Headers.ToBuilder(); + if (!String.IsNullOrEmpty(reason)) + headers.Set(KnownHeaders.DeadLetterReason, reason); + + var result = await transport.SendAsync($"{entry.Destination}-deadletter", [ + new TransportMessage + { + MessageId = $"{entry.Id}:deadletter", + Body = entry.Body, + Headers = headers.Build() + } + ], new TransportSendOptions(), cancellationToken).AnyContext(); + + if (!result.AllSucceeded) + throw new QueueException($"Unable to write message \"{entry.Id}\" to the managed dead-letter destination \"{entry.Destination}-deadletter\"."); + + await transport.CompleteAsync(entry, cancellationToken).AnyContext(); + } + private bool TryMarkHandled() { return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index b886f1114..d6c041819 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; +using Foundatio.Messaging; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -128,12 +129,13 @@ await scheduler.ScheduleAsync(new ScheduledJobDefinition } [Fact] - public async Task RunDueOccurrencesAsync_WhenDispatchIsNotJobOccurrence_ReleasesItAsync() + public async Task RunDueOccurrencesAsync_WhenDispatchIsQueueMessage_MaterializesItAsync() { var cancellationToken = TestContext.Current.CancellationToken; var scheduler = new InMemoryJobScheduler(); var store = new InMemoryJobRuntimeStore(); - var processor = CreateProcessor(scheduler, store, "node-a"); + await using var transport = new InMemoryMessageTransport(); + var processor = CreateProcessor(scheduler, store, "node-a", transport); var now = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); await store.ScheduleDispatchAsync(new ScheduledDispatchState @@ -147,19 +149,110 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState int completed = await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken); - Assert.Equal(0, completed); - var claimed = await store.ClaimDueDispatchesAsync(now, 10, "node-b", TimeSpan.FromMinutes(1), cancellationToken); - var dispatch = Assert.Single(claimed); - Assert.Equal("delayed-message", dispatch.DispatchId); - Assert.Equal("node-b", dispatch.ClaimOwner); + Assert.Equal(1, completed); + var pull = Assert.IsAssignableFrom(transport); + var entries = await pull.ReceiveAsync("work", new ReceiveRequest { MaxMessages = 1, MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var entry = Assert.Single(entries); + Assert.Equal("delayed-message", entry.Id); + Assert.Equal("hello"u8.ToArray(), entry.Body.ToArray()); } - private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId) + + [Fact] + public async Task RunDueOccurrencesAsync_WhenJobFails_RetriesThenDeadLettersAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var probe = new JobSchedulerProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(FailingScheduledJob), + MaxRetries = 1 + }, cancellationToken); + var scheduled = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + var dispatch = Assert.Single(scheduled); + + Assert.Equal(0, await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + var retried = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(retried); + Assert.Equal(JobStatus.Scheduled, retried.Status); + Assert.Equal(1, retried.Attempt); + + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now.AddMinutes(2), cancellationToken: cancellationToken)); + var deadlettered = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(deadlettered); + Assert.Equal(JobStatus.DeadLettered, deadlettered.Status); + Assert.Equal(2, deadlettered.Attempt); + Assert.Equal(2, probe.RunCount); + } + + [Fact] + public async Task RunDueOccurrencesAsync_WhenProcessingLeaseExpired_ReclaimsAndRunsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var probe = new JobSchedulerProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + const string jobId = "nightly:20260101000000:global"; + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(ScheduledProbeJob), + MaxRetries = 1 + }, cancellationToken); + await store.CreateIfAbsentAsync(new JobState + { + JobId = jobId, + Name = "nightly", + Status = JobStatus.Processing, + Attempt = 1, + NodeId = "node-b", + LeaseExpiresUtc = now.AddMinutes(-1), + ScheduledForUtc = now.AddSeconds(-30) + }, cancellationToken); + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = jobId, + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = "nightly", + Body = Array.Empty(), + DueUtc = now, + JobId = jobId + }, cancellationToken); + + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + + var state = await store.GetAsync(jobId, cancellationToken); + Assert.NotNull(state); + Assert.Equal(JobStatus.Completed, state.Status); + Assert.Equal(2, state.Attempt); + Assert.Equal(1, probe.RunCount); + } + + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId, IMessageTransport? transport = null) { var serviceProvider = new ServiceCollection() .AddSingleton(new JobSchedulerProbe()) .BuildServiceProvider(); var client = new JobClient(store, serviceProvider, nodeId: nodeId); - return new JobScheduleProcessor(scheduler, store, client, nodeId: nodeId); + return new JobScheduleProcessor(scheduler, store, client, nodeId: nodeId, transport: transport); } private sealed class JobSchedulerProbe @@ -174,6 +267,23 @@ public void RecordRun() } } + private sealed class FailingScheduledJob : IJob + { + private readonly JobSchedulerProbe _probe; + + public FailingScheduledJob(JobSchedulerProbe probe) + { + _probe = probe; + } + + public Task RunAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + _probe.RecordRun(); + return Task.FromResult(JobResult.FromException(new InvalidOperationException("failed"))); + } + } + private sealed class ScheduledProbeJob : IJob { private readonly JobSchedulerProbe _probe; diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index d4a5063da..d39e7a1f5 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -3,9 +3,11 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.AsyncEx; +using Foundatio.Jobs; using Foundatio.Messaging; using Foundatio.Queues; using Foundatio.Tests.Extensions; +using Microsoft.Extensions.DependencyInjection; using Xunit; namespace Foundatio.Tests.Messaging; @@ -121,10 +123,13 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() } [Fact] - public async Task PublishAsync_WithDelay_DelaysDeliveryAsync() + public async Task PublishAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var pubSub = new PubSub(new InMemoryMessageTransport()); + var store = new InMemoryJobRuntimeStore(); + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport, new PubSubOptions { RuntimeStore = store }); + var processor = CreateDispatchProcessor(store, transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(1); @@ -135,9 +140,10 @@ public async Task PublishAsync_WithDelay_DelaysDeliveryAsync() return Task.CompletedTask; }, new SubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); - await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PublishOptions { Delay = TimeSpan.FromMilliseconds(250) }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PublishOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); await Assert.ThrowsAsync(async () => await received.WaitAsync(TimeSpan.FromMilliseconds(50))); + Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); await received.WaitAsync(TimeSpan.FromSeconds(2)); await cts.CancelAsync(); @@ -178,6 +184,14 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() Assert.Equal(1, stats.Abandoned); } + + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) + { + var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + return new JobScheduleProcessor(new InMemoryJobScheduler(), store, client, nodeId: "node-a", transport: transport); + } + private sealed class PreviewEvent { public string? Data { get; set; } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 514167955..cc59597b3 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -3,9 +3,11 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.AsyncEx; +using Foundatio.Jobs; using Foundatio.Messaging; using Foundatio.Queues; using Foundatio.Tests.Extensions; +using Microsoft.Extensions.DependencyInjection; using Xunit; namespace Foundatio.Tests.Queue; @@ -91,6 +93,32 @@ public async Task RejectAsync_WithRetry_RedeliversAsync() await second.CompleteAsync(cancellationToken); } + [Fact] + public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(message); + + await Assert.ThrowsAsync(async () => await message.RenewLockAsync(cancellationToken: cancellationToken)); + } + + [Fact] + public async Task ReportProgressAsync_WhenUntracked_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "progress" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(message); + + await Assert.ThrowsAsync(async () => await message.ReportProgressAsync(50, "half", cancellationToken)); + } + [Fact] public async Task RejectAsync_WithoutRetry_DeadLettersAsync() { @@ -136,22 +164,80 @@ public async Task StartWorkingAsync_WithAutoAck_CompletesMessageAsync() } [Fact] - public async Task EnqueueAsync_WithDelay_DelaysVisibilityAsync() + public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + var store = new InMemoryJobRuntimeStore(); + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport, new MessageQueueOptions { RuntimeStore = store }); + var processor = CreateDispatchProcessor(store, transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMilliseconds(250) }, cancellationToken); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); var immediate = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); + Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); + var delayed = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); } + [Fact] + public async Task EnqueueAsync_WithDelayAndNoRuntimeStore_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await Assert.ThrowsAsync(async () => + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); + } + + [Fact] + public async Task StartWorkingAsync_WithRedeliveryBackoff_SchedulesRetryThroughRuntimeStoreAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport, new MessageQueueOptions { RuntimeStore = store }); + var processor = CreateDispatchProcessor(store, transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var firstAttempt = new AsyncCountdownEvent(1); + var secondAttempt = new AsyncCountdownEvent(1); + int attempts = 0; + + var worker = queue.StartWorkingAsync((message, _) => + { + attempts++; + if (attempts == 1) + { + Assert.Equal(1, message.Attempts); + firstAttempt.Signal(); + throw new InvalidOperationException("try again later"); + } + + Assert.Equal(2, message.Attempts); + Assert.Equal("retry", message.Message.Data); + secondAttempt.Signal(); + return Task.CompletedTask; + }, new WorkerOptions { RedeliveryBackoff = _ => TimeSpan.FromMinutes(1), MaxAttempts = 3 }, cts.Token); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cts.Token); + await firstAttempt.WaitAsync(TimeSpan.FromSeconds(2)); + + var immediate = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + Assert.Null(immediate); + + Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); + await secondAttempt.WaitAsync(TimeSpan.FromSeconds(2)); + + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await worker); + } + [Fact] public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync() { @@ -193,6 +279,14 @@ await Assert.ThrowsAsync(async () => Assert.Equal(0, stats.Working); } + + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) + { + var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + return new JobScheduleProcessor(new InMemoryJobScheduler(), store, client, nodeId: "node-a", transport: transport); + } + private sealed class PreviewWorkItem { public string? Data { get; set; } From 83629334a254c988ebf75d240c092b870a07a524 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 27 Jun 2026 17:13:41 -0500 Subject: [PATCH 08/57] feat: align messaging and job APIs --- .agents/skills/foundatio/SKILL.md | 10 + docs/guide/messaging-jobs-redesign.md | 94 ++++ .../MessageTransportConformanceTests.cs | 7 +- src/Foundatio/FoundatioServicesExtensions.cs | 104 +++++ src/Foundatio/Jobs/JobRuntime.cs | 198 +++++++-- src/Foundatio/Jobs/JobScheduler.cs | 17 +- .../Messaging/InMemoryMessageTransport.cs | 7 +- .../{Queues => Messaging}/MessageQueue.cs | 402 +++++++++++------- .../Messaging/MessageQueueException.cs | 17 + .../Messaging/MessageRouteAttribute.cs | 21 + src/Foundatio/Messaging/MessageTransport.cs | 16 +- src/Foundatio/Messaging/PubSub.cs | 239 ++++++++--- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 20 +- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 16 +- .../Foundatio.Tests/Messaging/PubSubTests.cs | 60 ++- .../Queue/MessageQueueTests.cs | 127 ++++-- 16 files changed, 995 insertions(+), 360 deletions(-) create mode 100644 docs/guide/messaging-jobs-redesign.md rename src/Foundatio/{Queues => Messaging}/MessageQueue.cs (59%) create mode 100644 src/Foundatio/Messaging/MessageQueueException.cs create mode 100644 src/Foundatio/Messaging/MessageRouteAttribute.cs diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index 9fdfe9107..2056da4f7 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -23,6 +23,16 @@ query-docs(libraryId="/foundatiofx/foundatio", query="How to configure queue ret Query with specific questions, not single keywords. All provider docs (Redis, Azure, AWS, Kafka, etc.) are included in the main library. + +## Messaging/Jobs Redesign Notes + +- New queue/pub-sub APIs live under `Foundatio.Messaging`: app-facing `IQueue`, `IPubSub`, shared `IReceivedMessage`, `QueueMessageOptions`, `QueueReceiveOptions`, `QueueConsumerOptions`, `PubSubMessageOptions`, and `PubSubSubscriptionOptions`. +- Route resolution is type-driven: operation override > resolver/registration > `MessageRouteAttribute` > kebab-case type-name convention. `Destination` and `Source` are advanced queue overrides; `Topic` and `Subscription` are advanced pub/sub overrides. +- Listener startup returns handles: `StartConsumerAsync` returns `IMessageConsumer`; `SubscribeAsync` returns `IMessageSubscription`. Use `RunConsumerAsync` or `RunSubscriptionAsync` only for blocking lifetime loops. +- Received-message settlement uses explicit verbs only: `CompleteAsync`, `AbandonAsync`, `DeadLetterAsync`, `RenewLockAsync`, and `ReportProgressAsync`. Unsupported capabilities should throw clearly instead of silently downgrading. +- New durable job runtime roles are separated: `IJobClient` submits and returns `JobHandle`, `IJobMonitor` queries state, `IJobRuntimeStore` persists runtime state, and `IJobWorker` claims and executes queued jobs. +- In-memory setup for the redesign is `services.AddFoundatio().Messaging.UseInMemory().Jobs.UseInMemoryRuntime()`. + ## Core Interfaces | Interface | Purpose | In-Memory | Production | diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md new file mode 100644 index 000000000..f4b403cf3 --- /dev/null +++ b/docs/guide/messaging-jobs-redesign.md @@ -0,0 +1,94 @@ +# Messaging and Jobs Redesign + +The new messaging API is app-facing and type-driven. Queue, pub/sub, received-message, headers, options, and transport abstractions live under `Foundatio.Messaging`; folders may separate queue, pub/sub, and transport code, but consumers should not need a separate queue namespace for the new API. + +## Setup + +Register the in-memory messaging transport and durable job runtime through DI: + +```csharp +services.AddFoundatio() + .Messaging.UseInMemory() + .Jobs.UseInMemoryRuntime(); +``` + +Application code should depend on `Foundatio.Messaging.IQueue`, `IPubSub`, `IJobClient`, `IJobMonitor`, and `IJobWorker` instead of constructing `InMemoryMessageTransport`, `MessageQueue`, `PubSub`, or `JobClient` directly. + +## Queue + +The default queue model is send or receive this message type: + +```csharp +await queue.EnqueueAsync(new OrderSubmitted(id)); + +IReceivedMessage? received = await queue.ReceiveAsync(); +``` + +Destination and source are advanced overrides: + +```csharp +await queue.EnqueueAsync(message, new QueueMessageOptions { + Destination = "orders-high-priority" +}); + +IReceivedMessage? received = await queue.ReceiveAsync(new QueueReceiveOptions { + Source = "orders-high-priority" +}); +``` + +Consumers return handles and do not block unexpectedly: + +```csharp +await using IMessageConsumer consumer = await queue.StartConsumerAsync(HandleAsync); +``` + +Use `RunConsumerAsync` when the desired behavior is a blocking lifetime loop. + +## Pub/Sub + +Pub/sub follows the same type-driven pattern: + +```csharp +await pubsub.PublishAsync(new OrderSubmitted(id)); + +await using IMessageSubscription subscription = await pubsub.SubscribeAsync( + HandleAsync, + new PubSubSubscriptionOptions { Subscription = "billing-service" }); +``` + +`PubSubMessageOptions` mirrors queue send options where concepts overlap: priority, delay, TTL, correlation id, deduplication id, headers, and topic override. + +## Routing + +Default route precedence is: + +```text +options override > resolver/registration > MessageRouteAttribute > kebab-case type-name convention +``` + +`QueueMessageOptions.Destination`, `QueueReceiveOptions.Source`, `PubSubMessageOptions.Topic`, and `PubSubSubscriptionOptions.Topic`/`Subscription` are explicit operation overrides. `QueueOptions.DestinationResolver`, `PubSubOptions.TopicResolver`, and `PubSubOptions.SubscriptionResolver` are the registration/resolver layer. `MessageRouteAttribute` is the type-local fallback before the final convention. + +## Delivery Settlement + +Received messages use explicit settlement verbs for both queue and pub/sub: + +```csharp +await message.CompleteAsync(); +await message.AbandonAsync(); +await message.DeadLetterAsync("validation"); +await message.RenewLockAsync(); +await message.ReportProgressAsync(50, "half"); +``` + +Unsupported capabilities fail clearly with `NotSupportedException` or a validation exception. There are no silent no-ops for dead-lettering, lock renewal, progress, priority, expiration, or delayed delivery. + +## Jobs + +`IJobClient` submits durable work and returns a `JobHandle`; it does not execute jobs synchronously: + +```csharp +JobHandle handle = await jobs.EnqueueAsync(); +JobState? state = await handle.GetStateAsync(); +``` + +Execution belongs to `IJobWorker`, which claims queued jobs from `IJobRuntimeStore`. State and operational queries belong to `IJobMonitor`. Scheduled occurrences are created by `IJobScheduler` and materialized by `JobScheduleProcessor` through the runtime store. diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 933937762..7726b4844 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -4,7 +4,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Messaging; -using Foundatio.Queues; using Foundatio.Xunit; using Xunit; @@ -60,7 +59,7 @@ public virtual async Task CanSendAndReceiveBatchAsync() if (transport is ISupportsStats stats) { - QueueStats queueStats = await stats.GetStatsAsync("orders", TestCancellationToken); + MessageDestinationStats queueStats = await stats.GetStatsAsync("orders", TestCancellationToken); Assert.Equal(0, queueStats.Queued); Assert.Equal(0, queueStats.Working); Assert.Equal(2, queueStats.Completed); @@ -258,7 +257,7 @@ public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() var entry = Assert.Single(await pull.ReceiveAsync("deadletter", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); await ((ISupportsDeadLetter)transport).DeadLetterAsync(entry, "bad-payload", TestCancellationToken); - QueueStats queueStats = await stats.GetStatsAsync("deadletter", TestCancellationToken); + MessageDestinationStats queueStats = await stats.GetStatsAsync("deadletter", TestCancellationToken); Assert.Equal(0, queueStats.Working); Assert.Equal(1, queueStats.Deadletter); } @@ -290,7 +289,7 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy var entries = await pull.ReceiveAsync("expiration", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); Assert.Empty(entries); - QueueStats queueStats = await stats.GetStatsAsync("expiration", TestCancellationToken); + MessageDestinationStats queueStats = await stats.GetStatsAsync("expiration", TestCancellationToken); Assert.Equal(0, queueStats.Queued); Assert.Equal(1, queueStats.Deadletter); } diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 07b184b78..35fd2f7b7 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -1,6 +1,7 @@ using System; using Foundatio.Caching; using Foundatio.Extensions; +using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; using Foundatio.Queues; @@ -36,6 +37,7 @@ internal FoundatioBuilder(IServiceCollection services) Storage = new StorageBuilder(this); Messaging = new MessagingBuilder(this); Queueing = new QueueingBuilder(this); + Jobs = new JobsBuilder(this); Locking = new LockingBuilder(this); } @@ -62,6 +64,11 @@ internal FoundatioBuilder(IServiceCollection services) /// public QueueingBuilder Queueing { get; } + /// + /// Configure background job runtime services for Foundatio. + /// + public JobsBuilder Jobs { get; } + /// /// Configure locking services for Foundatio. /// @@ -268,6 +275,7 @@ public FoundatioBuilder UseInMemory(InMemoryMessageBusOptions? options = null) _services.ReplaceSingleton(sp => new InMemoryMessageBus(options.UseServices(sp))); _services.ReplaceSingleton(sp => sp.GetRequiredService()); _services.ReplaceSingleton(sp => sp.GetRequiredService()); + RegisterMessagingRuntime(sp => new InMemoryMessageTransport(sp.GetService())); return _builder; } @@ -276,8 +284,104 @@ public FoundatioBuilder UseInMemory(Builder(sp => new InMemoryMessageBus(b => b.Configure(config).UseServices(sp))); _services.ReplaceSingleton(sp => sp.GetRequiredService()); _services.ReplaceSingleton(sp => sp.GetRequiredService()); + RegisterMessagingRuntime(sp => new InMemoryMessageTransport(sp.GetService())); + return _builder; + } + + public FoundatioBuilder UseTransport(IMessageTransport transport) + { + _services.ReplaceSingleton(_ => transport); + RegisterMessageClients(); + return _builder; + } + + public FoundatioBuilder UseTransport(Func factory) + { + RegisterMessagingRuntime(factory); + return _builder; + } + + private void RegisterMessagingRuntime(Func factory) + { + _services.ReplaceSingleton(factory); + RegisterMessageClients(); + } + + private void RegisterMessageClients() + { + _services.ReplaceSingleton(sp => new MessageQueue(sp.GetRequiredService(), CreateQueueOptions(sp))); + _services.ReplaceSingleton(sp => new PubSub(sp.GetRequiredService(), CreatePubSubOptions(sp))); + } + + private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) + { + return new QueueOptions + { + Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, + RuntimeStore = serviceProvider.GetService(), + TimeProvider = serviceProvider.GetService() ?? TimeProvider.System + }; + } + + private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvider) + { + return new PubSubOptions + { + Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, + RuntimeStore = serviceProvider.GetService(), + TimeProvider = serviceProvider.GetService() ?? TimeProvider.System + }; + } + } + + public class JobsBuilder : IFoundatioBuilder + { + private readonly FoundatioBuilder _builder; + private readonly IServiceCollection _services; + + internal JobsBuilder(IFoundatioBuilder builder) + { + _builder = builder.Builder; + _services = builder.Services; + } + + IServiceCollection IFoundatioBuilder.Services => _services; + FoundatioBuilder IFoundatioBuilder.Builder => _builder; + + public FoundatioBuilder UseRuntimeStore(IJobRuntimeStore store) + { + _services.ReplaceSingleton(_ => store); + RegisterJobServices(); return _builder; } + + public FoundatioBuilder UseRuntimeStore(Func factory) + { + _services.ReplaceSingleton(factory); + RegisterJobServices(); + return _builder; + } + + public FoundatioBuilder UseInMemoryRuntime() + { + _services.ReplaceSingleton(sp => new InMemoryJobRuntimeStore(sp.GetService())); + RegisterJobServices(); + return _builder; + } + + private void RegisterJobServices() + { + _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService())); + _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService())); + _services.ReplaceSingleton(); + _services.ReplaceSingleton(sp => new JobScheduleProcessor( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetService(), + transport: sp.GetService())); + } } public class QueueingBuilder : IFoundatioBuilder diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 22096eb14..db3cb3b6a 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -31,6 +31,7 @@ public sealed record JobState { public required string JobId { get; init; } public required string Name { get; init; } + public string? JobType { get; init; } public JobStatus Status { get; init; } = JobStatus.Queued; public int? Progress { get; init; } public string? ProgressMessage { get; init; } @@ -49,6 +50,7 @@ public sealed record JobState public sealed record JobStatePatch { public JobStatus? Status { get; init; } + public string? JobType { get; init; } public int? Progress { get; init; } public string? ProgressMessage { get; init; } public string? Error { get; init; } @@ -85,11 +87,35 @@ public sealed record ScheduledDispatchState public string? JobId { get; init; } } -public sealed record RunJobOptions +public sealed record JobRequestOptions { public string? JobId { get; init; } public string? Name { get; init; } - public string? NodeId { get; init; } +} + +public sealed class JobHandle +{ + private readonly IJobMonitor _monitor; + private readonly Func> _requestCancellation; + + internal JobHandle(string jobId, IJobMonitor monitor, Func> requestCancellation) + { + JobId = jobId; + _monitor = monitor; + _requestCancellation = requestCancellation; + } + + public string JobId { get; } + + public Task GetStateAsync(CancellationToken cancellationToken = default) + { + return _monitor.GetAsync(JobId, cancellationToken); + } + + public Task RequestCancellationAsync(CancellationToken cancellationToken = default) + { + return _requestCancellation(JobId, cancellationToken); + } } public interface IJobMonitor @@ -98,13 +124,19 @@ public interface IJobMonitor Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default); } -public interface IJobClient : IJobMonitor +public interface IJobClient { - Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob; - Task RunAsync(Type jobType, RunJobOptions? options = null, CancellationToken cancellationToken = default); + Task EnqueueAsync(JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob; + Task EnqueueAsync(Type jobType, JobRequestOptions? options = null, CancellationToken cancellationToken = default); Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); } +public interface IJobWorker +{ + Task RunAsync(string jobId, CancellationToken cancellationToken = default); + Task RunQueuedAsync(int limit = 100, CancellationToken cancellationToken = default); +} + public interface IJobRuntimeStore : IJobMonitor { Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default); @@ -380,6 +412,7 @@ private JobState ApplyPatch(JobState state, JobStatePatch? patch) return state with { Status = patch.Status ?? state.Status, + JobType = patch.JobType ?? state.JobType, Progress = patch.Progress ?? state.Progress, ProgressMessage = patch.ProgressMessage ?? state.ProgressMessage, Error = patch.Error ?? state.Error, @@ -397,114 +430,189 @@ private JobState ApplyPatch(JobState state, JobStatePatch? patch) public sealed class JobClient : IJobClient { private readonly IJobRuntimeStore _store; - private readonly IServiceProvider _serviceProvider; private readonly TimeProvider _timeProvider; - private readonly string _nodeId; - public JobClient(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null) + public JobClient(IJobRuntimeStore store, TimeProvider? timeProvider = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); - _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); _timeProvider = timeProvider ?? TimeProvider.System; - _nodeId = !String.IsNullOrEmpty(nodeId) - ? nodeId - : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; - } - - public Task GetAsync(string jobId, CancellationToken cancellationToken = default) - { - return _store.GetAsync(jobId, cancellationToken); } - public Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default) + public Task EnqueueAsync(JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob { - return _store.QueryAsync(query, cancellationToken); + return EnqueueAsync(typeof(TJob), options, cancellationToken); } - public Task RunAsync(RunJobOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob - { - return RunAsync(typeof(TJob), options, cancellationToken); - } - - public async Task RunAsync(Type jobType, RunJobOptions? options = null, CancellationToken cancellationToken = default) + public async Task EnqueueAsync(Type jobType, JobRequestOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(jobType); if (!typeof(IJob).IsAssignableFrom(jobType)) throw new ArgumentException("Job type must implement IJob.", nameof(jobType)); - options ??= new RunJobOptions(); + options ??= new JobRequestOptions(); string jobId = options.JobId ?? Guid.NewGuid().ToString("N"); string name = options.Name ?? jobType.Name; - string nodeId = options.NodeId ?? _nodeId; var now = _timeProvider.GetUtcNow(); await _store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = name, + JobType = jobType.AssemblyQualifiedName, Status = JobStatus.Queued, CreatedUtc = now, LastUpdatedUtc = now }, cancellationToken).ConfigureAwait(false); - if (!await _store.TryTransitionAsync(jobId, JobStatus.Queued, JobStatus.Processing, new JobStatePatch + return new JobHandle(jobId, _store, RequestCancellationAsync); + } + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + { + return _store.RequestCancellationAsync(jobId, cancellationToken); + } +} + +public sealed class JobWorker : IJobWorker +{ + private static readonly TimeSpan DefaultLease = TimeSpan.FromMinutes(5); + + private readonly IJobRuntimeStore _store; + private readonly IServiceProvider _serviceProvider; + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly TimeSpan _lease; + + public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _timeProvider = timeProvider ?? TimeProvider.System; + _nodeId = !String.IsNullOrEmpty(nodeId) + ? nodeId + : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + _lease = lease ?? DefaultLease; + } + + public async Task RunQueuedAsync(int limit = 100, CancellationToken cancellationToken = default) + { + var queued = await _store.QueryAsync(new JobQuery + { + Status = JobStatus.Queued, + Limit = limit + }, cancellationToken).ConfigureAwait(false); + + int completed = 0; + foreach (var state in queued) { - NodeId = nodeId, + if (await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false)) + completed++; + } + + return completed; + } + + public async Task RunAsync(string jobId, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(jobId); + + var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + return state is not null && await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false); + } + + private async Task RunJobStateAsync(JobState state, CancellationToken cancellationToken) + { + if (state.Status != JobStatus.Queued) + return false; + + var now = _timeProvider.GetUtcNow(); + if (!await _store.TryTransitionAsync(state.JobId, JobStatus.Queued, JobStatus.Processing, new JobStatePatch + { + NodeId = _nodeId, StartedUtc = now, - LeaseExpiresUtc = now.AddMinutes(5), + LeaseExpiresUtc = now.Add(_lease), AttemptDelta = 1 }, cancellationToken).ConfigureAwait(false)) { - return jobId; + return false; } using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - using var cancellationWatcher = WatchCancellation(jobId, linkedCancellationTokenSource); - var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); + using var cancellationWatcher = WatchCancellation(state.JobId, linkedCancellationTokenSource); try { + var jobType = ResolveJobType(state); + var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); var result = await job.TryRunAsync(linkedCancellationTokenSource.Token).ConfigureAwait(false); var completedAt = _timeProvider.GetUtcNow(); + if (result.IsCancelled) { - await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Cancelled, new JobStatePatch + await _store.TryTransitionAsync(state.JobId, JobStatus.Processing, JobStatus.Cancelled, new JobStatePatch { Error = result.Message, CompletedUtc = completedAt, - LeaseExpiresUtc = null + ClearNodeId = true, + ClearLeaseExpiresUtc = true }, CancellationToken.None).ConfigureAwait(false); } else if (result.IsSuccess) { - await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Completed, new JobStatePatch + await _store.TryTransitionAsync(state.JobId, JobStatus.Processing, JobStatus.Completed, new JobStatePatch { CompletedUtc = completedAt, - LeaseExpiresUtc = null, + ClearNodeId = true, + ClearLeaseExpiresUtc = true, Progress = 100 }, CancellationToken.None).ConfigureAwait(false); } else { - await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Failed, new JobStatePatch + await _store.TryTransitionAsync(state.JobId, JobStatus.Processing, JobStatus.Failed, new JobStatePatch { Error = result.Message, CompletedUtc = completedAt, - LeaseExpiresUtc = null + ClearNodeId = true, + ClearLeaseExpiresUtc = true }, CancellationToken.None).ConfigureAwait(false); } + + return true; } - finally + catch (Exception ex) { - await _store.ReleaseClaimAsync(jobId, nodeId, CancellationToken.None).ConfigureAwait(false); + await _store.TryTransitionAsync(state.JobId, JobStatus.Processing, JobStatus.Failed, new JobStatePatch + { + Error = ex.Message, + CompletedUtc = _timeProvider.GetUtcNow(), + ClearNodeId = true, + ClearLeaseExpiresUtc = true + }, CancellationToken.None).ConfigureAwait(false); + throw; } - - return jobId; } - public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + private static Type ResolveJobType(JobState state) { - return _store.RequestCancellationAsync(jobId, cancellationToken); + if (String.IsNullOrEmpty(state.JobType)) + throw new InvalidOperationException($"Job \"{state.JobId}\" does not have a job type and cannot be executed by a worker."); + + var jobType = Type.GetType(state.JobType, throwOnError: false); + if (jobType is null) + { + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + jobType = assembly.GetType(state.JobType, throwOnError: false); + if (jobType is not null) + break; + } + } + + if (jobType is null || !typeof(IJob).IsAssignableFrom(jobType)) + throw new InvalidOperationException($"Job type \"{state.JobType}\" for job \"{state.JobId}\" could not be resolved to an IJob implementation."); + + return jobType; } private IDisposable WatchCancellation(string jobId, CancellationTokenSource cancellationTokenSource) diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 506517303..d0141fc3a 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -84,16 +84,16 @@ public sealed class JobScheduleProcessor private readonly IJobScheduler _scheduler; private readonly IJobRuntimeStore _store; - private readonly IJobClient _jobClient; + private readonly IJobWorker _jobWorker; private readonly TimeProvider _timeProvider; private readonly string _nodeId; private readonly IMessageTransport? _transport; - public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobClient jobClient, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null) + public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobWorker jobWorker, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null) { _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _store = store ?? throw new ArgumentNullException(nameof(store)); - _jobClient = jobClient ?? throw new ArgumentNullException(nameof(jobClient)); + _jobWorker = jobWorker ?? throw new ArgumentNullException(nameof(jobWorker)); _timeProvider = timeProvider ?? TimeProvider.System; _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId @@ -136,6 +136,7 @@ await _store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = definition.Name, + JobType = definition.JobType?.AssemblyQualifiedName, Status = JobStatus.Scheduled, CreatedUtc = utcNow, LastUpdatedUtc = utcNow, @@ -206,12 +207,7 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = continue; } - await _jobClient.RunAsync(definition.JobType, new RunJobOptions - { - JobId = jobId, - Name = definition.Name, - NodeId = _nodeId - }, cancellationToken).ConfigureAwait(false); + await _jobWorker.RunAsync(jobId, cancellationToken).ConfigureAwait(false); var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); if (state?.Status == JobStatus.Failed) @@ -271,7 +267,7 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat private async Task TryPrepareOccurrenceForRunAsync(string jobId, ScheduledJobDefinition definition, DateTimeOffset utcNow, CancellationToken cancellationToken) { - if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) + if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { JobType = definition.JobType?.AssemblyQualifiedName, LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) return true; var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); @@ -291,6 +287,7 @@ private async Task TryPrepareOccurrenceForRunAsync(string jobId, Scheduled return await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Queued, new JobStatePatch { + JobType = definition.JobType?.AssemblyQualifiedName, ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index a89723aa2..717bb0b2b 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using System.Threading.Channels; using Foundatio.AsyncEx; -using Foundatio.Queues; using Foundatio.Utility; namespace Foundatio.Messaging; @@ -186,16 +185,16 @@ public Task SubscribeAsync(string source, Func(subscription); } - public Task GetStatsAsync(string destination, CancellationToken ct) + public Task GetStatsAsync(string destination, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); ArgumentException.ThrowIfNullOrEmpty(destination); if (!_destinations.TryGetValue(destination, out var state)) - return Task.FromResult(new QueueStats()); + return Task.FromResult(new MessageDestinationStats()); - return Task.FromResult(new QueueStats + return Task.FromResult(new MessageDestinationStats { Queued = state.QueuedCount, Working = state.InFlight.Count, diff --git a/src/Foundatio/Queues/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs similarity index 59% rename from src/Foundatio/Queues/MessageQueue.cs rename to src/Foundatio/Messaging/MessageQueue.cs index ac6bddbb9..5cc476f2d 100644 --- a/src/Foundatio/Queues/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -1,16 +1,17 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; -using Foundatio.Messaging; using Foundatio.Serializer; using Foundatio.Utility; -namespace Foundatio.Queues; +namespace Foundatio.Messaging; public enum AckMode { @@ -18,7 +19,7 @@ public enum AckMode Manual } -public sealed record EnqueueOptions +public sealed record QueueMessageOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public TimeSpan? Delay { get; init; } @@ -30,22 +31,23 @@ public sealed record EnqueueOptions public MessageHeaders? Headers { get; init; } } -public sealed record ReceiveOptions +public sealed record QueueReceiveOptions { public string? Source { get; init; } public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); } -public sealed record WorkerOptions +public sealed record QueueConsumerOptions { public AckMode AckMode { get; init; } = AckMode.Auto; public string? Source { get; init; } + public string? Key { get; init; } public int MaxConcurrency { get; init; } = 1; public int MaxAttempts { get; init; } = 5; public Func? RedeliveryBackoff { get; init; } } -public sealed record MessageQueueOptions +public sealed record QueueOptions { public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; @@ -55,12 +57,19 @@ public sealed record MessageQueueOptions public TimeProvider TimeProvider { get; init; } = TimeProvider.System; } -public interface IMessageQueue : IAsyncDisposable +public interface IQueue : IAsyncDisposable { - Task EnqueueAsync(T message, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task?> ReceiveAsync(ReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task StartWorkingAsync(Func, CancellationToken, Task> handler, WorkerOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; +} + +public interface IMessageConsumer : IAsyncDisposable +{ + string Source { get; } + string Key { get; } } public interface IReceivedMessage where T : class @@ -75,30 +84,33 @@ public interface IReceivedMessage where T : class bool IsHandled { get; } CancellationToken CancellationToken { get; } Task CompleteAsync(CancellationToken cancellationToken = default); - Task RejectAsync(bool retry = true, string? reason = null, CancellationToken cancellationToken = default); + Task AbandonAsync(CancellationToken cancellationToken = default); Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default); Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default); } -public sealed class MessageQueue : IMessageQueue +public sealed class MessageQueue : IQueue { private readonly IMessageTransport _transport; - private readonly MessageQueueOptions _options; + private readonly QueueOptions _options; + private readonly ConcurrentDictionary _consumers = new(StringComparer.Ordinal); private int _isDisposed; - public MessageQueue(IMessageTransport transport, MessageQueueOptions? options = null) + public MessageQueue(IMessageTransport transport, QueueOptions? options = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); - _options = options ?? new MessageQueueOptions(); + _options = options ?? new QueueOptions(); } - public async Task EnqueueAsync(T message, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); ThrowIfDisposed(); - options ??= new EnqueueOptions(); + options ??= new QueueMessageOptions(); + ValidateSendOptions(options); + string destination = GetDestination(typeof(T), options.Destination); var sendOptions = CreateSendOptions(options); string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); @@ -111,17 +123,19 @@ public async Task EnqueueAsync(T message, EnqueueOptions? options = n var item = result.Items.Count > 0 ? result.Items[0] : null; if (item is null || !item.Success) - throw new QueueException($"Unable to enqueue message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}"); + throw new MessageQueueException($"Unable to enqueue message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}"); return item.MessageId ?? messageId; } - public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); ThrowIfDisposed(); - options ??= new EnqueueOptions(); + options ??= new QueueMessageOptions(); + ValidateSendOptions(options); + string destination = GetDestination(typeof(T), options.Destination); var sendOptions = CreateSendOptions(options); int index = 0; @@ -141,17 +155,17 @@ public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? var result = await _transport.SendAsync(destination, transportMessages, sendOptions, cancellationToken).AnyContext(); if (!result.AllSucceeded) - throw new QueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{destination}\"."); + throw new MessageQueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{destination}\"."); } - public async Task?> ReceiveAsync(ReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class { ThrowIfDisposed(); if (_transport is not ISupportsPull pull) - throw new QueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); + throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); - options ??= new ReceiveOptions(); + options ??= new QueueReceiveOptions(); string source = GetDestination(typeof(T), options.Source); var entries = await pull.ReceiveAsync(source, new ReceiveRequest { @@ -165,29 +179,73 @@ public async Task EnqueueBatchAsync(IEnumerable messages, EnqueueOptions? return await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); } - public async Task StartWorkingAsync(Func, CancellationToken, Task> handler, WorkerOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); ThrowIfDisposed(); + cancellationToken.ThrowIfCancellationRequested(); - options ??= new WorkerOptions(); + options ??= new QueueConsumerOptions(); string source = GetDestination(typeof(T), options.Source); + string key = GetConsumerKey(typeof(T), source, options.Key); + + if (_consumers.TryGetValue(key, out var existing) && !existing.IsDisposed) + return existing; - if (_transport is ISupportsPush push) + var handle = new MessageConsumerHandle(source, key, RemoveConsumer); + if (!_consumers.TryAdd(key, handle)) { - await using var subscription = await push.SubscribeAsync(source, async (entry, token) => + await handle.DisposeAsync().AnyContext(); + return _consumers[key]; + } + + try + { + if (_transport is ISupportsPush push) { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + var subscription = await push.SubscribeAsync(source, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + + handle.SetPushSubscription(subscription); + return handle; + } - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - return; + if (_transport is not ISupportsPull pull) + throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support receiving messages."); + + handle.Start(RunPullConsumerLoopAsync(source, pull, handler, options, handle.CancellationToken)); + return handle; + } + catch + { + await handle.DisposeAsync().AnyContext(); + throw; } + } - if (_transport is not ISupportsPull pull) - throw new QueueException($"Transport \"{_transport.GetType().Name}\" does not support receiving messages."); + public async Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + var consumers = _consumers.Values.ToArray(); + foreach (var consumer in consumers) + await consumer.DisposeAsync().AnyContext(); + + await _transport.DisposeAsync().AnyContext(); + } + private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull, Func, CancellationToken, Task> handler, QueueConsumerOptions options, CancellationToken cancellationToken) where T : class + { while (!cancellationToken.IsCancellationRequested) { var entries = await pull.ReceiveAsync(source, new ReceiveRequest @@ -196,20 +254,14 @@ public async Task StartWorkingAsync(Func, CancellationTok MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken).AnyContext(); - foreach (var entry in entries) + var tasks = entries.Select(async entry => { var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); - } - } - } - - public ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return ValueTask.CompletedTask; + }).ToArray(); - return _transport.DisposeAsync(); + await Task.WhenAll(tasks).AnyContext(); + } } private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken ct) where T : class @@ -218,23 +270,23 @@ private async Task> CreateReceivedMessageAsync(TransportE { var message = _options.Serializer.Deserialize(entry.Body); if (message is null) - throw new QueueException($"Message \"{entry.Id}\" deserialized to null."); + throw new MessageQueueException($"Message \"{entry.Id}\" deserialized to null."); return new ReceivedMessage(_transport, entry, message, ct, _options.RuntimeStore, _options.TimeProvider); } - catch (Exception ex) when (ex is not QueueException) + catch (Exception ex) when (ex is not MessageQueueException) { await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); - throw new QueueException($"Unable to deserialize message \"{entry.Id}\".", ex); + throw new MessageQueueException($"Unable to deserialize message \"{entry.Id}\".", ex); } - catch (QueueException) + catch (MessageQueueException) { await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); throw; } } - private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, WorkerOptions options, CancellationToken ct) where T : class + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, QueueConsumerOptions options, CancellationToken ct) where T : class { try { @@ -245,20 +297,24 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func received) - await received.RejectAsync(retry, "handler-error", redeliveryDelay, ct).AnyContext(); - else - await message.RejectAsync(retry, "handler-error", ct).AnyContext(); + if (message.Attempts >= options.MaxAttempts) + { + await message.DeadLetterAsync("handler-error", ct).AnyContext(); + return; } + + TimeSpan? redeliveryDelay = options.RedeliveryBackoff?.Invoke(message.Attempts); + if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ReceivedMessage received) + await received.AbandonAsync(delay, ct).AnyContext(); + else + await message.AbandonAsync(ct).AnyContext(); } } - private TransportMessage CreateTransportMessage(T message, EnqueueOptions options, string? messageId = null) where T : class + private TransportMessage CreateTransportMessage(T message, QueueMessageOptions options, string? messageId = null) where T : class { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) @@ -288,7 +344,7 @@ private TransportMessage CreateTransportMessage(T message, EnqueueOptions opt }; } - private TransportSendOptions CreateSendOptions(EnqueueOptions options) + private TransportSendOptions CreateSendOptions(QueueMessageOptions options) { return new TransportSendOptions { @@ -298,6 +354,15 @@ private TransportSendOptions CreateSendOptions(EnqueueOptions options) }; } + private void ValidateSendOptions(QueueMessageOptions options) + { + if (options.Priority != MessagePriority.Normal && _transport is not ISupportsPriority) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); + + if (options.TimeToLive is not null && _transport is not ISupportsExpiration) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); + } + private async Task TryScheduleDispatchesAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) @@ -328,7 +393,7 @@ private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out return false; if (_options.RuntimeStore is null) - throw new QueueException($"Delayed queue delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(MessageQueueOptions)}.{nameof(MessageQueueOptions.RuntimeStore)}."); + throw new MessageQueueException($"Delayed queue delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(QueueOptions)}.{nameof(QueueOptions.RuntimeStore)}."); return true; } @@ -349,9 +414,13 @@ private Task ScheduleDispatchAsync(ScheduledDispatchKind kind, string destinatio private string GetDestination(Type messageType, string? destination) { - return !String.IsNullOrEmpty(destination) - ? destination - : (_options.DestinationResolver?.Invoke(messageType) ?? ToKebabCase(messageType.Name)); + if (!String.IsNullOrEmpty(destination)) + return destination; + + if (_options.DestinationResolver?.Invoke(messageType) is { Length: > 0 } resolved) + return resolved; + + return messageType.GetCustomAttribute()?.Destination ?? MessageRoutingConventions.ToKebabCase(messageType.Name); } private string GetMessageType(Type messageType) @@ -359,40 +428,26 @@ private string GetMessageType(Type messageType) return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; } + private static string GetConsumerKey(Type messageType, string source, string? key) + { + return !String.IsNullOrEmpty(key) + ? key + : $"{source}:{messageType.FullName ?? messageType.Name}"; + } + private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken ct) { await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, ct).AnyContext(); } - private void ThrowIfDisposed() + private void RemoveConsumer(string key, MessageConsumerHandle handle) { - ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + _consumers.TryRemove(new KeyValuePair(key, handle)); } - private static string ToKebabCase(string value) + private void ThrowIfDisposed() { - if (String.IsNullOrEmpty(value)) - return value; - - Span buffer = stackalloc char[value.Length * 2]; - int position = 0; - for (int index = 0; index < value.Length; index++) - { - char current = value[index]; - if (Char.IsUpper(current)) - { - if (index > 0) - buffer[position++] = '-'; - - buffer[position++] = Char.ToLowerInvariant(current); - } - else - { - buffer[position++] = current; - } - } - - return new String(buffer[..position]); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); } } @@ -432,19 +487,45 @@ public Task CompleteAsync(CancellationToken cancellationToken = default) return _transport.CompleteAsync(_entry, cancellationToken); } - public Task RejectAsync(bool retry = true, string? reason = null, CancellationToken cancellationToken = default) + public Task AbandonAsync(CancellationToken cancellationToken = default) { - return RejectAsync(retry, reason, redeliveryDelay: null, cancellationToken); + if (!TryMarkHandled()) + return Task.CompletedTask; + + return _transport.AbandonAsync(_entry, cancellationToken); } - internal Task RejectAsync(bool retry, string? reason, TimeSpan? redeliveryDelay, CancellationToken cancellationToken = default) + internal async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) { - if (!retry) - return DeadLetterAsync(reason, cancellationToken); + if (!TryMarkHandled()) + return; + + if (_transport is ISupportsRedeliveryDelay redelivery) + { + await redelivery.AbandonAsync(_entry, redeliveryDelay, cancellationToken).AnyContext(); + return; + } + + if (_runtimeStore is null) + throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or {nameof(QueueOptions)}.{nameof(QueueOptions.RuntimeStore)}."); - return redeliveryDelay is { } delay && delay > TimeSpan.Zero - ? AbandonAsync(delay, cancellationToken) - : AbandonAsync(cancellationToken); + int nextAttempt = _entry.DeliveryCount + 1; + var headers = _entry.Headers.ToBuilder() + .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) + .Build(); + + await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = $"{_entry.Id}:retry:{nextAttempt}", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = _entry.Destination, + Body = _entry.Body, + Headers = headers, + Options = new TransportSendOptions { Priority = Priority }, + DueUtc = _timeProvider.GetUtcNow().Add(redeliveryDelay) + }, cancellationToken).AnyContext(); + + await _transport.CompleteAsync(_entry, cancellationToken).AnyContext(); } public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) @@ -464,79 +545,102 @@ public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancella public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) { - throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue messages."); + throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue or pub/sub messages."); } - private Task AbandonAsync(CancellationToken ct) + internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) { - if (!TryMarkHandled()) - return Task.CompletedTask; + if (transport is not ISupportsDeadLetter deadLetter) + throw new NotSupportedException($"Transport \"{transport.GetType().Name}\" does not support dead-lettering."); - return _transport.AbandonAsync(_entry, ct); + await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); } - private async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken ct) + private bool TryMarkHandled() { - if (!TryMarkHandled()) - return; + return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; + } +} - if (_transport is ISupportsRedeliveryDelay redelivery) +internal static class MessageRoutingConventions +{ + public static string ToKebabCase(string value) + { + if (String.IsNullOrEmpty(value)) + return value; + + Span buffer = stackalloc char[value.Length * 2]; + int position = 0; + for (int index = 0; index < value.Length; index++) { - await redelivery.AbandonAsync(_entry, redeliveryDelay, ct).AnyContext(); - return; + char current = value[index]; + if (Char.IsUpper(current)) + { + if (index > 0) + buffer[position++] = '-'; + + buffer[position++] = Char.ToLowerInvariant(current); + } + else + { + buffer[position++] = current; + } } - if (_runtimeStore is null) - throw new QueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or {nameof(MessageQueueOptions)}.{nameof(MessageQueueOptions.RuntimeStore)}."); + return new String(buffer[..position]); + } +} - int nextAttempt = _entry.DeliveryCount + 1; - var headers = _entry.Headers.ToBuilder() - .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) - .Build(); +internal sealed class MessageConsumerHandle : IMessageConsumer +{ + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly Action _remove; + private IPushSubscription? _pushSubscription; + private Task? _worker; + private int _isDisposed; - await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState - { - DispatchId = $"{_entry.Id}:retry:{nextAttempt}", - Kind = ScheduledDispatchKind.QueueMessage, - Destination = _entry.Destination, - Body = _entry.Body, - Headers = headers, - Options = new TransportSendOptions { Priority = Priority }, - DueUtc = _timeProvider.GetUtcNow().Add(redeliveryDelay) - }, ct).AnyContext(); + public MessageConsumerHandle(string source, string key, Action remove) + { + Source = source; + Key = key; + _remove = remove; + } + + public string Source { get; } + public string Key { get; } + public CancellationToken CancellationToken => _cancellationTokenSource.Token; + public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; - await _transport.CompleteAsync(_entry, ct).AnyContext(); + public void SetPushSubscription(IPushSubscription subscription) + { + _pushSubscription = subscription; } - internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) + public void Start(Task worker) { - if (transport is ISupportsDeadLetter deadLetter) - { - await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + _worker = worker; + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) return; - } - var headers = entry.Headers.ToBuilder(); - if (!String.IsNullOrEmpty(reason)) - headers.Set(KnownHeaders.DeadLetterReason, reason); + await _cancellationTokenSource.CancelAsync().AnyContext(); - var result = await transport.SendAsync($"{entry.Destination}-deadletter", [ - new TransportMessage + if (_pushSubscription is not null) + await _pushSubscription.DisposeAsync().AnyContext(); + + if (_worker is not null) + { + try { - MessageId = $"{entry.Id}:deadletter", - Body = entry.Body, - Headers = headers.Build() + await _worker.AnyContext(); } - ], new TransportSendOptions(), cancellationToken).AnyContext(); - - if (!result.AllSucceeded) - throw new QueueException($"Unable to write message \"{entry.Id}\" to the managed dead-letter destination \"{entry.Destination}-deadletter\"."); - - await transport.CompleteAsync(entry, cancellationToken).AnyContext(); - } + catch (OperationCanceledException) { } + } - private bool TryMarkHandled() - { - return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; + _cancellationTokenSource.Dispose(); + _remove(Key, this); } } diff --git a/src/Foundatio/Messaging/MessageQueueException.cs b/src/Foundatio/Messaging/MessageQueueException.cs new file mode 100644 index 000000000..f9935d4dd --- /dev/null +++ b/src/Foundatio/Messaging/MessageQueueException.cs @@ -0,0 +1,17 @@ +using System; + +namespace Foundatio.Messaging; + +/// +/// Exception thrown when a message queue operation fails. +/// +public class MessageQueueException : MessageBusException +{ + public MessageQueueException(string message) : base(message) + { + } + + public MessageQueueException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Foundatio/Messaging/MessageRouteAttribute.cs b/src/Foundatio/Messaging/MessageRouteAttribute.cs new file mode 100644 index 000000000..418816365 --- /dev/null +++ b/src/Foundatio/Messaging/MessageRouteAttribute.cs @@ -0,0 +1,21 @@ +using System; + +namespace Foundatio.Messaging; + +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct, Inherited = false)] +public sealed class MessageRouteAttribute : Attribute +{ + public MessageRouteAttribute() + { + } + + public MessageRouteAttribute(string name) + { + Destination = name; + Topic = name; + } + + public string? Destination { get; set; } + public string? Topic { get; set; } + public string? Subscription { get; set; } +} diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 3c2d22b72..3cd13bcff 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Foundatio.Queues; namespace Foundatio.Messaging; @@ -72,6 +71,19 @@ public sealed record ReceiveRequest public TimeSpan? MaxWaitTime { get; init; } } +public sealed record MessageDestinationStats +{ + public long Queued { get; init; } + public long Working { get; init; } + public long Deadletter { get; init; } + public long Enqueued { get; init; } + public long Dequeued { get; init; } + public long Completed { get; init; } + public long Abandoned { get; init; } + public long Errors { get; init; } + public long Timeouts { get; init; } +} + public sealed record SendItemResult { public string? MessageId { get; init; } @@ -156,7 +168,7 @@ public interface ISupportsVisibilityTimeout : IMessageTransport public interface ISupportsStats : IMessageTransport { - Task GetStatsAsync(string destination, CancellationToken ct); + Task GetStatsAsync(string destination, CancellationToken ct); } public interface ISupportsPriority : IMessageTransport { } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 6e52f9cd1..7345ff493 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -1,18 +1,19 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; -using Foundatio.Queues; using Foundatio.Serializer; using Foundatio.Utility; namespace Foundatio.Messaging; -public sealed record PublishOptions +public sealed record PubSubMessageOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public TimeSpan? Delay { get; init; } @@ -24,10 +25,11 @@ public sealed record PublishOptions public MessageHeaders? Headers { get; init; } } -public sealed record SubscriptionOptions +public sealed record PubSubSubscriptionOptions { public string? Topic { get; init; } public string? Subscription { get; init; } + public string? Key { get; init; } public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; public int MaxAttempts { get; init; } = 5; @@ -46,15 +48,24 @@ public sealed record PubSubOptions public interface IPubSub : IAsyncDisposable { - Task PublishAsync(T message, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task PublishBatchAsync(IEnumerable messages, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task SubscribeAsync(Func, CancellationToken, Task> handler, SubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; +} + +public interface IMessageSubscription : IAsyncDisposable +{ + string Topic { get; } + string Subscription { get; } + string Key { get; } } public sealed class PubSub : IPubSub { private readonly IMessageTransport _transport; private readonly PubSubOptions _options; + private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); private int _isDisposed; public PubSub(IMessageTransport transport, PubSubOptions? options = null) @@ -63,12 +74,14 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) _options = options ?? new PubSubOptions(); } - public async Task PublishAsync(T message, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); ThrowIfDisposed(); - options ??= new PublishOptions(); + options ??= new PubSubMessageOptions(); + ValidateSendOptions(options); + string topic = GetTopic(typeof(T), options.Topic); await EnsureTopicAsync(topic, cancellationToken).AnyContext(); @@ -85,12 +98,14 @@ public async Task PublishAsync(T message, PublishOptions? options = null, Can throw new MessageBusException($"Unable to publish message to \"{topic}\": {item?.ErrorCode ?? "unknown error"}"); } - public async Task PublishBatchAsync(IEnumerable messages, PublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); ThrowIfDisposed(); - options ??= new PublishOptions(); + options ??= new PubSubMessageOptions(); + ValidateSendOptions(options); + string topic = GetTopic(typeof(T), options.Topic); await EnsureTopicAsync(topic, cancellationToken).AnyContext(); @@ -115,31 +130,75 @@ public async Task PublishBatchAsync(IEnumerable messages, PublishOptions? throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{topic}\"."); } - public async Task SubscribeAsync(Func, CancellationToken, Task> handler, SubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); ThrowIfDisposed(); + cancellationToken.ThrowIfCancellationRequested(); - options ??= new SubscriptionOptions(); + options ??= new PubSubSubscriptionOptions(); string topic = GetTopic(typeof(T), options.Topic); string subscription = GetSubscription(typeof(T), topic, options.Subscription); + string key = GetSubscriptionKey(typeof(T), topic, subscription, options.Key); await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); - if (_transport is ISupportsPush push) + if (_subscriptions.TryGetValue(key, out var existing) && !existing.IsDisposed) + return existing; + + var handle = new MessageSubscriptionHandle(topic, subscription, key, RemoveSubscription); + if (!_subscriptions.TryAdd(key, handle)) + { + await handle.DisposeAsync().AnyContext(); + return _subscriptions[key]; + } + + try { - await using var pushSubscription = await push.SubscribeAsync(subscription, async (entry, token) => + if (_transport is ISupportsPush push) { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + var pushSubscription = await push.SubscribeAsync(subscription, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + + handle.SetPushSubscription(pushSubscription); + return handle; + } - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - return; + if (_transport is not ISupportsPull pull) + throw new MessageBusException($"Transport \"{_transport.GetType().Name}\" does not support subscriptions."); + + handle.Start(RunPullSubscriptionLoopAsync(subscription, pull, handler, options, handle.CancellationToken)); + return handle; } + catch + { + await handle.DisposeAsync().AnyContext(); + throw; + } + } - if (_transport is not ISupportsPull pull) - throw new MessageBusException($"Transport \"{_transport.GetType().Name}\" does not support subscriptions."); + public async Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + var subscriptions = _subscriptions.Values.ToArray(); + foreach (var subscription in subscriptions) + await subscription.DisposeAsync().AnyContext(); + + await _transport.DisposeAsync().AnyContext(); + } + + private async Task RunPullSubscriptionLoopAsync(string subscription, ISupportsPull pull, Func, CancellationToken, Task> handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) where T : class + { while (!cancellationToken.IsCancellationRequested) { var entries = await pull.ReceiveAsync(subscription, new ReceiveRequest @@ -148,20 +207,14 @@ public async Task SubscribeAsync(Func, CancellationToken, MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken).AnyContext(); - foreach (var entry in entries) + var tasks = entries.Select(async entry => { var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); - } - } - } + }).ToArray(); - public ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return ValueTask.CompletedTask; - - return _transport.DisposeAsync(); + await Task.WhenAll(tasks).AnyContext(); + } } private async Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) @@ -203,7 +256,7 @@ private async Task> CreateReceivedMessageAsync(TransportE } } - private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, SubscriptionOptions options, CancellationToken cancellationToken) where T : class + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) where T : class { try { @@ -214,12 +267,20 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func= options.MaxAttempts) + { + await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); + return; + } + + await message.AbandonAsync(cancellationToken).AnyContext(); } } - private TransportMessage CreateTransportMessage(T message, PublishOptions options, string? messageId = null) where T : class + private TransportMessage CreateTransportMessage(T message, PubSubMessageOptions options, string? messageId = null) where T : class { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) @@ -249,7 +310,7 @@ private TransportMessage CreateTransportMessage(T message, PublishOptions opt }; } - private TransportSendOptions CreateSendOptions(PublishOptions options) + private TransportSendOptions CreateSendOptions(PubSubMessageOptions options) { return new TransportSendOptions { @@ -259,6 +320,15 @@ private TransportSendOptions CreateSendOptions(PublishOptions options) }; } + private void ValidateSendOptions(PubSubMessageOptions options) + { + if (options.Priority != MessagePriority.Normal && _transport is not ISupportsPriority) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); + + if (options.TimeToLive is not null && _transport is not ISupportsExpiration) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); + } + private async Task TryScheduleDispatchesAsync(string topic, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) @@ -310,16 +380,32 @@ private Task ScheduleDispatchAsync(string topic, TransportMessage message, Trans private string GetTopic(Type messageType, string? topic) { - return !String.IsNullOrEmpty(topic) - ? topic - : (_options.TopicResolver?.Invoke(messageType) ?? ToKebabCase(messageType.Name)); + if (!String.IsNullOrEmpty(topic)) + return topic; + + if (_options.TopicResolver?.Invoke(messageType) is { Length: > 0 } resolved) + return resolved; + + var route = messageType.GetCustomAttribute(); + return route?.Topic ?? route?.Destination ?? MessageRoutingConventions.ToKebabCase(messageType.Name); } private string GetSubscription(Type messageType, string topic, string? subscription) { - return !String.IsNullOrEmpty(subscription) - ? subscription - : (_options.SubscriptionResolver?.Invoke(messageType, topic) ?? $"{topic}.{ToKebabCase(messageType.Name)}"); + if (!String.IsNullOrEmpty(subscription)) + return subscription; + + if (_options.SubscriptionResolver?.Invoke(messageType, topic) is { Length: > 0 } resolved) + return resolved; + + return messageType.GetCustomAttribute()?.Subscription ?? $"{topic}.{MessageRoutingConventions.ToKebabCase(messageType.Name)}"; + } + + private static string GetSubscriptionKey(Type messageType, string topic, string subscription, string? key) + { + return !String.IsNullOrEmpty(key) + ? key + : $"{topic}:{subscription}:{messageType.FullName ?? messageType.Name}"; } private string GetMessageType(Type messageType) @@ -332,34 +418,69 @@ private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string rea await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken).AnyContext(); } + private void RemoveSubscription(string key, MessageSubscriptionHandle handle) + { + _subscriptions.TryRemove(new KeyValuePair(key, handle)); + } + private void ThrowIfDisposed() { ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); } +} - private static string ToKebabCase(string value) +internal sealed class MessageSubscriptionHandle : IMessageSubscription +{ + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly Action _remove; + private IPushSubscription? _pushSubscription; + private Task? _worker; + private int _isDisposed; + + public MessageSubscriptionHandle(string topic, string subscription, string key, Action remove) { - if (String.IsNullOrEmpty(value)) - return value; + Topic = topic; + Subscription = subscription; + Key = key; + _remove = remove; + } - Span buffer = stackalloc char[value.Length * 2]; - int position = 0; - for (int index = 0; index < value.Length; index++) - { - char current = value[index]; - if (Char.IsUpper(current)) - { - if (index > 0) - buffer[position++] = '-'; + public string Topic { get; } + public string Subscription { get; } + public string Key { get; } + public CancellationToken CancellationToken => _cancellationTokenSource.Token; + public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; - buffer[position++] = Char.ToLowerInvariant(current); - } - else + public void SetPushSubscription(IPushSubscription subscription) + { + _pushSubscription = subscription; + } + + public void Start(Task worker) + { + _worker = worker; + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + await _cancellationTokenSource.CancelAsync().AnyContext(); + + if (_pushSubscription is not null) + await _pushSubscription.DisposeAsync().AnyContext(); + + if (_worker is not null) + { + try { - buffer[position++] = current; + await _worker.AnyContext(); } + catch (OperationCanceledException) { } } - return new String(buffer[..position]); + _cancellationTokenSource.Dispose(); + _remove(Key, this); } } diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index 522d389bf..a6b48d120 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -120,11 +120,13 @@ public async Task RunAsync_WhenJobSucceeds_TracksCompletedStateAsync() await using var serviceProvider = new ServiceCollection() .AddSingleton(probe) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); - string jobId = await client.RunAsync(new RunJobOptions { JobId = "job-1" }, cancellationToken); + JobHandle handle = await client.EnqueueAsync(new JobRequestOptions { JobId = "job-1" }, cancellationToken); + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); - var state = await client.GetAsync(jobId, cancellationToken); + var state = await handle.GetStateAsync(cancellationToken); Assert.NotNull(state); Assert.Equal(1, probe.RunCount); Assert.Equal(JobStatus.Completed, state.Status); @@ -145,16 +147,18 @@ public async Task RequestCancellationAsync_WhenJobIsRunning_CancelsAndTracksStat await using var serviceProvider = new ServiceCollection() .AddSingleton(probe) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); - var runTask = client.RunAsync(new RunJobOptions { JobId = "job-1" }, cancellationToken); + JobHandle handle = await client.EnqueueAsync(new JobRequestOptions { JobId = "job-1" }, cancellationToken); + var runTask = worker.RunAsync(handle.JobId, cancellationToken); await probe.Started.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - Assert.True(await client.RequestCancellationAsync("job-1", cancellationToken)); + Assert.True(await handle.RequestCancellationAsync(cancellationToken)); await probe.Cancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - string jobId = await runTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken); - var state = await client.GetAsync(jobId, cancellationToken); + Assert.True(await runTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken)); + var state = await handle.GetStateAsync(cancellationToken); Assert.NotNull(state); Assert.Equal(JobStatus.Cancelled, state.Status); diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index d6c041819..a4c8bf29c 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -51,8 +51,8 @@ public async Task RunDueOccurrencesAsync_WhenOccurrenceIsDue_RunsConfiguredJobAs await using var serviceProvider = new ServiceCollection() .AddSingleton(probe) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); - var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, worker, nodeId: "node-a"); var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); await scheduler.ScheduleAsync(new ScheduledJobDefinition @@ -167,8 +167,8 @@ public async Task RunDueOccurrencesAsync_WhenJobFails_RetriesThenDeadLettersAsyn await using var serviceProvider = new ServiceCollection() .AddSingleton(probe) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); - var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, worker, nodeId: "node-a"); var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); await scheduler.ScheduleAsync(new ScheduledJobDefinition @@ -205,8 +205,8 @@ public async Task RunDueOccurrencesAsync_WhenProcessingLeaseExpired_ReclaimsAndR await using var serviceProvider = new ServiceCollection() .AddSingleton(probe) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); - var processor = new JobScheduleProcessor(scheduler, store, client, nodeId: "node-a"); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + var processor = new JobScheduleProcessor(scheduler, store, worker, nodeId: "node-a"); var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); const string jobId = "nightly:20260101000000:global"; @@ -251,8 +251,8 @@ private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJo var serviceProvider = new ServiceCollection() .AddSingleton(new JobSchedulerProbe()) .BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: nodeId); - return new JobScheduleProcessor(scheduler, store, client, nodeId: nodeId, transport: transport); + var worker = new JobWorker(store, serviceProvider, nodeId: nodeId); + return new JobScheduleProcessor(scheduler, store, worker, nodeId: nodeId, transport: transport); } private sealed class JobSchedulerProbe diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index d39e7a1f5..a8ff8bcd0 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -5,7 +5,6 @@ using Foundatio.AsyncEx; using Foundatio.Jobs; using Foundatio.Messaging; -using Foundatio.Queues; using Foundatio.Tests.Extensions; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -26,28 +25,24 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() var firstReceived = new AsyncCountdownEvent(1); var secondReceived = new AsyncCountdownEvent(1); - var first = pubSub.SubscribeAsync((message, _) => + await using var first = await pubSub.SubscribeAsync((message, _) => { Assert.Equal("published", message.Message.Data); firstReceived.Signal(); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "subscriber-a" }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "subscriber-a" }, cts.Token); - var second = pubSub.SubscribeAsync((message, _) => + await using var second = await pubSub.SubscribeAsync((message, _) => { Assert.Equal("published", message.Message.Data); secondReceived.Signal(); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "subscriber-b" }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "subscriber-b" }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "published" }, cancellationToken: cancellationToken); await firstReceived.WaitAsync(TimeSpan.FromSeconds(2)); await secondReceived.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await first); - await Assert.ThrowsAnyAsync(async () => await second); - var firstStats = await transport.GetStatsAsync("subscriber-a", cancellationToken); var secondStats = await transport.GetStatsAsync("subscriber-b", cancellationToken); Assert.Equal(1, firstStats.Completed); @@ -64,12 +59,12 @@ public async Task PublishBatchAsync_DeliversAllMessagesAsync() cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(2); - var subscription = pubSub.SubscribeAsync((message, _) => + await using var subscription = await pubSub.SubscribeAsync((message, _) => { Assert.StartsWith("batch-", message.Message.Data); received.Signal(); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "batch-subscription" }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "batch-subscription" }, cts.Token); await pubSub.PublishBatchAsync([ new PreviewEvent { Data = "batch-one" }, @@ -77,9 +72,6 @@ await pubSub.PublishBatchAsync([ ], cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await subscription); - var stats = await transport.GetStatsAsync("batch-subscription", cancellationToken); Assert.Equal(2, stats.Completed); } @@ -93,13 +85,13 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - var subscription = pubSub.SubscribeAsync((message, _) => + await using var subscription = await pubSub.SubscribeAsync((message, _) => { received.TrySetResult(message); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); - await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new PublishOptions + await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new PubSubMessageOptions { CorrelationId = "corr-456", Priority = MessagePriority.High, @@ -118,8 +110,6 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() Assert.Equal("acme", message.Headers["tenant"]); Assert.Equal(typeof(PreviewEvent).FullName, message.MessageType); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await subscription); } [Fact] @@ -134,20 +124,18 @@ public async Task PublishAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(1); - var subscription = pubSub.SubscribeAsync((_, _) => + await using var subscription = await pubSub.SubscribeAsync((_, _) => { received.Signal(); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); - await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PublishOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PubSubMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); await Assert.ThrowsAsync(async () => await received.WaitAsync(TimeSpan.FromMilliseconds(50))); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); await received.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await subscription); } [Fact] @@ -161,7 +149,7 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() var received = new AsyncCountdownEvent(2); int attempts = 0; - var subscription = pubSub.SubscribeAsync((message, _) => + await using var subscription = await pubSub.SubscribeAsync((message, _) => { attempts++; Assert.Equal(attempts, message.Attempts); @@ -171,25 +159,35 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() throw new InvalidOperationException("try again"); return Task.CompletedTask; - }, new SubscriptionOptions { Subscription = "retry-subscription", MaxAttempts = 2 }, cts.Token); + }, new PubSubSubscriptionOptions { Subscription = "retry-subscription", MaxAttempts = 2 }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "retry" }, cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await subscription); - var stats = await transport.GetStatsAsync("retry-subscription", cancellationToken); Assert.Equal(1, stats.Completed); Assert.Equal(1, stats.Abandoned); } + [Fact] + public async Task SubscribeAsync_WithSameKey_ReturnsExistingSubscriptionAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var pubSub = new PubSub(new InMemoryMessageTransport()); + + await using var first = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + var second = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + + Assert.Same(first, second); + } + + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { var serviceProvider = new ServiceCollection().BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); - return new JobScheduleProcessor(new InMemoryJobScheduler(), store, client, nodeId: "node-a", transport: transport); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + return new JobScheduleProcessor(new InMemoryJobScheduler(), store, worker, nodeId: "node-a", transport: transport); } private sealed class PreviewEvent diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index cc59597b3..4b20bbbe1 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -2,10 +2,10 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Foundatio; using Foundatio.AsyncEx; using Foundatio.Jobs; using Foundatio.Messaging; -using Foundatio.Queues; using Foundatio.Tests.Extensions; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -21,7 +21,7 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() await using var transport = new InMemoryMessageTransport(); await using var queue = new MessageQueue(transport); - string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new EnqueueOptions + string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new QueueMessageOptions { CorrelationId = "corr-123", Priority = MessagePriority.High, @@ -30,7 +30,7 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() ]) }, cancellationToken); - var received = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(received); Assert.Equal(id, received.Id); @@ -58,10 +58,10 @@ public async Task EnqueueBatchAsync_UsesDestinationOverrideAsync() await queue.EnqueueBatchAsync([ new PreviewWorkItem { Data = "one" }, new PreviewWorkItem { Data = "two" } - ], new EnqueueOptions { Destination = "custom-work" }, cancellationToken); + ], new QueueMessageOptions { Destination = "custom-work" }, cancellationToken); - var first = await queue.ReceiveAsync(new ReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - var second = await queue.ReceiveAsync(new ReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var first = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); Assert.NotNull(second); @@ -73,18 +73,18 @@ await queue.EnqueueBatchAsync([ } [Fact] - public async Task RejectAsync_WithRetry_RedeliversAsync() + public async Task AbandonAsync_RedeliversAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var queue = new MessageQueue(new InMemoryMessageTransport()); await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cancellationToken); - var first = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var first = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); - await first.RejectAsync(cancellationToken: cancellationToken); + await first.AbandonAsync(cancellationToken); - var second = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(second); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.Attempts); @@ -100,7 +100,7 @@ public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() await using var queue = new MessageQueue(new InMemoryMessageTransport()); await queue.EnqueueAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); await Assert.ThrowsAsync(async () => await message.RenewLockAsync(cancellationToken: cancellationToken)); @@ -113,24 +113,24 @@ public async Task ReportProgressAsync_WhenUntracked_ThrowsAsync() await using var queue = new MessageQueue(new InMemoryMessageTransport()); await queue.EnqueueAsync(new PreviewWorkItem { Data = "progress" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); await Assert.ThrowsAsync(async () => await message.ReportProgressAsync(50, "half", cancellationToken)); } [Fact] - public async Task RejectAsync_WithoutRetry_DeadLettersAsync() + public async Task DeadLetterAsync_DeadLettersAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); await using var queue = new MessageQueue(transport); await queue.EnqueueAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); - await message.RejectAsync(retry: false, reason: "validation", cancellationToken: cancellationToken); + await message.DeadLetterAsync("validation", cancellationToken); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Deadletter); @@ -138,7 +138,7 @@ public async Task RejectAsync_WithoutRetry_DeadLettersAsync() } [Fact] - public async Task StartWorkingAsync_WithAutoAck_CompletesMessageAsync() + public async Task StartConsumerAsync_WithAutoAck_CompletesMessageAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); @@ -147,7 +147,7 @@ public async Task StartWorkingAsync_WithAutoAck_CompletesMessageAsync() cts.CancelAfter(TimeSpan.FromSeconds(10)); var handled = new AsyncCountdownEvent(1); - var worker = queue.StartWorkingAsync((message, _) => + await using var consumer = await queue.StartConsumerAsync((message, _) => { Assert.Equal("work", message.Message.Data); handled.Signal(); @@ -156,9 +156,6 @@ public async Task StartWorkingAsync_WithAutoAck_CompletesMessageAsync() await queue.EnqueueAsync(new PreviewWorkItem { Data = "work" }, cancellationToken: cts.Token); await handled.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await worker); - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Completed); } @@ -169,17 +166,17 @@ public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() var cancellationToken = TestContext.Current.CancellationToken; var store = new InMemoryJobRuntimeStore(); await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport, new MessageQueueOptions { RuntimeStore = store }); + await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); - var immediate = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var immediate = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); - var delayed = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var delayed = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); @@ -191,17 +188,17 @@ public async Task EnqueueAsync_WithDelayAndNoRuntimeStore_ThrowsAsync() var cancellationToken = TestContext.Current.CancellationToken; await using var queue = new MessageQueue(new InMemoryMessageTransport()); - await Assert.ThrowsAsync(async () => - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new EnqueueOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); + await Assert.ThrowsAsync(async () => + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); } [Fact] - public async Task StartWorkingAsync_WithRedeliveryBackoff_SchedulesRetryThroughRuntimeStoreAsync() + public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThroughRuntimeStoreAsync() { var cancellationToken = TestContext.Current.CancellationToken; var store = new InMemoryJobRuntimeStore(); await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport, new MessageQueueOptions { RuntimeStore = store }); + await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -209,7 +206,7 @@ public async Task StartWorkingAsync_WithRedeliveryBackoff_SchedulesRetryThroughR var secondAttempt = new AsyncCountdownEvent(1); int attempts = 0; - var worker = queue.StartWorkingAsync((message, _) => + await using var consumer = await queue.StartConsumerAsync((message, _) => { attempts++; if (attempts == 1) @@ -223,19 +220,17 @@ public async Task StartWorkingAsync_WithRedeliveryBackoff_SchedulesRetryThroughR Assert.Equal("retry", message.Message.Data); secondAttempt.Signal(); return Task.CompletedTask; - }, new WorkerOptions { RedeliveryBackoff = _ => TimeSpan.FromMinutes(1), MaxAttempts = 3 }, cts.Token); + }, new QueueConsumerOptions { RedeliveryBackoff = _ => TimeSpan.FromMinutes(1), MaxAttempts = 3 }, cts.Token); await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cts.Token); await firstAttempt.WaitAsync(TimeSpan.FromSeconds(2)); - var immediate = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var immediate = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); await secondAttempt.WaitAsync(TimeSpan.FromSeconds(2)); - await cts.CancelAsync(); - await Assert.ThrowsAnyAsync(async () => await worker); } [Fact] @@ -245,9 +240,9 @@ public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync await using var transport = new InMemoryMessageTransport(); await using var queue = new MessageQueue(transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new EnqueueOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new QueueMessageOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); - var received = await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(received); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); @@ -255,7 +250,7 @@ public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync } [Fact] - public async Task ReceiveAsync_WithPoisonPayload_DeadLettersAndThrowsQueueExceptionAsync() + public async Task ReceiveAsync_WithPoisonPayload_DeadLettersAndThrowsMessageQueueExceptionAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); @@ -271,8 +266,8 @@ await transport.SendAsync("preview-work-item", [ } ], new TransportSendOptions(), cancellationToken); - await Assert.ThrowsAsync(async () => - await queue.ReceiveAsync(new ReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); + await Assert.ThrowsAsync(async () => + await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Deadletter); @@ -280,11 +275,63 @@ await Assert.ThrowsAsync(async () => } + [Fact] + public async Task AddFoundatio_WithInMemoryMessagingAndJobs_RegistersAppFacingServices() + { + var services = new ServiceCollection(); + + services.AddFoundatio() + .Messaging.UseInMemory() + .Jobs.UseInMemoryRuntime(); + + await using var provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + } + + [Fact] + public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await queue.EnqueueAsync(new RoutedWorkItem { Data = "route" }, cancellationToken: cancellationToken); + + var received = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "routed-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + + Assert.NotNull(received); + Assert.Equal("route", received.Message.Data); + await received.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task StartConsumerAsync_WithSameKey_ReturnsExistingConsumerAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await using var first = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + var second = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + + Assert.Same(first, second); + } + + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { var serviceProvider = new ServiceCollection().BuildServiceProvider(); - var client = new JobClient(store, serviceProvider, nodeId: "node-a"); - return new JobScheduleProcessor(new InMemoryJobScheduler(), store, client, nodeId: "node-a", transport: transport); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + return new JobScheduleProcessor(new InMemoryJobScheduler(), store, worker, nodeId: "node-a", transport: transport); + } + + [MessageRoute("routed-work")] + private sealed class RoutedWorkItem + { + public string? Data { get; set; } } private sealed class PreviewWorkItem From 8827af16bc755a2297509d989dbd9d967b90120d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 27 Jun 2026 17:56:16 -0500 Subject: [PATCH 09/57] feat: centralize message routing --- .agents/skills/foundatio/SKILL.md | 12 +- docs/guide/messaging-jobs-redesign.md | 118 +++++- src/Foundatio/FoundatioServicesExtensions.cs | 31 +- src/Foundatio/Jobs/JobRuntime.cs | 105 ++++- src/Foundatio/Jobs/JobScheduler.cs | 15 +- src/Foundatio/Messaging/MessageQueue.cs | 362 +++++++++++++----- src/Foundatio/Messaging/MessageRouting.cs | 257 +++++++++++++ src/Foundatio/Messaging/PubSub.cs | 257 ++++++++----- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 27 ++ .../Foundatio.Tests/Messaging/PubSubTests.cs | 71 +++- .../Queue/MessageQueueTests.cs | 95 ++++- 11 files changed, 1121 insertions(+), 229 deletions(-) create mode 100644 src/Foundatio/Messaging/MessageRouting.cs diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index 2056da4f7..d48cb0376 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -26,12 +26,14 @@ Query with specific questions, not single keywords. All provider docs (Redis, Az ## Messaging/Jobs Redesign Notes -- New queue/pub-sub APIs live under `Foundatio.Messaging`: app-facing `IQueue`, `IPubSub`, shared `IReceivedMessage`, `QueueMessageOptions`, `QueueReceiveOptions`, `QueueConsumerOptions`, `PubSubMessageOptions`, and `PubSubSubscriptionOptions`. -- Route resolution is type-driven: operation override > resolver/registration > `MessageRouteAttribute` > kebab-case type-name convention. `Destination` and `Source` are advanced queue overrides; `Topic` and `Subscription` are advanced pub/sub overrides. -- Listener startup returns handles: `StartConsumerAsync` returns `IMessageConsumer`; `SubscribeAsync` returns `IMessageSubscription`. Use `RunConsumerAsync` or `RunSubscriptionAsync` only for blocking lifetime loops. +- New queue/pub-sub APIs live under `Foundatio.Messaging`: app-facing `IQueue`, `IPubSub`, shared `IReceivedMessage` / `IReceivedMessage`, `QueueMessageOptions`, `QueueReceiveOptions`, `QueueConsumerOptions`, `PubSubMessageOptions`, and `PubSubSubscriptionOptions`. +- Route resolution is centralized in `IMessageRouter`: operation override > explicit route map > interface/base-type map > `MessageRouteAttribute` > configured convention. Configure with `.Messaging.ConfigureRouting(...)`; `Destination`, `Source`, `Topic`, and `Subscription` are advanced operation overrides. +- Pub/sub topic routing and subscription identity are separate. Topic answers where an event is published; subscription answers the logical service/consumer group. Prefer `UseSubscriptionIdentity(...)` or service identity configuration instead of deriving subscriptions from message type. +- Raw/envelope paths make grouped/global routes usable: `queue.ReceiveAsync(...)`, `queue.StartConsumerAsync(Func)`, and `pubsub.SubscribeAsync(Func)`. Typed `ReceiveAsync` / `SubscribeAsync` remain the simple path. +- Listener startup returns handles: `StartConsumerAsync` returns `IMessageConsumer`; `SubscribeAsync` returns `IMessageSubscription`. Same-key duplicate registrations are idempotent only for the same handler/options; conflicting registrations throw. Use `RunConsumerAsync` or `RunSubscriptionAsync` only for blocking lifetime loops. - Received-message settlement uses explicit verbs only: `CompleteAsync`, `AbandonAsync`, `DeadLetterAsync`, `RenewLockAsync`, and `ReportProgressAsync`. Unsupported capabilities should throw clearly instead of silently downgrading. -- New durable job runtime roles are separated: `IJobClient` submits and returns `JobHandle`, `IJobMonitor` queries state, `IJobRuntimeStore` persists runtime state, and `IJobWorker` claims and executes queued jobs. -- In-memory setup for the redesign is `services.AddFoundatio().Messaging.UseInMemory().Jobs.UseInMemoryRuntime()`. +- New durable job runtime roles are separated: `IJobClient` submits and returns `JobHandle`, `IJobMonitor` queries state, `IJobRuntimeStore` persists runtime state, and `IJobWorker` claims and executes queued jobs. Job types persist stable registry names via `IJobTypeRegistry` / `.Jobs.Register(name)`, not assembly-qualified names. +- In-memory setup for the redesign is `services.AddFoundatio().Messaging.ConfigureRouting(...).UseInMemory().Jobs.UseInMemoryRuntime()`. ## Core Interfaces diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index f4b403cf3..7bd16a2b6 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -1,15 +1,22 @@ # Messaging and Jobs Redesign -The new messaging API is app-facing and type-driven. Queue, pub/sub, received-message, headers, options, and transport abstractions live under `Foundatio.Messaging`; folders may separate queue, pub/sub, and transport code, but consumers should not need a separate queue namespace for the new API. +The new messaging API is app-facing and type-driven. Queue, pub/sub, received-message, headers, options, and common message abstractions live under `Foundatio.Messaging`; folders may separate queue, pub/sub, and provider contracts, but application code should start with `IQueue` and `IPubSub`. + +Provider-facing transport contracts such as `IMessageTransport`, `ISupportsPull`, `ISupportsPush`, and `ISupportsDeadLetter` are still public so external providers can implement them. They are infrastructure contracts, not the primary application surface. ## Setup -Register the in-memory messaging transport and durable job runtime through DI: +Register the in-memory messaging transport, central routing policy, and durable job runtime through DI: ```csharp services.AddFoundatio() - .Messaging.UseInMemory() - .Jobs.UseInMemoryRuntime(); + .Messaging.ConfigureRouting(r => r + .MapQueue("orders") + .MapTopic("order-events", typeof(IOrderEvent)) + .UseSubscriptionIdentity("billing-service")) + .UseInMemory() + .Jobs.UseInMemoryRuntime() + .Jobs.Register("search.rebuild"); ``` Application code should depend on `Foundatio.Messaging.IQueue`, `IPubSub`, `IJobClient`, `IJobMonitor`, and `IJobWorker` instead of constructing `InMemoryMessageTransport`, `MessageQueue`, `PubSub`, or `JobClient` directly. @@ -24,7 +31,7 @@ await queue.EnqueueAsync(new OrderSubmitted(id)); IReceivedMessage? received = await queue.ReceiveAsync(); ``` -Destination and source are advanced overrides: +Destination and source are advanced operation overrides: ```csharp await queue.EnqueueAsync(message, new QueueMessageOptions { @@ -36,37 +43,86 @@ IReceivedMessage? received = await queue.ReceiveAsync(HandleAsync); ``` -Use `RunConsumerAsync` when the desired behavior is a blocking lifetime loop. +Use `RunConsumerAsync` when the desired behavior is a blocking lifetime loop. Starting the same consumer key with the same handler and options is idempotent; starting the same key with conflicting handler/options throws. ## Pub/Sub -Pub/sub follows the same type-driven pattern: +Pub/sub follows the same type-driven publishing pattern: ```csharp await pubsub.PublishAsync(new OrderSubmitted(id)); +await using IMessageSubscription subscription = await pubsub.SubscribeAsync(HandleAsync); +``` + +Topic routing and subscription identity are separate. The topic answers where the event is published. The subscription answers which logical service or consumer group receives it: + +```csharp +services.AddFoundatio() + .Messaging.ConfigureRouting(r => r + .MapTopic("order-events", typeof(IOrderEvent)) + .UseSubscriptionIdentity("billing-service")); +``` + +Advanced operation overrides remain available: + +```csharp +await pubsub.PublishAsync(message, new PubSubMessageOptions { + Topic = "order-events-replay" +}); + await using IMessageSubscription subscription = await pubsub.SubscribeAsync( HandleAsync, - new PubSubSubscriptionOptions { Subscription = "billing-service" }); + new PubSubSubscriptionOptions { + Topic = "order-events-replay", + Subscription = "billing-replay" + }); ``` -`PubSubMessageOptions` mirrors queue send options where concepts overlap: priority, delay, TTL, correlation id, deduplication id, headers, and topic override. +`PubSubMessageOptions` mirrors queue send options where concepts overlap: priority, delay, TTL, correlation id, deduplication id, headers, and topic override. `PublishBatchAsync(IEnumerable)` supports heterogeneous event batches and groups sends by resolved topic. ## Routing Default route precedence is: ```text -options override > resolver/registration > MessageRouteAttribute > kebab-case type-name convention +operation override > explicit route map > interface/base-type map > MessageRouteAttribute > configured convention ``` -`QueueMessageOptions.Destination`, `QueueReceiveOptions.Source`, `PubSubMessageOptions.Topic`, and `PubSubSubscriptionOptions.Topic`/`Subscription` are explicit operation overrides. `QueueOptions.DestinationResolver`, `PubSubOptions.TopicResolver`, and `PubSubOptions.SubscriptionResolver` are the registration/resolver layer. `MessageRouteAttribute` is the type-local fallback before the final convention. +`IMessageRouter` is shared by queues and pub/sub. Configure routes once with `MessageRoutingOptionsBuilder`: + +```csharp +services.AddFoundatio() + .Messaging.ConfigureRouting(r => r + .UseGlobalQueue("all-work") + .UseGlobalTopic("all-events") + .MapQueue("orders") + .MapQueue("orders", typeof(OrderSubmitted), typeof(OrderCancelled)) + .MapQueue("order-work", typeof(IOrderMessage)) + .MapTopic("order-events", typeof(IOrderEvent)) + .UseConvention(ctx => $"app-{ctx.MessageType.Name.ToLowerInvariant()}")); +``` + +`QueueMessageOptions.Destination`, `QueueReceiveOptions.Source`, `PubSubMessageOptions.Topic`, and `PubSubSubscriptionOptions.Topic`/`Subscription` are final escape hatches for one operation. Attribute routing remains available for type-local defaults, but central routing should be the normal path. ## Delivery Settlement @@ -92,3 +148,43 @@ JobState? state = await handle.GetStateAsync(); ``` Execution belongs to `IJobWorker`, which claims queued jobs from `IJobRuntimeStore`. State and operational queries belong to `IJobMonitor`. Scheduled occurrences are created by `IJobScheduler` and materialized by `JobScheduleProcessor` through the runtime store. + +Persisted job type names come from `IJobTypeRegistry`. Register stable names for jobs that may move between assemblies or namespaces: + +```csharp +services.AddFoundatio() + .Jobs.Register("search.rebuild") + .Jobs.UseInMemoryRuntime(); +``` + +Unregistered jobs fall back to `Type.FullName`, not `AssemblyQualifiedName`. + +## Migration + +Legacy queue code usually moves from one queue instance per payload type to one app-facing queue plus routing: + +```csharp +// Legacy +await queue.EnqueueAsync(new OrderSubmitted(id)); // IQueue + +// New +await queue.EnqueueAsync(new OrderSubmitted(id)); // Foundatio.Messaging.IQueue +``` + +Legacy `IMessageBus` publish/subscribe code maps to `IPubSub` with explicit subscription identity: + +```csharp +// Legacy +await messageBus.PublishAsync(new OrderSubmitted(id)); +await messageBus.SubscribeAsync(HandleAsync); + +// New +await pubsub.PublishAsync(new OrderSubmitted(id)); +await using var subscription = await pubsub.SubscribeAsync(HandleAsync); +``` + +For per-type routing, register each type. For grouped routing, map an interface or base type. For global routing, set one queue destination or topic for all messages. Operation-level overrides should be reserved for exceptional paths such as replays or priority lanes. + +## Rollout Notes + +The in-memory transport proves the API shape and conformance coverage for local development. Before locking this as a stable public API, validate at least one external provider against the same routing, topic/subscription, delayed delivery, dead-letter, TTL, priority, and batch constraints. diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 35fd2f7b7..d016b43c0 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -270,6 +270,20 @@ public FoundatioBuilder Use(Func factory) return _builder; } + public MessagingBuilder ConfigureRouting(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + _services.ReplaceSingleton(_ => + { + var options = new MessageRoutingOptions(); + configure(new MessageRoutingOptionsBuilder(options)); + return new DefaultMessageRouter(options); + }); + + return this; + } + public FoundatioBuilder UseInMemory(InMemoryMessageBusOptions? options = null) { _services.ReplaceSingleton(sp => new InMemoryMessageBus(options.UseServices(sp))); @@ -318,6 +332,7 @@ private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) return new QueueOptions { Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, + Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), TimeProvider = serviceProvider.GetService() ?? TimeProvider.System }; @@ -328,6 +343,7 @@ private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvide return new PubSubOptions { Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, + Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), TimeProvider = serviceProvider.GetService() ?? TimeProvider.System }; @@ -369,18 +385,27 @@ public FoundatioBuilder UseInMemoryRuntime() return _builder; } + public FoundatioBuilder Register(string name) where TJob : IJob + { + ArgumentException.ThrowIfNullOrEmpty(name); + _services.AddSingleton(new JobTypeRegistration(name, typeof(TJob))); + return _builder; + } + private void RegisterJobServices() { + _services.ReplaceSingleton(sp => new JobTypeRegistry(sp.GetServices())); _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService())); - _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService())); + _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService(), sp.GetRequiredService())); + _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService(), jobTypes: sp.GetRequiredService())); _services.ReplaceSingleton(); _services.ReplaceSingleton(sp => new JobScheduleProcessor( sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), sp.GetService(), - transport: sp.GetService())); + transport: sp.GetService(), + jobTypes: sp.GetRequiredService())); } } diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index db3cb3b6a..7f4423baf 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -93,6 +93,80 @@ public sealed record JobRequestOptions public string? Name { get; init; } } +public sealed record JobTypeRegistration(string Name, Type JobType); + +public interface IJobTypeRegistry +{ + string GetName(Type jobType); + Type Resolve(string name); +} + +public sealed class JobTypeRegistry : IJobTypeRegistry +{ + private readonly Dictionary _nameToType; + private readonly Dictionary _typeToName; + + public JobTypeRegistry(IEnumerable? registrations = null) + { + _nameToType = new Dictionary(StringComparer.Ordinal); + _typeToName = new Dictionary(); + + foreach (var registration in registrations ?? []) + Add(registration); + } + + public string GetName(Type jobType) + { + ArgumentNullException.ThrowIfNull(jobType); + if (!typeof(IJob).IsAssignableFrom(jobType)) + throw new ArgumentException("Job type must implement IJob.", nameof(jobType)); + + return _typeToName.TryGetValue(jobType, out string? name) + ? name + : jobType.FullName ?? jobType.Name; + } + + public Type Resolve(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + + if (_nameToType.TryGetValue(name, out var registered)) + return registered; + + var jobType = Type.GetType(name, throwOnError: false); + if (jobType is null) + { + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + jobType = assembly.GetType(name, throwOnError: false); + if (jobType is not null) + break; + } + } + + if (jobType is null || !typeof(IJob).IsAssignableFrom(jobType)) + throw new InvalidOperationException($"Job type \"{name}\" could not be resolved to an IJob implementation."); + + return jobType; + } + + private void Add(JobTypeRegistration registration) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentException.ThrowIfNullOrEmpty(registration.Name); + ArgumentNullException.ThrowIfNull(registration.JobType); + + if (!typeof(IJob).IsAssignableFrom(registration.JobType)) + throw new ArgumentException("Job type must implement IJob.", nameof(registration)); + + if (_nameToType.TryGetValue(registration.Name, out var existing) && existing != registration.JobType) + throw new InvalidOperationException($"Job type name \"{registration.Name}\" is already registered for \"{existing.FullName}\"."); + + _nameToType[registration.Name] = registration.JobType; + _typeToName[registration.JobType] = registration.Name; + } +} + public sealed class JobHandle { private readonly IJobMonitor _monitor; @@ -431,11 +505,13 @@ public sealed class JobClient : IJobClient { private readonly IJobRuntimeStore _store; private readonly TimeProvider _timeProvider; + private readonly IJobTypeRegistry _jobTypes; - public JobClient(IJobRuntimeStore store, TimeProvider? timeProvider = null) + public JobClient(IJobRuntimeStore store, TimeProvider? timeProvider = null, IJobTypeRegistry? jobTypes = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); _timeProvider = timeProvider ?? TimeProvider.System; + _jobTypes = jobTypes ?? new JobTypeRegistry(); } public Task EnqueueAsync(JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob @@ -458,7 +534,7 @@ await _store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = name, - JobType = jobType.AssemblyQualifiedName, + JobType = _jobTypes.GetName(jobType), Status = JobStatus.Queued, CreatedUtc = now, LastUpdatedUtc = now @@ -480,14 +556,16 @@ public sealed class JobWorker : IJobWorker private readonly IJobRuntimeStore _store; private readonly IServiceProvider _serviceProvider; private readonly TimeProvider _timeProvider; + private readonly IJobTypeRegistry _jobTypes; private readonly string _nodeId; private readonly TimeSpan _lease; - public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null) + public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); _timeProvider = timeProvider ?? TimeProvider.System; + _jobTypes = jobTypes ?? new JobTypeRegistry(); _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; @@ -593,26 +671,19 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc } } - private static Type ResolveJobType(JobState state) + private Type ResolveJobType(JobState state) { if (String.IsNullOrEmpty(state.JobType)) throw new InvalidOperationException($"Job \"{state.JobId}\" does not have a job type and cannot be executed by a worker."); - var jobType = Type.GetType(state.JobType, throwOnError: false); - if (jobType is null) + try { - foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) - { - jobType = assembly.GetType(state.JobType, throwOnError: false); - if (jobType is not null) - break; - } + return _jobTypes.Resolve(state.JobType); + } + catch (Exception ex) when (ex is InvalidOperationException or ArgumentException) + { + throw new InvalidOperationException($"Job type \"{state.JobType}\" for job \"{state.JobId}\" could not be resolved to an IJob implementation.", ex); } - - if (jobType is null || !typeof(IJob).IsAssignableFrom(jobType)) - throw new InvalidOperationException($"Job type \"{state.JobType}\" for job \"{state.JobId}\" could not be resolved to an IJob implementation."); - - return jobType; } private IDisposable WatchCancellation(string jobId, CancellationTokenSource cancellationTokenSource) diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index d0141fc3a..bc9831eec 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -86,15 +86,17 @@ public sealed class JobScheduleProcessor private readonly IJobRuntimeStore _store; private readonly IJobWorker _jobWorker; private readonly TimeProvider _timeProvider; + private readonly IJobTypeRegistry _jobTypes; private readonly string _nodeId; private readonly IMessageTransport? _transport; - public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobWorker jobWorker, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null) + public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobWorker jobWorker, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null, IJobTypeRegistry? jobTypes = null) { _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _store = store ?? throw new ArgumentNullException(nameof(store)); _jobWorker = jobWorker ?? throw new ArgumentNullException(nameof(jobWorker)); _timeProvider = timeProvider ?? TimeProvider.System; + _jobTypes = jobTypes ?? new JobTypeRegistry(); _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; @@ -136,7 +138,7 @@ await _store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = definition.Name, - JobType = definition.JobType?.AssemblyQualifiedName, + JobType = GetJobTypeName(definition.JobType), Status = JobStatus.Scheduled, CreatedUtc = utcNow, LastUpdatedUtc = utcNow, @@ -267,7 +269,7 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat private async Task TryPrepareOccurrenceForRunAsync(string jobId, ScheduledJobDefinition definition, DateTimeOffset utcNow, CancellationToken cancellationToken) { - if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { JobType = definition.JobType?.AssemblyQualifiedName, LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) + if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { JobType = GetJobTypeName(definition.JobType), LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) return true; var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); @@ -287,13 +289,18 @@ private async Task TryPrepareOccurrenceForRunAsync(string jobId, Scheduled return await _store.TryTransitionAsync(jobId, JobStatus.Processing, JobStatus.Queued, new JobStatePatch { - JobType = definition.JobType?.AssemblyQualifiedName, + JobType = GetJobTypeName(definition.JobType), ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false); } + private string? GetJobTypeName(Type? jobType) + { + return jobType is null ? null : _jobTypes.GetName(jobType); + } + private async Task HasActiveOccurrenceAsync(string name, string scopeKey, CancellationToken cancellationToken) { var states = await _store.QueryAsync(new JobQuery { Name = name, Limit = 1000 }, cancellationToken).ConfigureAwait(false); diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index 5cc476f2d..52e58d834 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -4,7 +4,6 @@ using System.Diagnostics; using System.Globalization; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; @@ -34,6 +33,7 @@ public sealed record QueueMessageOptions public sealed record QueueReceiveOptions { public string? Source { get; init; } + public Type? RouteType { get; init; } public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); } @@ -41,6 +41,7 @@ public sealed record QueueConsumerOptions { public AckMode AckMode { get; init; } = AckMode.Auto; public string? Source { get; init; } + public Type? RouteType { get; init; } public string? Key { get; init; } public int MaxConcurrency { get; init; } = 1; public int MaxAttempts { get; init; } = 5; @@ -51,8 +52,7 @@ public sealed record QueueOptions { public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; - public Func? DestinationResolver { get; init; } - public Func? MessageTypeResolver { get; init; } + public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IJobRuntimeStore? RuntimeStore { get; init; } public TimeProvider TimeProvider { get; init; } = TimeProvider.System; } @@ -61,8 +61,12 @@ public interface IQueue : IAsyncDisposable { Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default); + Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default); Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default); Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task RunConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default); Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; } @@ -72,10 +76,10 @@ public interface IMessageConsumer : IAsyncDisposable string Key { get; } } -public interface IReceivedMessage where T : class +public interface IReceivedMessage { - T Message { get; } string Id { get; } + ReadOnlyMemory Body { get; } MessageHeaders Headers { get; } string? CorrelationId { get; } string? MessageType { get; } @@ -90,6 +94,11 @@ public interface IReceivedMessage where T : class Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default); } +public interface IReceivedMessage : IReceivedMessage where T : class +{ + T Message { get; } +} + public sealed class MessageQueue : IQueue { private readonly IMessageTransport _transport; @@ -114,7 +123,7 @@ public async Task EnqueueAsync(T message, QueueMessageOptions? option string destination = GetDestination(typeof(T), options.Destination); var sendOptions = CreateSendOptions(options); string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); - var transportMessage = CreateTransportMessage(message, options, messageId); + var transportMessage = CreateTransportMessage(message, typeof(T), options, messageId); if (await TryScheduleDispatchAsync(ScheduledDispatchKind.QueueMessage, destination, transportMessage, sendOptions, cancellationToken).AnyContext()) return messageId; @@ -131,31 +140,35 @@ public async Task EnqueueAsync(T message, QueueMessageOptions? option public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); + await EnqueueBatchCoreAsync(messages.Cast(), typeof(T), options, cancellationToken).AnyContext(); + } + + public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(messages); + await EnqueueBatchCoreAsync(messages, null, options, cancellationToken).AnyContext(); + } + + public async Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) + { ThrowIfDisposed(); - options ??= new QueueMessageOptions(); - ValidateSendOptions(options); + if (_transport is not ISupportsPull pull) + throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); - string destination = GetDestination(typeof(T), options.Destination); - var sendOptions = CreateSendOptions(options); - int index = 0; - var transportMessages = messages.Select(message => + options ??= new QueueReceiveOptions(); + Type routeType = options.RouteType ?? typeof(object); + string source = GetDestination(routeType, options.Source); + var entries = await pull.ReceiveAsync(source, new ReceiveRequest { - ArgumentNullException.ThrowIfNull(message); - string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; - index++; - return CreateTransportMessage(message, options, messageId); - }).ToArray(); - - if (transportMessages.Length == 0) - return; + MaxMessages = 1, + MaxWaitTime = options.MaxWaitTime + }, cancellationToken).AnyContext(); - if (await TryScheduleDispatchesAsync(ScheduledDispatchKind.QueueMessage, destination, transportMessages, sendOptions, cancellationToken).AnyContext()) - return; + if (entries.Count == 0) + return null; - var result = await _transport.SendAsync(destination, transportMessages, sendOptions, cancellationToken).AnyContext(); - if (!result.AllSucceeded) - throw new MessageQueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{destination}\"."); + return CreateReceivedMessage(entries[0], cancellationToken); } public async Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class @@ -166,7 +179,7 @@ public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOpti throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); options ??= new QueueReceiveOptions(); - string source = GetDestination(typeof(T), options.Source); + string source = GetDestination(options.RouteType ?? typeof(T), options.Source); var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, @@ -179,35 +192,126 @@ public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOpti return await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); } + public async Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + options ??= new QueueConsumerOptions(); + Type routeType = options.RouteType ?? typeof(object); + string source = GetDestination(routeType, options.Source); + string key = GetConsumerKey(routeType, source, options.Key); + var registration = MessageListenerRegistration.Create(handler, routeType, source, options); + + return await StartConsumerCoreAsync(source, key, registration, options, async (entry, token) => + { + var received = CreateReceivedMessage(entry, token); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, cancellationToken).AnyContext(); + } + public async Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); + options ??= new QueueConsumerOptions(); + Type routeType = options.RouteType ?? typeof(T); + string source = GetDestination(routeType, options.Source); + string key = GetConsumerKey(routeType, source, options.Key); + var registration = MessageListenerRegistration.Create(handler, routeType, source, options); + + return await StartConsumerCoreAsync(source, key, registration, options, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, cancellationToken).AnyContext(); + } + + public async Task RunConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) + { + await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + var consumers = _consumers.Values.ToArray(); + foreach (var consumer in consumers) + await consumer.DisposeAsync().AnyContext(); + + await _transport.DisposeAsync().AnyContext(); + } + + private async Task EnqueueBatchCoreAsync(IEnumerable messages, Type? declaredType, QueueMessageOptions? options, CancellationToken cancellationToken) + { ThrowIfDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - options ??= new QueueConsumerOptions(); - string source = GetDestination(typeof(T), options.Source); - string key = GetConsumerKey(typeof(T), source, options.Key); + options ??= new QueueMessageOptions(); + ValidateSendOptions(options); + + var sendOptions = CreateSendOptions(options); + var grouped = new Dictionary>(StringComparer.Ordinal); + int index = 0; + + foreach (var message in messages) + { + ArgumentNullException.ThrowIfNull(message); + Type messageType = declaredType ?? message.GetType(); + string destination = GetDestination(messageType, options.Destination); + string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; + index++; + + if (!grouped.TryGetValue(destination, out var transportMessages)) + { + transportMessages = []; + grouped.Add(destination, transportMessages); + } + + transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); + } + + foreach (var group in grouped) + { + if (await TryScheduleDispatchesAsync(ScheduledDispatchKind.QueueMessage, group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) + continue; + + var result = await _transport.SendAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); + if (!result.AllSucceeded) + throw new MessageQueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{group.Key}\"."); + } + } + + private async Task StartConsumerCoreAsync(string source, string key, MessageListenerRegistration registration, QueueConsumerOptions options, Func onMessage, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + cancellationToken.ThrowIfCancellationRequested(); if (_consumers.TryGetValue(key, out var existing) && !existing.IsDisposed) + { + existing.ThrowIfConflicting(registration); return existing; + } - var handle = new MessageConsumerHandle(source, key, RemoveConsumer); + var handle = new MessageConsumerHandle(source, key, registration, RemoveConsumer); if (!_consumers.TryAdd(key, handle)) { await handle.DisposeAsync().AnyContext(); - return _consumers[key]; + var current = _consumers[key]; + current.ThrowIfConflicting(registration); + return current; } try { if (_transport is ISupportsPush push) { - var subscription = await push.SubscribeAsync(source, async (entry, token) => - { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + var subscription = await push.SubscribeAsync(source, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); handle.SetPushSubscription(subscription); return handle; @@ -216,7 +320,7 @@ public async Task StartConsumerAsync(Func StartConsumerAsync(Func(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - var consumers = _consumers.Values.ToArray(); - foreach (var consumer in consumers) - await consumer.DisposeAsync().AnyContext(); - - await _transport.DisposeAsync().AnyContext(); - } - - private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull, Func, CancellationToken, Task> handler, QueueConsumerOptions options, CancellationToken cancellationToken) where T : class + private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull, Func onMessage, QueueConsumerOptions options, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -254,16 +340,16 @@ private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken).AnyContext(); - var tasks = entries.Select(async entry => - { - var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); - await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); - }).ToArray(); - + var tasks = entries.Select(entry => onMessage(entry, cancellationToken)).ToArray(); await Task.WhenAll(tasks).AnyContext(); } } + private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken ct) + { + return new ReceivedMessage(_transport, entry, ct, _options.RuntimeStore, _options.TimeProvider); + } + private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken ct) where T : class { try @@ -286,6 +372,21 @@ private async Task> CreateReceivedMessageAsync(TransportE } } + private async Task HandleMessageAsync(IReceivedMessage message, Func handler, QueueConsumerOptions options, CancellationToken ct) + { + try + { + await handler(message, ct).AnyContext(); + + if (options.AckMode == AckMode.Auto && !message.IsHandled) + await message.CompleteAsync(ct).AnyContext(); + } + catch + { + await SettleFailedMessageAsync(message, options, ct).AnyContext(); + } + } + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, QueueConsumerOptions options, CancellationToken ct) where T : class { try @@ -297,27 +398,32 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func= options.MaxAttempts) - { - await message.DeadLetterAsync("handler-error", ct).AnyContext(); - return; - } + private static async Task SettleFailedMessageAsync(IReceivedMessage message, QueueConsumerOptions options, CancellationToken ct) + { + if (message.IsHandled) + return; - TimeSpan? redeliveryDelay = options.RedeliveryBackoff?.Invoke(message.Attempts); - if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ReceivedMessage received) - await received.AbandonAsync(delay, ct).AnyContext(); - else - await message.AbandonAsync(ct).AnyContext(); + if (message.Attempts >= options.MaxAttempts) + { + await message.DeadLetterAsync("handler-error", ct).AnyContext(); + return; } + + TimeSpan? redeliveryDelay = options.RedeliveryBackoff?.Invoke(message.Attempts); + if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ISupportsDelayedMessageAbandon received) + await received.AbandonAsync(delay, ct).AnyContext(); + else + await message.AbandonAsync(ct).AnyContext(); } - private TransportMessage CreateTransportMessage(T message, QueueMessageOptions options, string? messageId = null) where T : class + private TransportMessage CreateTransportMessage(object message, Type messageType, QueueMessageOptions options, string? messageId = null) { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() - .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) + .Set(KnownHeaders.MessageType, GetMessageType(messageType)) .Set(KnownHeaders.ContentType, _options.ContentType) .Set(KnownHeaders.Priority, options.Priority.ToString()); @@ -414,18 +520,17 @@ private Task ScheduleDispatchAsync(ScheduledDispatchKind kind, string destinatio private string GetDestination(Type messageType, string? destination) { - if (!String.IsNullOrEmpty(destination)) - return destination; - - if (_options.DestinationResolver?.Invoke(messageType) is { Length: > 0 } resolved) - return resolved; - - return messageType.GetCustomAttribute()?.Destination ?? MessageRoutingConventions.ToKebabCase(messageType.Name); + return _options.Router.ResolveRoute(new MessageRouteContext + { + MessageType = messageType, + Role = MessageRouteRole.QueueDestination, + OperationOverride = destination + }); } private string GetMessageType(Type messageType) { - return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; + return _options.Router.ResolveMessageType(messageType); } private static string GetConsumerKey(Type messageType, string source, string? key) @@ -437,7 +542,7 @@ private static string GetConsumerKey(Type messageType, string source, string? ke private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken ct) { - await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, ct).AnyContext(); + await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, ct).AnyContext(); } private void RemoveConsumer(string key, MessageConsumerHandle handle) @@ -451,7 +556,12 @@ private void ThrowIfDisposed() } } -internal sealed class ReceivedMessage : IReceivedMessage where T : class +internal interface ISupportsDelayedMessageAbandon +{ + Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default); +} + +internal class ReceivedMessage : IReceivedMessage, ISupportsDelayedMessageAbandon { private readonly IMessageTransport _transport; private readonly TransportEntry _entry; @@ -459,18 +569,17 @@ internal sealed class ReceivedMessage : IReceivedMessage where T : class private readonly TimeProvider _timeProvider; private int _isHandled; - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) { _transport = transport; _entry = entry; _runtimeStore = runtimeStore; _timeProvider = timeProvider ?? TimeProvider.System; - Message = message; CancellationToken = cancellationToken; } - public T Message { get; } public string Id => _entry.Id; + public ReadOnlyMemory Body => _entry.Body; public MessageHeaders Headers => _entry.Headers; public string? CorrelationId => Headers.GetValueOrDefault(KnownHeaders.CorrelationId); public string? MessageType => Headers.GetValueOrDefault(KnownHeaders.MessageType); @@ -495,7 +604,7 @@ public Task AbandonAsync(CancellationToken cancellationToken = default) return _transport.AbandonAsync(_entry, cancellationToken); } - internal async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) + public async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) { if (!TryMarkHandled()) return; @@ -562,6 +671,17 @@ private bool TryMarkHandled() } } +internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class +{ + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) + : base(transport, entry, cancellationToken, runtimeStore, timeProvider) + { + Message = message; + } + + public T Message { get; } +} + internal static class MessageRoutingConventions { public static string ToKebabCase(string value) @@ -599,18 +719,26 @@ internal sealed class MessageConsumerHandle : IMessageConsumer private Task? _worker; private int _isDisposed; - public MessageConsumerHandle(string source, string key, Action remove) + public MessageConsumerHandle(string source, string key, MessageListenerRegistration registration, Action remove) { Source = source; Key = key; + Registration = registration; _remove = remove; } public string Source { get; } public string Key { get; } + public MessageListenerRegistration Registration { get; } public CancellationToken CancellationToken => _cancellationTokenSource.Token; public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; + public void ThrowIfConflicting(MessageListenerRegistration registration) + { + if (!Registration.Matches(registration)) + throw new InvalidOperationException($"A consumer with key \"{Key}\" is already registered with different handler or options."); + } + public void SetPushSubscription(IPushSubscription subscription) { _pushSubscription = subscription; @@ -644,3 +772,53 @@ public async ValueTask DisposeAsync() _remove(Key, this); } } + +internal sealed record MessageListenerRegistration +{ + public required Type MessageType { get; init; } + public required string Source { get; init; } + public required Delegate Handler { get; init; } + public required AckMode AckMode { get; init; } + public required int MaxConcurrency { get; init; } + public required int MaxAttempts { get; init; } + public required bool HasRedeliveryBackoff { get; init; } + + public static MessageListenerRegistration Create(Delegate handler, Type messageType, string source, QueueConsumerOptions options) + { + return new MessageListenerRegistration + { + MessageType = messageType, + Source = source, + Handler = handler, + AckMode = options.AckMode, + MaxConcurrency = Math.Max(1, options.MaxConcurrency), + MaxAttempts = options.MaxAttempts, + HasRedeliveryBackoff = options.RedeliveryBackoff is not null + }; + } + + public static MessageListenerRegistration Create(Delegate handler, Type messageType, string topic, string subscription, PubSubSubscriptionOptions options) + { + return new MessageListenerRegistration + { + MessageType = messageType, + Source = $"{topic}:{subscription}", + Handler = handler, + AckMode = options.AckMode, + MaxConcurrency = Math.Max(1, options.MaxConcurrency), + MaxAttempts = options.MaxAttempts, + HasRedeliveryBackoff = false + }; + } + + public bool Matches(MessageListenerRegistration other) + { + return MessageType == other.MessageType + && String.Equals(Source, other.Source, StringComparison.Ordinal) + && Handler == other.Handler + && AckMode == other.AckMode + && MaxConcurrency == other.MaxConcurrency + && MaxAttempts == other.MaxAttempts + && HasRedeliveryBackoff == other.HasRedeliveryBackoff; + } +} diff --git a/src/Foundatio/Messaging/MessageRouting.cs b/src/Foundatio/Messaging/MessageRouting.cs new file mode 100644 index 000000000..f40f33a5a --- /dev/null +++ b/src/Foundatio/Messaging/MessageRouting.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Foundatio.Messaging; + +public enum MessageRouteRole +{ + QueueDestination, + PubSubTopic +} + +public sealed record MessageRouteContext +{ + public required Type MessageType { get; init; } + public required MessageRouteRole Role { get; init; } + public string? OperationOverride { get; init; } +} + +public sealed record MessageSubscriptionContext +{ + public required Type MessageType { get; init; } + public required string Topic { get; init; } + public string? OperationOverride { get; init; } +} + +public interface IMessageRouter +{ + string ResolveRoute(MessageRouteContext context); + string ResolveSubscription(MessageSubscriptionContext context); + string ResolveMessageType(Type messageType); +} + +public sealed record MessageRouteMap +{ + public required Type MessageType { get; init; } + public required MessageRouteRole Role { get; init; } + public required string Route { get; init; } +} + +public sealed class MessageRoutingOptions +{ + internal List RouteMaps { get; } = []; + + public string? GlobalQueueDestination { get; set; } + public string? GlobalPubSubTopic { get; set; } + public string? SubscriptionIdentity { get; set; } + public string? ServiceIdentity { get; set; } + public Func? Convention { get; set; } + public Func? MessageTypeResolver { get; set; } +} + +public sealed class MessageRoutingOptionsBuilder +{ + private readonly MessageRoutingOptions _options; + + public MessageRoutingOptionsBuilder() + : this(new MessageRoutingOptions()) + { + } + + internal MessageRoutingOptionsBuilder(MessageRoutingOptions options) + { + _options = options; + } + + public MessageRoutingOptionsBuilder UseGlobalQueue(string destination) + { + ArgumentException.ThrowIfNullOrEmpty(destination); + _options.GlobalQueueDestination = destination; + return this; + } + + public MessageRoutingOptionsBuilder UseGlobalTopic(string topic) + { + ArgumentException.ThrowIfNullOrEmpty(topic); + _options.GlobalPubSubTopic = topic; + return this; + } + + public MessageRoutingOptionsBuilder MapQueue(string destination) + { + return MapQueue(typeof(T), destination); + } + + public MessageRoutingOptionsBuilder MapQueue(Type messageType, string destination) + { + return Map(MessageRouteRole.QueueDestination, destination, messageType); + } + + public MessageRoutingOptionsBuilder MapQueue(string destination, params Type[] messageTypes) + { + return Map(MessageRouteRole.QueueDestination, destination, messageTypes); + } + + public MessageRoutingOptionsBuilder MapTopic(string topic) + { + return MapTopic(typeof(T), topic); + } + + public MessageRoutingOptionsBuilder MapTopic(Type messageType, string topic) + { + return Map(MessageRouteRole.PubSubTopic, topic, messageType); + } + + public MessageRoutingOptionsBuilder MapTopic(string topic, params Type[] messageTypes) + { + return Map(MessageRouteRole.PubSubTopic, topic, messageTypes); + } + + public MessageRoutingOptionsBuilder UseSubscriptionIdentity(string subscription) + { + ArgumentException.ThrowIfNullOrEmpty(subscription); + _options.SubscriptionIdentity = subscription; + return this; + } + + public MessageRoutingOptionsBuilder UseServiceIdentity(string serviceIdentity) + { + ArgumentException.ThrowIfNullOrEmpty(serviceIdentity); + _options.ServiceIdentity = serviceIdentity; + return this; + } + + public MessageRoutingOptionsBuilder UseConvention(Func convention) + { + _options.Convention = convention ?? throw new ArgumentNullException(nameof(convention)); + return this; + } + + public MessageRoutingOptionsBuilder UseMessageTypeName(Func resolver) + { + _options.MessageTypeResolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); + return this; + } + + public MessageRoutingOptions Build() + { + return _options; + } + + private MessageRoutingOptionsBuilder Map(MessageRouteRole role, string route, params Type[] messageTypes) + { + ArgumentException.ThrowIfNullOrEmpty(route); + ArgumentNullException.ThrowIfNull(messageTypes); + + if (messageTypes.Length == 0) + throw new ArgumentException("At least one message type is required.", nameof(messageTypes)); + + foreach (var messageType in messageTypes) + { + ArgumentNullException.ThrowIfNull(messageType); + _options.RouteMaps.Add(new MessageRouteMap + { + MessageType = messageType, + Role = role, + Route = route + }); + } + + return this; + } +} + +public sealed class DefaultMessageRouter : IMessageRouter +{ + public static DefaultMessageRouter Instance { get; } = new(new MessageRoutingOptions()); + + private readonly MessageRoutingOptions _options; + + public DefaultMessageRouter(MessageRoutingOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public string ResolveRoute(MessageRouteContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.MessageType); + + if (!String.IsNullOrEmpty(context.OperationOverride)) + return context.OperationOverride; + + var exact = _options.RouteMaps.LastOrDefault(m => m.Role == context.Role && m.MessageType == context.MessageType); + if (exact is not null) + return exact.Route; + + var assignable = _options.RouteMaps.LastOrDefault(m => m.Role == context.Role && m.MessageType != context.MessageType && m.MessageType.IsAssignableFrom(context.MessageType)); + if (assignable is not null) + return assignable.Route; + + var attribute = context.MessageType.GetCustomAttribute(); + string? attributedRoute = context.Role == MessageRouteRole.QueueDestination + ? attribute?.Destination + : attribute?.Topic ?? attribute?.Destination; + + if (!String.IsNullOrEmpty(attributedRoute)) + return attributedRoute; + + string? configuredConvention = context.Role == MessageRouteRole.QueueDestination + ? _options.GlobalQueueDestination + : _options.GlobalPubSubTopic; + + if (!String.IsNullOrEmpty(configuredConvention)) + return configuredConvention; + + if (_options.Convention is not null) + { + string convention = _options.Convention(context); + if (!String.IsNullOrEmpty(convention)) + return convention; + } + + return MessageRoutingConventions.ToKebabCase(context.MessageType.Name); + } + + public string ResolveSubscription(MessageSubscriptionContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.MessageType); + ArgumentException.ThrowIfNullOrEmpty(context.Topic); + + if (!String.IsNullOrEmpty(context.OperationOverride)) + return context.OperationOverride; + + if (!String.IsNullOrEmpty(_options.SubscriptionIdentity)) + return _options.SubscriptionIdentity; + + if (context.MessageType.GetCustomAttribute()?.Subscription is { Length: > 0 } subscription) + return subscription; + + if (!String.IsNullOrEmpty(_options.ServiceIdentity)) + return _options.ServiceIdentity; + + return GetDefaultServiceIdentity(); + } + + public string ResolveMessageType(Type messageType) + { + ArgumentNullException.ThrowIfNull(messageType); + return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; + } + + private static string GetDefaultServiceIdentity() + { + string? configured = Environment.GetEnvironmentVariable("FOUNDATIO_SUBSCRIPTION_ID"); + if (!String.IsNullOrEmpty(configured)) + return configured; + + configured = Environment.GetEnvironmentVariable("FOUNDATIO_SERVICE_ID"); + if (!String.IsNullOrEmpty(configured)) + return configured; + + return MessageRoutingConventions.ToKebabCase(AppDomain.CurrentDomain.FriendlyName); + } +} diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 7345ff493..28c9e82da 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -4,7 +4,6 @@ using System.Diagnostics; using System.Globalization; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; @@ -28,6 +27,7 @@ public sealed record PubSubMessageOptions public sealed record PubSubSubscriptionOptions { public string? Topic { get; init; } + public Type? RouteType { get; init; } public string? Subscription { get; init; } public string? Key { get; init; } public AckMode AckMode { get; init; } = AckMode.Auto; @@ -39,9 +39,7 @@ public sealed record PubSubOptions { public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; - public Func? TopicResolver { get; init; } - public Func? MessageTypeResolver { get; init; } - public Func? SubscriptionResolver { get; init; } + public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IJobRuntimeStore? RuntimeStore { get; init; } public TimeProvider TimeProvider { get; init; } = TimeProvider.System; } @@ -50,7 +48,10 @@ public interface IPubSub : IAsyncDisposable { Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default); + Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; } @@ -87,7 +88,7 @@ public async Task PublishAsync(T message, PubSubMessageOptions? options = nul var sendOptions = CreateSendOptions(options); string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); - var transportMessage = CreateTransportMessage(message, options, messageId); + var transportMessage = CreateTransportMessage(message, typeof(T), options, messageId); if (await TryScheduleDispatchAsync(topic, transportMessage, sendOptions, cancellationToken).AnyContext()) return; @@ -101,66 +102,141 @@ public async Task PublishAsync(T message, PubSubMessageOptions? options = nul public async Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); + await PublishBatchCoreAsync(messages.Cast(), typeof(T), options, cancellationToken).AnyContext(); + } + + public async Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(messages); + await PublishBatchCoreAsync(messages, null, options, cancellationToken).AnyContext(); + } + + public async Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + options ??= new PubSubSubscriptionOptions(); + Type routeType = options.RouteType ?? typeof(object); + string topic = GetTopic(routeType, options.Topic); + string subscription = GetSubscription(routeType, topic, options.Subscription); + string key = GetSubscriptionKey(routeType, topic, subscription, options.Key); + var registration = MessageListenerRegistration.Create(handler, routeType, topic, subscription, options); + await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); + + return await SubscribeCoreAsync(topic, subscription, key, registration, options, async (entry, token) => + { + var received = CreateReceivedMessage(entry, token); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, cancellationToken).AnyContext(); + } + + public async Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + ArgumentNullException.ThrowIfNull(handler); + options ??= new PubSubSubscriptionOptions(); + Type routeType = options.RouteType ?? typeof(T); + string topic = GetTopic(routeType, options.Topic); + string subscription = GetSubscription(routeType, topic, options.Subscription); + string key = GetSubscriptionKey(routeType, topic, subscription, options.Key); + var registration = MessageListenerRegistration.Create(handler, routeType, topic, subscription, options); + await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); + + return await SubscribeCoreAsync(topic, subscription, key, registration, options, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, handler, options, token).AnyContext(); + }, cancellationToken).AnyContext(); + } + + public async Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) + { + await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + var subscriptions = _subscriptions.Values.ToArray(); + foreach (var subscription in subscriptions) + await subscription.DisposeAsync().AnyContext(); + + await _transport.DisposeAsync().AnyContext(); + } + + private async Task PublishBatchCoreAsync(IEnumerable messages, Type? declaredType, PubSubMessageOptions? options, CancellationToken cancellationToken) + { ThrowIfDisposed(); options ??= new PubSubMessageOptions(); ValidateSendOptions(options); - string topic = GetTopic(typeof(T), options.Topic); - await EnsureTopicAsync(topic, cancellationToken).AnyContext(); - var sendOptions = CreateSendOptions(options); + var grouped = new Dictionary>(StringComparer.Ordinal); int index = 0; - var transportMessages = messages.Select(message => + + foreach (var message in messages) { ArgumentNullException.ThrowIfNull(message); + Type messageType = declaredType ?? message.GetType(); + string topic = GetTopic(messageType, options.Topic); string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; index++; - return CreateTransportMessage(message, options, messageId); - }).ToArray(); - if (transportMessages.Length == 0) - return; + if (!grouped.TryGetValue(topic, out var transportMessages)) + { + transportMessages = []; + grouped.Add(topic, transportMessages); + } - if (await TryScheduleDispatchesAsync(topic, transportMessages, sendOptions, cancellationToken).AnyContext()) - return; + transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); + } + + foreach (var group in grouped) + { + await EnsureTopicAsync(group.Key, cancellationToken).AnyContext(); - var result = await _transport.SendAsync(topic, transportMessages, sendOptions, cancellationToken).AnyContext(); - if (!result.AllSucceeded) - throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{topic}\"."); + if (await TryScheduleDispatchesAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) + continue; + + var result = await _transport.SendAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); + if (!result.AllSucceeded) + throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{group.Key}\"."); + } } - public async Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + private async Task SubscribeCoreAsync(string topic, string subscription, string key, MessageListenerRegistration registration, PubSubSubscriptionOptions options, Func onMessage, CancellationToken cancellationToken) { - ArgumentNullException.ThrowIfNull(handler); ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); - options ??= new PubSubSubscriptionOptions(); - string topic = GetTopic(typeof(T), options.Topic); - string subscription = GetSubscription(typeof(T), topic, options.Subscription); - string key = GetSubscriptionKey(typeof(T), topic, subscription, options.Key); - await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); - if (_subscriptions.TryGetValue(key, out var existing) && !existing.IsDisposed) + { + existing.ThrowIfConflicting(registration); return existing; + } - var handle = new MessageSubscriptionHandle(topic, subscription, key, RemoveSubscription); + var handle = new MessageSubscriptionHandle(topic, subscription, key, registration, RemoveSubscription); if (!_subscriptions.TryAdd(key, handle)) { await handle.DisposeAsync().AnyContext(); - return _subscriptions[key]; + var current = _subscriptions[key]; + current.ThrowIfConflicting(registration); + return current; } try { if (_transport is ISupportsPush push) { - var pushSubscription = await push.SubscribeAsync(subscription, async (entry, token) => - { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); + var pushSubscription = await push.SubscribeAsync(subscription, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); handle.SetPushSubscription(pushSubscription); return handle; @@ -169,7 +245,7 @@ public async Task SubscribeAsync(Func SubscribeAsync(Func(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - var subscriptions = _subscriptions.Values.ToArray(); - foreach (var subscription in subscriptions) - await subscription.DisposeAsync().AnyContext(); - - await _transport.DisposeAsync().AnyContext(); - } - - private async Task RunPullSubscriptionLoopAsync(string subscription, ISupportsPull pull, Func, CancellationToken, Task> handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) where T : class + private async Task RunPullSubscriptionLoopAsync(string subscription, ISupportsPull pull, Func onMessage, PubSubSubscriptionOptions options, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -207,16 +265,16 @@ private async Task RunPullSubscriptionLoopAsync(string subscription, ISupport MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken).AnyContext(); - var tasks = entries.Select(async entry => - { - var received = await CreateReceivedMessageAsync(entry, cancellationToken).AnyContext(); - await HandleMessageAsync(received, handler, options, cancellationToken).AnyContext(); - }).ToArray(); - + var tasks = entries.Select(entry => onMessage(entry, cancellationToken)).ToArray(); await Task.WhenAll(tasks).AnyContext(); } } + private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) + { + return new ReceivedMessage(_transport, entry, cancellationToken, _options.RuntimeStore, _options.TimeProvider); + } + private async Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) { if (_transport is ISupportsProvisioning provisioning) @@ -256,6 +314,21 @@ private async Task> CreateReceivedMessageAsync(TransportE } } + private async Task HandleMessageAsync(IReceivedMessage message, Func handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) + { + try + { + await handler(message, cancellationToken).AnyContext(); + + if (options.AckMode == AckMode.Auto && !message.IsHandled) + await message.CompleteAsync(cancellationToken).AnyContext(); + } + catch + { + await SettleFailedMessageAsync(message, options, cancellationToken).AnyContext(); + } + } + private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) where T : class { try @@ -267,23 +340,28 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func= options.MaxAttempts) - { - await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); - return; - } + private static async Task SettleFailedMessageAsync(IReceivedMessage message, PubSubSubscriptionOptions options, CancellationToken cancellationToken) + { + if (message.IsHandled) + return; - await message.AbandonAsync(cancellationToken).AnyContext(); + if (message.Attempts >= options.MaxAttempts) + { + await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); + return; } + + await message.AbandonAsync(cancellationToken).AnyContext(); } - private TransportMessage CreateTransportMessage(T message, PubSubMessageOptions options, string? messageId = null) where T : class + private TransportMessage CreateTransportMessage(object message, Type messageType, PubSubMessageOptions options, string? messageId = null) { var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() - .Set(KnownHeaders.MessageType, GetMessageType(typeof(T))) + .Set(KnownHeaders.MessageType, GetMessageType(messageType)) .Set(KnownHeaders.ContentType, _options.ContentType) .Set(KnownHeaders.Priority, options.Priority.ToString()); @@ -380,25 +458,22 @@ private Task ScheduleDispatchAsync(string topic, TransportMessage message, Trans private string GetTopic(Type messageType, string? topic) { - if (!String.IsNullOrEmpty(topic)) - return topic; - - if (_options.TopicResolver?.Invoke(messageType) is { Length: > 0 } resolved) - return resolved; - - var route = messageType.GetCustomAttribute(); - return route?.Topic ?? route?.Destination ?? MessageRoutingConventions.ToKebabCase(messageType.Name); + return _options.Router.ResolveRoute(new MessageRouteContext + { + MessageType = messageType, + Role = MessageRouteRole.PubSubTopic, + OperationOverride = topic + }); } private string GetSubscription(Type messageType, string topic, string? subscription) { - if (!String.IsNullOrEmpty(subscription)) - return subscription; - - if (_options.SubscriptionResolver?.Invoke(messageType, topic) is { Length: > 0 } resolved) - return resolved; - - return messageType.GetCustomAttribute()?.Subscription ?? $"{topic}.{MessageRoutingConventions.ToKebabCase(messageType.Name)}"; + return _options.Router.ResolveSubscription(new MessageSubscriptionContext + { + MessageType = messageType, + Topic = topic, + OperationOverride = subscription + }); } private static string GetSubscriptionKey(Type messageType, string topic, string subscription, string? key) @@ -410,12 +485,12 @@ private static string GetSubscriptionKey(Type messageType, string topic, string private string GetMessageType(Type messageType) { - return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; + return _options.Router.ResolveMessageType(messageType); } private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) { - await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken).AnyContext(); + await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken).AnyContext(); } private void RemoveSubscription(string key, MessageSubscriptionHandle handle) @@ -437,20 +512,28 @@ internal sealed class MessageSubscriptionHandle : IMessageSubscription private Task? _worker; private int _isDisposed; - public MessageSubscriptionHandle(string topic, string subscription, string key, Action remove) + public MessageSubscriptionHandle(string topic, string subscription, string key, MessageListenerRegistration registration, Action remove) { Topic = topic; Subscription = subscription; Key = key; + Registration = registration; _remove = remove; } public string Topic { get; } public string Subscription { get; } public string Key { get; } + public MessageListenerRegistration Registration { get; } public CancellationToken CancellationToken => _cancellationTokenSource.Token; public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; + public void ThrowIfConflicting(MessageListenerRegistration registration) + { + if (!Registration.Matches(registration)) + throw new InvalidOperationException($"A subscription with key \"{Key}\" is already registered with different handler or options."); + } + public void SetPushSubscription(IPushSubscription subscription) { _pushSubscription = subscription; diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index a6b48d120..65f2e2e1e 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -138,6 +138,33 @@ public async Task RunAsync_WhenJobSucceeds_TracksCompletedStateAsync() Assert.Null(state.LeaseExpiresUtc); } + [Fact] + public async Task EnqueueAsync_WithRegisteredJobType_PersistsStableNameAndWorkerResolvesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobRuntimeProbe(); + var registry = new JobTypeRegistry([new JobTypeRegistration("search.rebuild", typeof(SuccessfulTrackedJob))]); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store, jobTypes: registry); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a", jobTypes: registry); + + JobHandle handle = await client.EnqueueAsync(new JobRequestOptions { JobId = "job-registered" }, cancellationToken); + var queued = await handle.GetStateAsync(cancellationToken); + + Assert.NotNull(queued); + Assert.Equal("search.rebuild", queued.JobType); + Assert.DoesNotContain(",", queued.JobType); + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); + + var completed = await handle.GetStateAsync(cancellationToken); + Assert.NotNull(completed); + Assert.Equal(JobStatus.Completed, completed.Status); + Assert.Equal(1, probe.RunCount); + } + [Fact] public async Task RequestCancellationAsync_WhenJobIsRunning_CancelsAndTracksStateAsync() { diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index a8ff8bcd0..e840e5584 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -171,17 +171,71 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() [Fact] - public async Task SubscribeAsync_WithSameKey_ReturnsExistingSubscriptionAsync() + public async Task SubscribeAsync_WithSameKeyAndSameRegistration_ReturnsExistingSubscriptionAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var pubSub = new PubSub(new InMemoryMessageTransport()); + Func, CancellationToken, Task> handler = (_, _) => Task.CompletedTask; - await using var first = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); - var second = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + await using var first = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + var second = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); Assert.Same(first, second); } + [Fact] + public async Task SubscribeAsync_WithSameKeyAndDifferentHandler_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var pubSub = new PubSub(new InMemoryMessageTransport()); + + await using var first = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + + await Assert.ThrowsAsync(async () => + await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken)); + } + + [Fact] + public async Task SubscribeAsync_WithGroupedTopicAndSubscriptionIdentity_ReceivesRawMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + var routing = new MessageRoutingOptionsBuilder() + .MapTopic("order-events", typeof(IGroupedEvent)) + .UseSubscriptionIdentity("billing-service") + .Build(); + await using var pubSub = new PubSub(transport, new PubSubOptions { Router = new DefaultMessageRouter(routing) }); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new AsyncCountdownEvent(2); + var messageTypes = new List(); + + await using var subscription = await pubSub.SubscribeAsync((message, _) => + { + lock (messageTypes) + messageTypes.Add(message.MessageType!); + + received.Signal(); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { RouteType = typeof(IGroupedEvent) }, cts.Token); + + await pubSub.PublishBatchAsync(new object[] + { + new PreviewEvent { Data = "one" }, + new OtherEvent { Data = "two" } + }, cancellationToken: cancellationToken); + + await received.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal("order-events", subscription.Topic); + Assert.Equal("billing-service", subscription.Subscription); + Assert.Contains(typeof(PreviewEvent).FullName!, messageTypes); + Assert.Contains(typeof(OtherEvent).FullName!, messageTypes); + + var stats = await transport.GetStatsAsync("billing-service", cancellationToken); + Assert.Equal(2, stats.Completed); + } + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { @@ -190,7 +244,16 @@ private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore sto return new JobScheduleProcessor(new InMemoryJobScheduler(), store, worker, nodeId: "node-a", transport: transport); } - private sealed class PreviewEvent + private interface IGroupedEvent + { + } + + private sealed class PreviewEvent : IGroupedEvent + { + public string? Data { get; set; } + } + + private sealed class OtherEvent : IGroupedEvent { public string? Data { get; set; } } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 4b20bbbe1..c12327238 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -156,8 +156,7 @@ public async Task StartConsumerAsync_WithAutoAck_CompletesMessageAsync() await queue.EnqueueAsync(new PreviewWorkItem { Data = "work" }, cancellationToken: cts.Token); await handled.WaitAsync(TimeSpan.FromSeconds(2)); - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); - Assert.Equal(1, stats.Completed); + await WaitForCompletedAsync(transport, "preview-work-item", cancellationToken); } [Fact] @@ -309,17 +308,92 @@ public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync } [Fact] - public async Task StartConsumerAsync_WithSameKey_ReturnsExistingConsumerAsync() + public async Task StartConsumerAsync_WithSameKeyAndSameRegistration_ReturnsExistingConsumerAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var queue = new MessageQueue(new InMemoryMessageTransport()); + Func, CancellationToken, Task> handler = (_, _) => Task.CompletedTask; - await using var first = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); - var second = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + await using var first = await queue.StartConsumerAsync(handler, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + var second = await queue.StartConsumerAsync(handler, new QueueConsumerOptions { Key = "shared" }, cancellationToken); Assert.Same(first, second); } + [Fact] + public async Task StartConsumerAsync_WithSameKeyAndDifferentHandler_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + + await using var first = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + + await Assert.ThrowsAsync(async () => + await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken)); + } + + [Fact] + public async Task ReceiveAsync_WithGroupedInterfaceRoute_ReturnsRawMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var routing = new MessageRoutingOptionsBuilder() + .MapQueue("grouped-work", typeof(IGroupedWorkItem)) + .Build(); + await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + + await queue.EnqueueBatchAsync(new object[] + { + new PreviewWorkItem { Data = "one" }, + new OtherWorkItem { Data = "two" } + }, cancellationToken: cancellationToken); + + var first = await queue.ReceiveAsync(new QueueReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new QueueReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + + Assert.NotNull(first); + Assert.NotNull(second); + Assert.NotEmpty(first.Body.ToArray()); + Assert.Equal(typeof(PreviewWorkItem).FullName, first.MessageType); + Assert.Equal(typeof(OtherWorkItem).FullName, second.MessageType); + + await first.CompleteAsync(cancellationToken); + await second.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task ReceiveAsync_WithGlobalQueueRoute_ReturnsRawMessageAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var routing = new MessageRoutingOptionsBuilder() + .UseGlobalQueue("all-work") + .Build(); + await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "global" }, cancellationToken: cancellationToken); + + var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + + Assert.NotNull(received); + Assert.Equal(typeof(PreviewWorkItem).FullName, received.MessageType); + await received.CompleteAsync(cancellationToken); + } + + + private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, string destination, CancellationToken cancellationToken) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(2); + while (DateTimeOffset.UtcNow < deadline) + { + var stats = await transport.GetStatsAsync(destination, cancellationToken); + if (stats.Completed == 1) + return; + + await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken); + } + + var finalStats = await transport.GetStatsAsync(destination, cancellationToken); + Assert.Equal(1, finalStats.Completed); + } private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { @@ -334,7 +408,16 @@ private sealed class RoutedWorkItem public string? Data { get; set; } } - private sealed class PreviewWorkItem + private interface IGroupedWorkItem + { + } + + private sealed class PreviewWorkItem : IGroupedWorkItem + { + public string? Data { get; set; } + } + + private sealed class OtherWorkItem : IGroupedWorkItem { public string? Data { get; set; } } From 2c0a362e0f8141eb7a9869f3ec6311c170450db4 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sat, 27 Jun 2026 18:54:45 -0500 Subject: [PATCH 10/57] feat: add messaging topology declarations --- .agents/skills/foundatio/SKILL.md | 7 +- docs/guide/messaging-jobs-redesign.md | 27 ++++-- src/Foundatio/FoundatioServicesExtensions.cs | 48 +++++++-- src/Foundatio/Messaging/MessageRouting.cs | 97 ++++++++++++++++--- src/Foundatio/Messaging/MessageTopology.cs | 71 ++++++++++++++ .../Foundatio.Tests/Messaging/PubSubTests.cs | 61 ++++++++++++ .../Queue/MessageQueueTests.cs | 44 ++++++++- 7 files changed, 321 insertions(+), 34 deletions(-) create mode 100644 src/Foundatio/Messaging/MessageTopology.cs diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index d48cb0376..d4cf806d8 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -27,9 +27,10 @@ Query with specific questions, not single keywords. All provider docs (Redis, Az ## Messaging/Jobs Redesign Notes - New queue/pub-sub APIs live under `Foundatio.Messaging`: app-facing `IQueue`, `IPubSub`, shared `IReceivedMessage` / `IReceivedMessage`, `QueueMessageOptions`, `QueueReceiveOptions`, `QueueConsumerOptions`, `PubSubMessageOptions`, and `PubSubSubscriptionOptions`. -- Route resolution is centralized in `IMessageRouter`: operation override > explicit route map > interface/base-type map > `MessageRouteAttribute` > configured convention. Configure with `.Messaging.ConfigureRouting(...)`; `Destination`, `Source`, `Topic`, and `Subscription` are advanced operation overrides. -- Pub/sub topic routing and subscription identity are separate. Topic answers where an event is published; subscription answers the logical service/consumer group. Prefer `UseSubscriptionIdentity(...)` or service identity configuration instead of deriving subscriptions from message type. -- Raw/envelope paths make grouped/global routes usable: `queue.ReceiveAsync(...)`, `queue.StartConsumerAsync(Func)`, and `pubsub.SubscribeAsync(Func)`. Typed `ReceiveAsync` / `SubscribeAsync` remain the simple path. +- Route resolution is centralized in `IMessageRouter`: operation override > explicit route map > interface/base-type map > `MessageRouteAttribute` > configured default/convention. Configure with `.Messaging.ConfigureRouting(...)`; use `UseDefaultQueue(...)`, `UseDefaultTopic(...)`, `MapQueue(...)`, and `MapTopic(...)` as the normal path. `Destination`, `Source`, `Topic`, and `Subscription` are advanced operation overrides. +- Routing configuration also declares startup topology. `IMessageTopology.GetDeclarations()` returns configured queues/topics/subscriptions, `EnsureAsync()` creates them through `ISupportsProvisioning`, and `ValidateAsync()` checks that they already exist for apps without create permissions. Operation overrides are not included in topology declarations. +- Pub/sub topic routing and subscription identity are separate. Topic answers where an event is published; subscription answers the logical service/consumer group. Same subscription across instances means competing consumers; different subscriptions on the same topic fan out. `PubSubSubscriptionOptions.Key` is only the local duplicate-listener key. +- Raw/envelope paths make grouped/default routes usable: `queue.ReceiveAsync(...)`, `queue.StartConsumerAsync(Func)`, and `pubsub.SubscribeAsync(Func)`. Typed `ReceiveAsync` / `SubscribeAsync` remain the simple path. - Listener startup returns handles: `StartConsumerAsync` returns `IMessageConsumer`; `SubscribeAsync` returns `IMessageSubscription`. Same-key duplicate registrations are idempotent only for the same handler/options; conflicting registrations throw. Use `RunConsumerAsync` or `RunSubscriptionAsync` only for blocking lifetime loops. - Received-message settlement uses explicit verbs only: `CompleteAsync`, `AbandonAsync`, `DeadLetterAsync`, `RenewLockAsync`, and `ReportProgressAsync`. Unsupported capabilities should throw clearly instead of silently downgrading. - New durable job runtime roles are separated: `IJobClient` submits and returns `JobHandle`, `IJobMonitor` queries state, `IJobRuntimeStore` persists runtime state, and `IJobWorker` claims and executes queued jobs. Job types persist stable registry names via `IJobTypeRegistry` / `.Jobs.Register(name)`, not assembly-qualified names. diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index 7bd16a2b6..dd7e2f784 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -14,12 +14,12 @@ services.AddFoundatio() .MapQueue("orders") .MapTopic("order-events", typeof(IOrderEvent)) .UseSubscriptionIdentity("billing-service")) - .UseInMemory() + .UseInMemory() .Jobs.UseInMemoryRuntime() .Jobs.Register("search.rebuild"); ``` -Application code should depend on `Foundatio.Messaging.IQueue`, `IPubSub`, `IJobClient`, `IJobMonitor`, and `IJobWorker` instead of constructing `InMemoryMessageTransport`, `MessageQueue`, `PubSub`, or `JobClient` directly. +Application code should depend on `Foundatio.Messaging.IQueue`, `IPubSub`, `IJobClient`, `IJobMonitor`, and `IJobWorker` instead of constructing `InMemoryMessageTransport`, `MessageQueue`, `PubSub`, or `JobClient` directly. Deployment or admin code can depend on `IMessageTopology` to inspect, create, or validate the destinations implied by routing configuration. ## Queue @@ -43,7 +43,7 @@ IReceivedMessage? received = await queue.ReceiveAsync(HandleAsync); ``` -Topic routing and subscription identity are separate. The topic answers where the event is published. The subscription answers which logical service or consumer group receives it: +Topic routing and subscription identity are separate. The topic answers where the event is published. The subscription answers which logical service or consumer group receives it. Multiple instances using the same subscription compete on the same transport subscription; different subscriptions on the same topic receive fan-out copies: ```csharp services.AddFoundatio() @@ -98,14 +98,14 @@ await using IMessageSubscription subscription = await pubsub.SubscribeAsync)` supports heterogeneous event batches and groups sends by resolved topic. +`PubSubMessageOptions` mirrors queue send options where concepts overlap: priority, delay, TTL, correlation id, deduplication id, headers, and topic override. `PubSubSubscriptionOptions.Key` is only the local duplicate-listener key; `Subscription` is the transport consumer group identity. `PublishBatchAsync(IEnumerable)` supports heterogeneous event batches and groups sends by resolved topic. ## Routing Default route precedence is: ```text -operation override > explicit route map > interface/base-type map > MessageRouteAttribute > configured convention +operation override > explicit route map > interface/base-type map > MessageRouteAttribute > configured default/convention ``` `IMessageRouter` is shared by queues and pub/sub. Configure routes once with `MessageRoutingOptionsBuilder`: @@ -113,8 +113,8 @@ operation override > explicit route map > interface/base-type map > MessageRoute ```csharp services.AddFoundatio() .Messaging.ConfigureRouting(r => r - .UseGlobalQueue("all-work") - .UseGlobalTopic("all-events") + .UseDefaultQueue("all-work") + .UseDefaultTopic("all-events") .MapQueue("orders") .MapQueue("orders", typeof(OrderSubmitted), typeof(OrderCancelled)) .MapQueue("order-work", typeof(IOrderMessage)) @@ -124,6 +124,15 @@ services.AddFoundatio() `QueueMessageOptions.Destination`, `QueueReceiveOptions.Source`, `PubSubMessageOptions.Topic`, and `PubSubSubscriptionOptions.Topic`/`Subscription` are final escape hatches for one operation. Attribute routing remains available for type-local defaults, but central routing should be the normal path. +Routing configuration is also the topology declaration source. `UseDefaultQueue`, `UseDefaultTopic`, `MapQueue`, and `MapTopic` declare the queue destinations or topics they name; `UseSubscriptionIdentity` declares subscriptions for configured topics. Operation-level overrides are intentionally not part of startup topology because they are exceptional one-off routes. + +```csharp +IMessageTopology topology = provider.GetRequiredService(); +IReadOnlyList declarations = topology.GetDeclarations(); +await topology.EnsureAsync(); // deploy/admin process with create permissions +await topology.ValidateAsync(); // app startup check without creating destinations +``` + ## Delivery Settlement Received messages use explicit settlement verbs for both queue and pub/sub: @@ -183,7 +192,7 @@ await pubsub.PublishAsync(new OrderSubmitted(id)); await using var subscription = await pubsub.SubscribeAsync(HandleAsync); ``` -For per-type routing, register each type. For grouped routing, map an interface or base type. For global routing, set one queue destination or topic for all messages. Operation-level overrides should be reserved for exceptional paths such as replays or priority lanes. +For per-type routing, register each type. For grouped routing, map an interface or base type. For default/global-style routing, set one default queue destination or topic for otherwise unmapped messages. Operation-level overrides should be reserved for exceptional paths such as replays or priority lanes. ## Rollout Notes diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index d016b43c0..1768edc3e 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -244,6 +244,8 @@ public class MessagingBuilder : IFoundatioBuilder { private readonly FoundatioBuilder _builder; private readonly IServiceCollection _services; + private bool _routingServicesRegistered; + private bool _topologyServicesRegistered; internal MessagingBuilder(IFoundatioBuilder builder) { @@ -274,13 +276,8 @@ public MessagingBuilder ConfigureRouting(Action co { ArgumentNullException.ThrowIfNull(configure); - _services.ReplaceSingleton(_ => - { - var options = new MessageRoutingOptions(); - configure(new MessageRoutingOptionsBuilder(options)); - return new DefaultMessageRouter(options); - }); - + _services.AddSingleton>(configure); + RegisterRoutingServices(); return this; } @@ -304,8 +301,8 @@ public FoundatioBuilder UseInMemory(Builder transport); - RegisterMessageClients(); + ArgumentNullException.ThrowIfNull(transport); + RegisterMessagingRuntime(_ => transport); return _builder; } @@ -318,11 +315,44 @@ public FoundatioBuilder UseTransport(Func f private void RegisterMessagingRuntime(Func factory) { _services.ReplaceSingleton(factory); + RegisterMessageTopology(); RegisterMessageClients(); } + private void RegisterRoutingServices() + { + if (_routingServicesRegistered) + return; + + _routingServicesRegistered = true; + _services.ReplaceSingleton(sp => + { + var options = new MessageRoutingOptions(); + var builder = new MessageRoutingOptionsBuilder(options); + foreach (var configure in sp.GetServices>()) + configure(builder); + + return options; + }); + _services.ReplaceSingleton(sp => new DefaultMessageRouter(sp.GetRequiredService())); + } + + private void RegisterMessageTopology() + { + RegisterRoutingServices(); + + if (_topologyServicesRegistered) + return; + + _topologyServicesRegistered = true; + _services.ReplaceSingleton(sp => new MessageTopology( + sp.GetRequiredService(), + sp.GetRequiredService())); + } + private void RegisterMessageClients() { + RegisterRoutingServices(); _services.ReplaceSingleton(sp => new MessageQueue(sp.GetRequiredService(), CreateQueueOptions(sp))); _services.ReplaceSingleton(sp => new PubSub(sp.GetRequiredService(), CreatePubSubOptions(sp))); } diff --git a/src/Foundatio/Messaging/MessageRouting.cs b/src/Foundatio/Messaging/MessageRouting.cs index f40f33a5a..fe04a1b38 100644 --- a/src/Foundatio/Messaging/MessageRouting.cs +++ b/src/Foundatio/Messaging/MessageRouting.cs @@ -42,13 +42,38 @@ public sealed record MessageRouteMap public sealed class MessageRoutingOptions { internal List RouteMaps { get; } = []; + internal List TopologyDeclarations { get; } = []; - public string? GlobalQueueDestination { get; set; } - public string? GlobalPubSubTopic { get; set; } + public string? DefaultQueueDestination { get; set; } + public string? DefaultPubSubTopic { get; set; } public string? SubscriptionIdentity { get; set; } public string? ServiceIdentity { get; set; } public Func? Convention { get; set; } public Func? MessageTypeResolver { get; set; } + + public IReadOnlyList GetTopologyDeclarations() + { + return TopologyDeclarations.ToArray(); + } + + internal void Declare(DestinationDeclaration declaration) + { + ArgumentNullException.ThrowIfNull(declaration); + ArgumentException.ThrowIfNullOrEmpty(declaration.Name); + + bool exists = TopologyDeclarations.Any(d => + String.Equals(d.Name, declaration.Name, StringComparison.Ordinal) + && d.Role == declaration.Role + && String.Equals(d.Source, declaration.Source, StringComparison.Ordinal)); + + if (!exists) + TopologyDeclarations.Add(declaration); + } + + internal void RemoveDeclarations(Predicate match) + { + TopologyDeclarations.RemoveAll(match); + } } public sealed class MessageRoutingOptionsBuilder @@ -65,17 +90,19 @@ internal MessageRoutingOptionsBuilder(MessageRoutingOptions options) _options = options; } - public MessageRoutingOptionsBuilder UseGlobalQueue(string destination) + public MessageRoutingOptionsBuilder UseDefaultQueue(string destination) { ArgumentException.ThrowIfNullOrEmpty(destination); - _options.GlobalQueueDestination = destination; + _options.DefaultQueueDestination = destination; + DeclareQueue(destination); return this; } - public MessageRoutingOptionsBuilder UseGlobalTopic(string topic) + public MessageRoutingOptionsBuilder UseDefaultTopic(string topic) { ArgumentException.ThrowIfNullOrEmpty(topic); - _options.GlobalPubSubTopic = topic; + _options.DefaultPubSubTopic = topic; + DeclareTopic(topic); return this; } @@ -113,6 +140,7 @@ public MessageRoutingOptionsBuilder UseSubscriptionIdentity(string subscription) { ArgumentException.ThrowIfNullOrEmpty(subscription); _options.SubscriptionIdentity = subscription; + RebuildSubscriptionDeclarations(); return this; } @@ -120,6 +148,7 @@ public MessageRoutingOptionsBuilder UseServiceIdentity(string serviceIdentity) { ArgumentException.ThrowIfNullOrEmpty(serviceIdentity); _options.ServiceIdentity = serviceIdentity; + RebuildSubscriptionDeclarations(); return this; } @@ -159,8 +188,54 @@ private MessageRoutingOptionsBuilder Map(MessageRouteRole role, string route, pa }); } + if (role == MessageRouteRole.QueueDestination) + DeclareQueue(route); + else + DeclareTopic(route); + return this; } + + private void DeclareQueue(string destination) + { + _options.Declare(new DestinationDeclaration { Name = destination, Role = DestinationRole.Queue }); + } + + private void DeclareTopic(string topic) + { + _options.Declare(new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }); + DeclareSubscription(topic); + } + + private void RebuildSubscriptionDeclarations() + { + _options.RemoveDeclarations(d => d.Role == DestinationRole.Subscription); + + if (!String.IsNullOrEmpty(_options.DefaultPubSubTopic)) + DeclareSubscription(_options.DefaultPubSubTopic); + + foreach (string topic in _options.RouteMaps + .Where(m => m.Role == MessageRouteRole.PubSubTopic) + .Select(m => m.Route) + .Distinct(StringComparer.Ordinal)) + { + DeclareSubscription(topic); + } + } + + private void DeclareSubscription(string topic) + { + string? subscription = _options.SubscriptionIdentity ?? _options.ServiceIdentity; + if (String.IsNullOrEmpty(subscription)) + return; + + DeclareSubscription(topic, subscription); + } + + private void DeclareSubscription(string topic, string subscription) + { + _options.Declare(new DestinationDeclaration { Name = subscription, Role = DestinationRole.Subscription, Source = topic }); + } } public sealed class DefaultMessageRouter : IMessageRouter @@ -198,12 +273,12 @@ public string ResolveRoute(MessageRouteContext context) if (!String.IsNullOrEmpty(attributedRoute)) return attributedRoute; - string? configuredConvention = context.Role == MessageRouteRole.QueueDestination - ? _options.GlobalQueueDestination - : _options.GlobalPubSubTopic; + string? configuredDefault = context.Role == MessageRouteRole.QueueDestination + ? _options.DefaultQueueDestination + : _options.DefaultPubSubTopic; - if (!String.IsNullOrEmpty(configuredConvention)) - return configuredConvention; + if (!String.IsNullOrEmpty(configuredDefault)) + return configuredDefault; if (_options.Convention is not null) { diff --git a/src/Foundatio/Messaging/MessageTopology.cs b/src/Foundatio/Messaging/MessageTopology.cs new file mode 100644 index 000000000..1d3af8b34 --- /dev/null +++ b/src/Foundatio/Messaging/MessageTopology.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; + +namespace Foundatio.Messaging; + +public interface IMessageTopology +{ + IReadOnlyList GetDeclarations(); + Task EnsureAsync(CancellationToken cancellationToken = default); + Task ValidateAsync(CancellationToken cancellationToken = default); +} + +public sealed class MessageTopology : IMessageTopology +{ + private readonly IMessageTransport _transport; + private readonly MessageRoutingOptions _options; + + public MessageTopology(IMessageTransport transport, MessageRoutingOptions options) + { + _transport = transport ?? throw new ArgumentNullException(nameof(transport)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public IReadOnlyList GetDeclarations() + { + return _options.GetTopologyDeclarations(); + } + + public async Task EnsureAsync(CancellationToken cancellationToken = default) + { + var declarations = GetDeclarations(); + if (declarations.Count == 0) + return; + + if (_transport is not ISupportsProvisioning provisioning) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support topology provisioning."); + + await provisioning.EnsureAsync(declarations, cancellationToken).AnyContext(); + } + + public async Task ValidateAsync(CancellationToken cancellationToken = default) + { + var declarations = GetDeclarations(); + if (declarations.Count == 0) + return; + + if (_transport is not ISupportsProvisioning provisioning) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support topology validation."); + + var missing = new List(); + foreach (var declaration in declarations) + { + if (!await provisioning.ExistsAsync(declaration.Name, cancellationToken).AnyContext()) + missing.Add(declaration); + } + + if (missing.Count > 0) + throw new InvalidOperationException($"Message topology is missing: {String.Join(", ", missing.Select(FormatDeclaration))}."); + } + + private static string FormatDeclaration(DestinationDeclaration declaration) + { + return String.IsNullOrEmpty(declaration.Source) + ? $"{declaration.Role} '{declaration.Name}'" + : $"{declaration.Role} '{declaration.Name}' from '{declaration.Source}'"; + } +} diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index e840e5584..997614ec2 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -49,6 +50,50 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() Assert.Equal(1, secondStats.Completed); } + [Fact] + public async Task SubscribeAsync_WithSameSubscriptionAndDifferentKeys_CompetesOnTransportSubscriptionAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var received = new AsyncCountdownEvent(2); + var deliveriesByMessageId = new ConcurrentDictionary(StringComparer.Ordinal); + + Func, CancellationToken, Task> handler = (message, _) => + { + deliveriesByMessageId.AddOrUpdate(message.Id, 1, (_, count) => count + 1); + received.Signal(); + return Task.CompletedTask; + }; + + await using var first = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions + { + Subscription = "billing-service", + Key = "node-a" + }, cts.Token); + await using var second = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions + { + Subscription = "billing-service", + Key = "node-b" + }, cts.Token); + + await pubSub.PublishBatchAsync([ + new PreviewEvent { Data = "one" }, + new PreviewEvent { Data = "two" } + ], cancellationToken: cancellationToken); + + await received.WaitAsync(TimeSpan.FromSeconds(2)); + await WaitForCompletedAsync(transport, "billing-service", 2, cancellationToken); + + Assert.Equal(first.Topic, second.Topic); + Assert.Equal(first.Subscription, second.Subscription); + Assert.NotEqual(first.Key, second.Key); + Assert.Equal(2, deliveriesByMessageId.Count); + Assert.All(deliveriesByMessageId.Values, count => Assert.Equal(1, count)); + } + [Fact] public async Task PublishBatchAsync_DeliversAllMessagesAsync() { @@ -237,6 +282,22 @@ await pubSub.PublishBatchAsync(new object[] } + private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, string destination, long expected, CancellationToken cancellationToken) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(2); + while (DateTimeOffset.UtcNow < deadline) + { + var stats = await transport.GetStatsAsync(destination, cancellationToken); + if (stats.Completed == expected) + return; + + await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken); + } + + var finalStats = await transport.GetStatsAsync(destination, cancellationToken); + Assert.Equal(expected, finalStats.Completed); + } + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { var serviceProvider = new ServiceCollection().BuildServiceProvider(); diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index c12327238..14962373a 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -287,11 +287,51 @@ public async Task AddFoundatio_WithInMemoryMessagingAndJobs_RegistersAppFacingSe Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); } + [Fact] + public async Task AddFoundatio_WithRouting_RegistersRouterAndTopologyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var services = new ServiceCollection(); + + services.AddFoundatio() + .Messaging.ConfigureRouting(r => r + .UseDefaultQueue("all-work") + .MapTopic("grouped-events", typeof(IGroupedWorkItem)) + .UseServiceIdentity("billing-service")) + .UseInMemory(); + + await using var provider = services.BuildServiceProvider(); + + var router = provider.GetRequiredService(); + Assert.Equal("all-work", router.ResolveRoute(new MessageRouteContext + { + MessageType = typeof(PreviewWorkItem), + Role = MessageRouteRole.QueueDestination + })); + Assert.Equal("grouped-events", router.ResolveRoute(new MessageRouteContext + { + MessageType = typeof(OtherWorkItem), + Role = MessageRouteRole.PubSubTopic + })); + + var topology = provider.GetRequiredService(); + var declarations = topology.GetDeclarations(); + Assert.Contains(declarations, d => d.Role == DestinationRole.Queue && d.Name == "all-work"); + Assert.Contains(declarations, d => d.Role == DestinationRole.Topic && d.Name == "grouped-events"); + Assert.Contains(declarations, d => d.Role == DestinationRole.Subscription && d.Name == "billing-service" && d.Source == "grouped-events"); + + await Assert.ThrowsAsync(async () => await topology.ValidateAsync(cancellationToken)); + await topology.EnsureAsync(cancellationToken); + await topology.ValidateAsync(cancellationToken); + } + [Fact] public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync() { @@ -361,11 +401,11 @@ await queue.EnqueueBatchAsync(new object[] } [Fact] - public async Task ReceiveAsync_WithGlobalQueueRoute_ReturnsRawMessageAsync() + public async Task ReceiveAsync_WithDefaultQueueRoute_ReturnsRawMessageAsync() { var cancellationToken = TestContext.Current.CancellationToken; var routing = new MessageRoutingOptionsBuilder() - .UseGlobalQueue("all-work") + .UseDefaultQueue("all-work") .Build(); await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); From 6e9abdec732771b96181b11e9e48757e69569d40 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 28 Jun 2026 00:01:12 -0500 Subject: [PATCH 11/57] fix: address messaging/jobs design-review feedback Durable jobs + CRON: - Add hosted JobRuntimeService that pumps occurrence materialization, due-dispatch, recovery, and queued-job execution (runtime previously never ran end-to-end). Register via AddJobRuntimeService(). - Enforce lease ownership in TryTransitionAsync (expectedNodeId) so a stale worker cannot overwrite the reclaiming node's terminal state. - Renew the claim on a heartbeat during execution and cancel the run when the lease is lost (RenewClaimAsync was dead code). - Strong, process-unique node identity (machine:pid:token). - Capped exponential CRON retry backoff (per-definition override). - Replace the hand-rolled cron parser with the vendored Cronos (moved into core Foundatio.Cronos); materialize every missed occurrence in the misfire window, not just the latest. Messaging: - Resilient consumer/subscription loops: a poison message or transient receive error no longer silently kills the consumer. - Fix in-memory push path double-settle (tolerate already-settled receipts in the safety-net abandon). - Honor visibility timeouts in the in-memory transport (reaper redelivers unsettled messages) so its advertised at-least-once guarantee is real. - Log handler exceptions instead of swallowing them. - Drop the misleading write-only content-type header. - Add ReceiveDeadLetteredAsync so poison payloads are inspectable. Conformance harness: - Replace silent capability skips with Assert.Skip. - Gate ordering assertions on the declared OrderingGuarantee. - Add visibility-timeout, competing-consumer, and DLQ-read scenarios. Tests cover lease-stomp rejection, manual ack, poison survival, multi-occurrence CRON, and the hosted runtime running a queued job. Co-Authored-By: Claude Opus 4.8 --- .../Jobs/JobHostExtensions.cs | 18 + .../Jobs/JobRuntimeService.cs | 93 ++++++ .../Jobs/ScheduledJobInstance.cs | 2 +- .../MessageTransportConformanceTests.cs | 136 +++++++- .../Cronos/CalendarHelper.cs | 2 +- .../Cronos/CronExpression.cs | 2 +- .../Cronos/CronExpressionFlag.cs | 2 +- .../Cronos/CronField.cs | 2 +- .../Cronos/CronFormat.cs | 2 +- .../Cronos/CronFormatException.cs | 2 +- .../Cronos/TimeZoneHelper.cs | 2 +- src/Foundatio/Foundatio.csproj | 4 + src/Foundatio/FoundatioServicesExtensions.cs | 6 +- src/Foundatio/Jobs/JobRuntime.cs | 68 +++- src/Foundatio/Jobs/JobScheduler.cs | 308 +++++------------- .../Messaging/InMemoryMessageTransport.cs | 86 ++++- src/Foundatio/Messaging/MessageQueue.cs | 58 +++- src/Foundatio/Messaging/MessageTransport.cs | 4 + src/Foundatio/Messaging/PubSub.cs | 56 +++- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 26 ++ .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 62 ++++ .../InMemoryMessageTransportTests.cs | 18 + .../Queue/MessageQueueTests.cs | 54 +++ 23 files changed, 749 insertions(+), 264 deletions(-) create mode 100644 src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CalendarHelper.cs (99%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CronExpression.cs (99%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CronExpressionFlag.cs (96%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CronField.cs (98%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CronFormat.cs (97%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/CronFormatException.cs (97%) rename src/{Foundatio.Extensions.Hosting => Foundatio}/Cronos/TimeZoneHelper.cs (99%) diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index 1a00ac6c0..3de0ac8dc 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -194,6 +194,24 @@ public static IServiceCollection AddJobScheduler(this IServiceCollection service return services; } + /// + /// Registers the hosted pump that drives the durable job runtime (): + /// materializing CRON occurrences, dispatching delayed/scheduled work, recovering stale occurrences, and running + /// jobs submitted via . Register the runtime store and job services first + /// (e.g. services.AddFoundatio().Jobs.UseInMemoryRuntime()). + /// + public static IServiceCollection AddJobRuntimeService(this IServiceCollection services, Action? configure = null) + { + var options = new JobRuntimeServiceOptions(); + configure?.Invoke(options); + services.AddSingleton(options); + + if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimeService))) + services.AddSingleton(); + + return services; + } + public static IServiceCollection AddJobLifetimeService(this IServiceCollection services) { services.AddSingleton(); diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs new file mode 100644 index 000000000..328faacf0 --- /dev/null +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs @@ -0,0 +1,93 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Utility; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Extensions.Hosting.Jobs; + +/// +/// Options controlling the cadence and batch size of . +/// +public class JobRuntimeServiceOptions +{ + /// + /// How often the runtime pump materializes CRON occurrences, dispatches due work, and runs queued jobs. + /// Defaults to one second so sub-minute CRON schedules and short delays are honored. + /// + public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Maximum number of due dispatches and queued jobs claimed per pump iteration. + /// + public int BatchSize { get; set; } = 100; +} + +/// +/// Drives the durable job runtime introduced by . Without this hosted service nothing +/// materializes CRON occurrences, dispatches delayed/scheduled work, recovers stale (lease-expired) occurrences, or +/// runs jobs submitted through — the runtime store would accumulate work that never executes. +/// +public class JobRuntimeService : BackgroundService +{ + private readonly JobScheduleProcessor _processor; + private readonly IJobWorker _worker; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly JobRuntimeServiceOptions _options; + + public JobRuntimeService(JobScheduleProcessor processor, IJobWorker worker, TimeProvider? timeProvider = null, ILoggerFactory? loggerFactory = null, JobRuntimeServiceOptions? options = null) + { + _processor = processor ?? throw new ArgumentNullException(nameof(processor)); + _worker = worker ?? throw new ArgumentNullException(nameof(worker)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _options = options ?? new JobRuntimeServiceOptions(); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var now = _timeProvider.GetUtcNow(); + + // Materialize CRON occurrences due within the misfire window (deduped, idempotent). + await _processor.EnqueueDueOccurrencesAsync(now, stoppingToken).AnyContext(); + + // Claim and run due dispatches: CRON occurrences plus delayed queue/pub-sub messages. This also + // recovers occurrences whose processing lease expired (crash mid-run) and applies retry/dead-letter. + await _processor.RunDueOccurrencesAsync(now, _options.BatchSize, lease: null, stoppingToken).AnyContext(); + + // Run jobs submitted via IJobClient that are sitting in the Queued state. + await _worker.RunQueuedAsync(_options.BatchSize, stoppingToken).AnyContext(); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error pumping job runtime: {Message}", ex.Message); + } + + try + { + await _timeProvider.Delay(_options.PollInterval, stoppingToken).AnyContext(); + } + catch (OperationCanceledException) + { + break; + } + } + + _logger.LogInformation("Job runtime pump stopped"); + } +} diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs index 8fe31d801..7ba51027a 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Extensions.Hosting.Cronos; +using Foundatio.Cronos; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 7726b4844..393267a79 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -27,7 +28,10 @@ public virtual async Task CanSendAndReceiveBatchAsync() { var transport = CreateTransport(); if (transport is not ISupportsPull pull) + { + Assert.Skip("Transport does not support pull receive (ISupportsPull)."); return; + } try { @@ -49,9 +53,19 @@ public virtual async Task CanSendAndReceiveBatchAsync() }, TestCancellationToken); Assert.Equal(2, entries.Count); - Assert.Equal("one", ReadBody(entries[0])); - Assert.Equal("two", ReadBody(entries[1])); - Assert.Equal("acme", entries[0].Headers["tenant"]); + var bodies = entries.Select(ReadBody).ToList(); + Assert.Contains("one", bodies); + Assert.Contains("two", bodies); + + // Only assert positional FIFO order when the transport actually guarantees ordering; a best-effort + // (OrderingGuarantee.None) transport may legitimately deliver out of order. + if (transport is not ITransportInfo { Ordering: OrderingGuarantee.None }) + { + Assert.Equal("one", ReadBody(entries[0])); + Assert.Equal("two", ReadBody(entries[1])); + } + + Assert.All(entries, e => Assert.Equal("acme", e.Headers["tenant"])); Assert.Equal(1, entries[0].DeliveryCount); await transport.CompleteAsync(entries[0], TestCancellationToken); @@ -75,7 +89,10 @@ public virtual async Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsy { var transport = CreateTransport(); if (transport is not ISupportsPull pull) + { + Assert.Skip("Transport does not support pull receive (ISupportsPull)."); return; + } try { @@ -104,7 +121,10 @@ public virtual async Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredE { var transport = CreateTransport(); if (transport is not ISupportsPull pull) + { + Assert.Skip("Transport does not support pull receive (ISupportsPull)."); return; + } try { @@ -127,7 +147,10 @@ public virtual async Task SubscribeAsync_DeliversPushMessagesAsync() { var transport = CreateTransport(); if (transport is not ISupportsPush push) + { + Assert.Skip("Transport does not support push delivery (ISupportsPush)."); return; + } try { @@ -157,7 +180,10 @@ public virtual async Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() { var transport = CreateTransport(); if (transport is not ISupportsPull pull || transport is not ISupportsProvisioning) + { + Assert.Skip("Transport does not support pull receive and provisioning (ISupportsPull + ISupportsProvisioning)."); return; + } try { @@ -187,7 +213,10 @@ public virtual async Task ReceiveAsync_RespectsPriorityAsync() { var transport = CreateTransport(); if (transport is not ISupportsPull pull || transport is not ISupportsPriority) + { + Assert.Skip("Transport does not support pull receive with priority (ISupportsPull + ISupportsPriority)."); return; + } try { @@ -220,7 +249,10 @@ public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() { var transport = CreateTransport(); if (transport is not ISupportsPull pull || transport is not ISupportsDelayedDelivery) + { + Assert.Skip("Transport does not support pull receive with delayed delivery (ISupportsPull + ISupportsDelayedDelivery)."); return; + } try { @@ -247,7 +279,10 @@ public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() { var transport = CreateTransport(); if (transport is not ISupportsPull pull || transport is not ISupportsDeadLetter || transport is not ISupportsStats stats) + { + Assert.Skip("Transport does not support pull receive with dead-letter and stats (ISupportsPull + ISupportsDeadLetter + ISupportsStats)."); return; + } try { @@ -271,7 +306,10 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy { var transport = CreateTransport(); if (transport is not ISupportsPull pull || transport is not ISupportsExpiration || transport is not ISupportsStats stats) + { + Assert.Skip("Transport does not support pull receive with expiration and stats (ISupportsPull + ISupportsExpiration + ISupportsStats)."); return; + } try { @@ -298,6 +336,98 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy await CleanupTransportIfNotNullAsync(transport); } } + public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsVisibilityTimeout visibility) + { + Assert.Skip("Transport does not support visibility timeout (ISupportsVisibilityTimeout)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "visibility", Role = DestinationRole.Queue }); + await transport.SendAsync("visibility", [CreateMessage("lease")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(250), TestCancellationToken)); + Assert.Equal(1, first.DeliveryCount); + + // Still within the visibility window: a competing receive must not see the in-flight message. + var hidden = await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TimeSpan.FromMilliseconds(250), TestCancellationToken); + Assert.Empty(hidden); + + // After the visibility window lapses without settlement, the message must be redelivered (at-least-once). + await Task.Delay(TimeSpan.FromMilliseconds(400), TestCancellationToken); + var second = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(250), TestCancellationToken)); + Assert.Equal(first.Id, second.Id); + Assert.Equal(2, second.DeliveryCount); + + await transport.CompleteAsync(second, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull) + { + Assert.Skip("Transport does not support pull receive (ISupportsPull)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "competing", Role = DestinationRole.Queue }); + await transport.SendAsync("competing", [CreateMessage("once")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await pull.ReceiveAsync("competing", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + + // A competing consumer must not receive the same message while it is in flight. + var second = await pull.ReceiveAsync("competing", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); + Assert.Empty(second); + + await transport.CompleteAsync(first, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsDeadLetter deadLetter) + { + Assert.Skip("Transport does not support pull receive and dead-letter (ISupportsPull + ISupportsDeadLetter)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "dlq-read", Role = DestinationRole.Queue }); + await transport.SendAsync("dlq-read", [CreateMessage("poison", ("tenant", "acme"))], new TransportSendOptions(), TestCancellationToken); + + var entry = Assert.Single(await pull.ReceiveAsync("dlq-read", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + await deadLetter.DeadLetterAsync(entry, "bad-payload", TestCancellationToken); + + // The raw (un-deserialized) payload and the dead-letter reason must be inspectable. + var deadLettered = Assert.Single(await deadLetter.ReceiveDeadLetteredAsync("dlq-read", new ReceiveRequest { MaxMessages = 10 }, TestCancellationToken)); + Assert.Equal("poison", ReadBody(deadLettered)); + Assert.Equal("acme", deadLettered.Headers["tenant"]); + Assert.Equal("bad-payload", deadLettered.Headers[KnownHeaders.DeadLetterReason]); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + private async ValueTask CleanupTransportIfNotNullAsync(IMessageTransport? transport) { if (transport is not null) diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CalendarHelper.cs b/src/Foundatio/Cronos/CalendarHelper.cs similarity index 99% rename from src/Foundatio.Extensions.Hosting/Cronos/CalendarHelper.cs rename to src/Foundatio/Cronos/CalendarHelper.cs index c447b746b..57630deca 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CalendarHelper.cs +++ b/src/Foundatio/Cronos/CalendarHelper.cs @@ -5,7 +5,7 @@ using System; using System.Runtime.CompilerServices; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; internal static class CalendarHelper { diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CronExpression.cs b/src/Foundatio/Cronos/CronExpression.cs similarity index 99% rename from src/Foundatio.Extensions.Hosting/Cronos/CronExpression.cs rename to src/Foundatio/Cronos/CronExpression.cs index 9068ed36d..0518952e3 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CronExpression.cs +++ b/src/Foundatio/Cronos/CronExpression.cs @@ -26,7 +26,7 @@ using System.Runtime.CompilerServices; using System.Text; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; /// /// Provides a parser and scheduler for cron expressions. diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CronExpressionFlag.cs b/src/Foundatio/Cronos/CronExpressionFlag.cs similarity index 96% rename from src/Foundatio.Extensions.Hosting/Cronos/CronExpressionFlag.cs rename to src/Foundatio/Cronos/CronExpressionFlag.cs index df7b2fc7e..74b7bba04 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CronExpressionFlag.cs +++ b/src/Foundatio/Cronos/CronExpressionFlag.cs @@ -22,7 +22,7 @@ using System; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; [Flags] internal enum CronExpressionFlag : byte diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CronField.cs b/src/Foundatio/Cronos/CronField.cs similarity index 98% rename from src/Foundatio.Extensions.Hosting/Cronos/CronField.cs rename to src/Foundatio/Cronos/CronField.cs index 44c9fe8ef..5065f0a01 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CronField.cs +++ b/src/Foundatio/Cronos/CronField.cs @@ -20,7 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; internal sealed class CronField { diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CronFormat.cs b/src/Foundatio/Cronos/CronFormat.cs similarity index 97% rename from src/Foundatio.Extensions.Hosting/Cronos/CronFormat.cs rename to src/Foundatio/Cronos/CronFormat.cs index 2e762e5e6..776e2684a 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CronFormat.cs +++ b/src/Foundatio/Cronos/CronFormat.cs @@ -22,7 +22,7 @@ using System; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; /// /// Defines the cron format options that customize string parsing for . diff --git a/src/Foundatio.Extensions.Hosting/Cronos/CronFormatException.cs b/src/Foundatio/Cronos/CronFormatException.cs similarity index 97% rename from src/Foundatio.Extensions.Hosting/Cronos/CronFormatException.cs rename to src/Foundatio/Cronos/CronFormatException.cs index 7df867527..fe13770a8 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/CronFormatException.cs +++ b/src/Foundatio/Cronos/CronFormatException.cs @@ -22,7 +22,7 @@ using System; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; /// /// Represents an exception that's thrown, when invalid Cron expression is given. diff --git a/src/Foundatio.Extensions.Hosting/Cronos/TimeZoneHelper.cs b/src/Foundatio/Cronos/TimeZoneHelper.cs similarity index 99% rename from src/Foundatio.Extensions.Hosting/Cronos/TimeZoneHelper.cs rename to src/Foundatio/Cronos/TimeZoneHelper.cs index 587cfa248..4eaef9ef0 100644 --- a/src/Foundatio.Extensions.Hosting/Cronos/TimeZoneHelper.cs +++ b/src/Foundatio/Cronos/TimeZoneHelper.cs @@ -22,7 +22,7 @@ using System; -namespace Foundatio.Extensions.Hosting.Cronos; +namespace Foundatio.Cronos; internal static class TimeZoneHelper { diff --git a/src/Foundatio/Foundatio.csproj b/src/Foundatio/Foundatio.csproj index 04a1ec8d7..b1ba08469 100644 --- a/src/Foundatio/Foundatio.csproj +++ b/src/Foundatio/Foundatio.csproj @@ -1,4 +1,8 @@ + + + true + diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 1768edc3e..85c5893be 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -364,7 +364,8 @@ private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), - TimeProvider = serviceProvider.GetService() ?? TimeProvider.System + TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, + LoggerFactory = serviceProvider.GetService() }; } @@ -375,7 +376,8 @@ private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvide Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), - TimeProvider = serviceProvider.GetService() ?? TimeProvider.System + TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, + LoggerFactory = serviceProvider.GetService() }; } } diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 7f4423baf..a3100e5cc 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -214,7 +214,10 @@ public interface IJobWorker public interface IJobRuntimeStore : IJobMonitor { Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default); - Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default); + // When expectedNodeId is non-null, the transition only succeeds if the job is currently owned by that node. + // Worker terminal transitions pass their node id so a stale worker whose lease was reclaimed cannot overwrite + // the new owner's state. + Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, string? expectedNodeId = null, CancellationToken cancellationToken = default); Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken cancellationToken = default); @@ -280,7 +283,7 @@ public Task> QueryAsync(JobQuery query, CancellationToke .ToArray()); } - public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default) + public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, string? expectedNodeId = null, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -289,6 +292,9 @@ public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, Job if (!_jobs.TryGetValue(jobId, out var current) || current.Status != expectedStatus) return Task.FromResult(false); + if (expectedNodeId is not null && !String.Equals(current.NodeId, expectedNodeId, StringComparison.Ordinal)) + return Task.FromResult(false); + _jobs[jobId] = ApplyPatch(current, patch) with { Status = newStatus, @@ -549,6 +555,25 @@ public Task RequestCancellationAsync(string jobId, CancellationToken cance } } +/// +/// Resolves a stable, process-unique node identity used for job claims and per-node scheduling. +/// Honors the FOUNDATIO_NODE_ID environment variable when set; otherwise combines machine name, +/// process id, and a process-lifetime token so co-located worker processes do not collapse to one identity. +/// +internal static class NodeIdentity +{ + public static string Current { get; } = Resolve(); + + private static string Resolve() + { + string? configured = Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID"); + if (!String.IsNullOrEmpty(configured)) + return configured; + + return $"{Environment.MachineName}:{Environment.ProcessId}:{Guid.NewGuid().ToString("N")[..8]}"; + } +} + public sealed class JobWorker : IJobWorker { private static readonly TimeSpan DefaultLease = TimeSpan.FromMinutes(5); @@ -566,9 +591,7 @@ public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeP _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); _timeProvider = timeProvider ?? TimeProvider.System; _jobTypes = jobTypes ?? new JobTypeRegistry(); - _nodeId = !String.IsNullOrEmpty(nodeId) - ? nodeId - : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _lease = lease ?? DefaultLease; } @@ -610,13 +633,14 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc StartedUtc = now, LeaseExpiresUtc = now.Add(_lease), AttemptDelta = 1 - }, cancellationToken).ConfigureAwait(false)) + }, cancellationToken: cancellationToken).ConfigureAwait(false)) { return false; } using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); using var cancellationWatcher = WatchCancellation(state.JobId, linkedCancellationTokenSource); + using var leaseRenewer = RenewLeasePeriodically(state.JobId, linkedCancellationTokenSource); try { @@ -633,7 +657,7 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc CompletedUtc = completedAt, ClearNodeId = true, ClearLeaseExpiresUtc = true - }, CancellationToken.None).ConfigureAwait(false); + }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); } else if (result.IsSuccess) { @@ -643,7 +667,7 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc ClearNodeId = true, ClearLeaseExpiresUtc = true, Progress = 100 - }, CancellationToken.None).ConfigureAwait(false); + }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); } else { @@ -653,7 +677,7 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc CompletedUtc = completedAt, ClearNodeId = true, ClearLeaseExpiresUtc = true - }, CancellationToken.None).ConfigureAwait(false); + }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); } return true; @@ -666,7 +690,7 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc CompletedUtc = _timeProvider.GetUtcNow(), ClearNodeId = true, ClearLeaseExpiresUtc = true - }, CancellationToken.None).ConfigureAwait(false); + }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); throw; } } @@ -691,6 +715,30 @@ private IDisposable WatchCancellation(string jobId, CancellationTokenSource canc return new Timer(_ => _ = PollCancellationAsync(jobId, cancellationTokenSource), null, TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50)); } + private IDisposable RenewLeasePeriodically(string jobId, CancellationTokenSource cancellationTokenSource) + { + // Renew well before the lease elapses so a slow-but-alive worker keeps ownership and is not reclaimed. + var interval = TimeSpan.FromMilliseconds(Math.Max(250, _lease.TotalMilliseconds / 3)); + return new Timer(_ => _ = RenewLeaseAsync(jobId, cancellationTokenSource), null, interval, interval); + } + + private async Task RenewLeaseAsync(string jobId, CancellationTokenSource cancellationTokenSource) + { + if (cancellationTokenSource.IsCancellationRequested) + return; + + try + { + // If renewal fails the lease was lost to another node; cancel the run so this worker stops and + // its terminal transition (guarded by expectedNodeId) cannot overwrite the new owner's state. + if (!await _store.RenewClaimAsync(jobId, _nodeId, _lease, CancellationToken.None).ConfigureAwait(false)) + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + } + } + private async Task PollCancellationAsync(string jobId, CancellationTokenSource cancellationTokenSource) { if (cancellationTokenSource.IsCancellationRequested) diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index bc9831eec..079806955 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Foundatio.Cronos; using Foundatio.Messaging; namespace Foundatio.Jobs; @@ -30,6 +31,13 @@ public sealed record ScheduledJobDefinition public OverlapPolicy Overlap { get; init; } = OverlapPolicy.SkipIfRunning; public TimeSpan? MisfireWindow { get; init; } public int MaxRetries { get; init; } = 3; + + /// + /// Computes the delay before a failed occurrence is retried, given the attempt number (1-based). + /// Defaults to capped exponential backoff when null. + /// + public Func? RetryBackoff { get; init; } + public bool Enabled { get; init; } = true; } @@ -97,9 +105,7 @@ public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJo _jobWorker = jobWorker ?? throw new ArgumentNullException(nameof(jobWorker)); _timeProvider = timeProvider ?? TimeProvider.System; _jobTypes = jobTypes ?? new JobTypeRegistry(); - _nodeId = !String.IsNullOrEmpty(nodeId) - ? nodeId - : Environment.GetEnvironmentVariable("FOUNDATIO_NODE_ID") ?? Environment.MachineName; + _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _transport = transport; } @@ -120,44 +126,63 @@ public async Task> EnqueueDueOccurrencesAs if (!definition.Enabled) continue; - var cron = CronSchedule.Parse(definition.Cron); - var scheduledForUtc = cron.GetLastOccurrence(utcNow, definition.TimeZone ?? TimeZoneInfo.Utc, definition.MisfireWindow ?? DefaultMisfireWindow); - if (scheduledForUtc is null) - continue; + var cron = ParseCron(definition.Cron); + var timeZone = definition.TimeZone ?? TimeZoneInfo.Utc; + var window = definition.MisfireWindow ?? DefaultMisfireWindow; + if (window < TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(definition), window, "MisfireWindow must be greater than or equal to zero."); string scopeKey = GetScopeKey(definition); - string jobId = CreateOccurrenceId(definition.Name, scheduledForUtc.Value, scopeKey); - if (await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false) is not null) + // Materialize every occurrence that fell due within the misfire window, not just the most recent, so a + // scheduler that lagged behind the cadence does not silently drop intermediate ticks. Deterministic + // occurrence ids dedupe across overlapping windows and across nodes ticking simultaneously. + var occurrences = cron.GetOccurrences(utcNow - window, utcNow, timeZone, fromInclusive: true, toInclusive: true).ToList(); + if (occurrences.Count == 0) continue; - if (definition.Overlap == OverlapPolicy.SkipIfRunning && await HasActiveOccurrenceAsync(definition.Name, scopeKey, cancellationToken).ConfigureAwait(false)) - continue; - - await _store.CreateIfAbsentAsync(new JobState + if (definition.Overlap == OverlapPolicy.SkipIfRunning) { - JobId = jobId, - Name = definition.Name, - JobType = GetJobTypeName(definition.JobType), - Status = JobStatus.Scheduled, - CreatedUtc = utcNow, - LastUpdatedUtc = utcNow, - ScheduledForUtc = scheduledForUtc - }, cancellationToken).ConfigureAwait(false); - - var dispatch = new ScheduledDispatchState + // Don't stampede: if a prior occurrence is still pending or running, skip this tick entirely; + // otherwise collapse the window to a single (most recent) catch-up occurrence. + if (await HasActiveOccurrenceAsync(definition.Name, scopeKey, cancellationToken).ConfigureAwait(false)) + continue; + + occurrences = [occurrences[^1]]; + } + + foreach (var occurrence in occurrences) { - DispatchId = jobId, - Kind = ScheduledDispatchKind.JobOccurrence, - Destination = definition.Name, - Body = Array.Empty(), - Headers = CreateOccurrenceHeaders(definition, scheduledForUtc.Value, scopeKey), - DueUtc = utcNow, - JobId = jobId - }; - - await _store.ScheduleDispatchAsync(dispatch, cancellationToken).ConfigureAwait(false); - scheduled.Add(dispatch); + string jobId = CreateOccurrenceId(definition.Name, occurrence, scopeKey); + + if (await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false) is not null) + continue; + + await _store.CreateIfAbsentAsync(new JobState + { + JobId = jobId, + Name = definition.Name, + JobType = GetJobTypeName(definition.JobType), + Status = JobStatus.Scheduled, + CreatedUtc = utcNow, + LastUpdatedUtc = utcNow, + ScheduledForUtc = occurrence + }, cancellationToken).ConfigureAwait(false); + + var dispatch = new ScheduledDispatchState + { + DispatchId = jobId, + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = definition.Name, + Body = Array.Empty(), + Headers = CreateOccurrenceHeaders(definition, occurrence, scopeKey), + DueUtc = utcNow, + JobId = jobId + }; + + await _store.ScheduleDispatchAsync(dispatch, cancellationToken).ConfigureAwait(false); + scheduled.Add(dispatch); + } } return scheduled; @@ -221,8 +246,8 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow - }, cancellationToken).ConfigureAwait(false); - await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + }, cancellationToken: cancellationToken).ConfigureAwait(false); + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.Add(GetRetryBackoff(definition, state.Attempt)), cancellationToken).ConfigureAwait(false); continue; } @@ -231,7 +256,7 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow - }, cancellationToken).ConfigureAwait(false); + }, cancellationToken: cancellationToken).ConfigureAwait(false); } await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); @@ -269,7 +294,7 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat private async Task TryPrepareOccurrenceForRunAsync(string jobId, ScheduledJobDefinition definition, DateTimeOffset utcNow, CancellationToken cancellationToken) { - if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { JobType = GetJobTypeName(definition.JobType), LastUpdatedUtc = utcNow }, cancellationToken).ConfigureAwait(false)) + if (await _store.TryTransitionAsync(jobId, JobStatus.Scheduled, JobStatus.Queued, new JobStatePatch { JobType = GetJobTypeName(definition.JobType), LastUpdatedUtc = utcNow }, cancellationToken: cancellationToken).ConfigureAwait(false)) return true; var state = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); @@ -283,7 +308,7 @@ private async Task TryPrepareOccurrenceForRunAsync(string jobId, Scheduled ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow - }, cancellationToken).ConfigureAwait(false); + }, cancellationToken: cancellationToken).ConfigureAwait(false); return false; } @@ -293,7 +318,7 @@ private async Task TryPrepareOccurrenceForRunAsync(string jobId, Scheduled ClearNodeId = true, ClearLeaseExpiresUtc = true, LastUpdatedUtc = utcNow - }, cancellationToken).ConfigureAwait(false); + }, cancellationToken: cancellationToken).ConfigureAwait(false); } private string? GetJobTypeName(Type? jobType) @@ -301,6 +326,16 @@ private async Task TryPrepareOccurrenceForRunAsync(string jobId, Scheduled return jobType is null ? null : _jobTypes.GetName(jobType); } + private static TimeSpan GetRetryBackoff(ScheduledJobDefinition definition, int attempt) + { + if (definition.RetryBackoff is { } custom) + return custom(attempt); + + // Capped exponential backoff: 1s, 2s, 4s, ... up to 5 minutes. + double seconds = Math.Min(300, Math.Pow(2, Math.Max(0, attempt - 1))); + return TimeSpan.FromSeconds(seconds); + } + private async Task HasActiveOccurrenceAsync(string name, string scopeKey, CancellationToken cancellationToken) { var states = await _store.QueryAsync(new JobQuery { Name = name, Limit = 1000 }, cancellationToken).ConfigureAwait(false); @@ -328,189 +363,24 @@ private static MessageHeaders CreateOccurrenceHeaders(ScheduledJobDefinition def internal static void ValidateCron(string expression) { - CronSchedule.Parse(expression); - } - - private sealed class CronSchedule - { - private readonly CronFieldSet _second; - private readonly CronFieldSet _minute; - private readonly CronFieldSet _hour; - private readonly CronFieldSet _dayOfMonth; - private readonly CronFieldSet _month; - private readonly CronFieldSet _dayOfWeek; - - private CronSchedule(CronFieldSet second, CronFieldSet minute, CronFieldSet hour, CronFieldSet dayOfMonth, CronFieldSet month, CronFieldSet dayOfWeek) - { - _second = second; - _minute = minute; - _hour = hour; - _dayOfMonth = dayOfMonth; - _month = month; - _dayOfWeek = dayOfWeek; - } - - public static CronSchedule Parse(string expression) - { - ArgumentException.ThrowIfNullOrWhiteSpace(expression); - - var parts = expression.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (parts.Length != 5 && parts.Length != 6) - throw new FormatException("Cron expressions must contain five fields, or six fields when seconds are included."); - - int offset = parts.Length == 6 ? 0 : -1; - return new CronSchedule( - offset == 0 ? CronFieldSet.Parse(parts[0], 0, 59) : CronFieldSet.Single(0, 0, 59), - CronFieldSet.Parse(parts[1 + offset], 0, 59), - CronFieldSet.Parse(parts[2 + offset], 0, 23), - CronFieldSet.Parse(parts[3 + offset], 1, 31, allowQuestion: true), - CronFieldSet.Parse(parts[4 + offset], 1, 12), - CronFieldSet.Parse(parts[5 + offset], 0, 6, allowQuestion: true, normalizeDayOfWeek: true)); - } - - public DateTimeOffset? GetLastOccurrence(DateTimeOffset utcNow, TimeZoneInfo timeZone, TimeSpan misfireWindow) - { - if (misfireWindow < TimeSpan.Zero) - throw new ArgumentOutOfRangeException(nameof(misfireWindow), misfireWindow, "MisfireWindow must be greater than or equal to zero."); - - var localNow = TimeZoneInfo.ConvertTime(utcNow, timeZone); - var candidate = new DateTimeOffset(localNow.Year, localNow.Month, localNow.Day, localNow.Hour, localNow.Minute, localNow.Second, localNow.Offset); - int secondsToSearch = Math.Max(1, (int)Math.Ceiling(misfireWindow.TotalSeconds)) + 1; - - for (int i = 0; i <= secondsToSearch; i++) - { - if (Matches(candidate.DateTime)) - return TimeZoneInfo.ConvertTime(candidate, TimeZoneInfo.Utc); - - candidate = candidate.AddSeconds(-1); - } - - return null; - } - - private bool Matches(DateTime local) - { - if (!_second.Contains(local.Second) || !_minute.Contains(local.Minute) || !_hour.Contains(local.Hour) || !_month.Contains(local.Month)) - return false; - - bool dayOfMonthMatches = _dayOfMonth.Contains(local.Day); - bool dayOfWeekMatches = _dayOfWeek.Contains((int)local.DayOfWeek); - return _dayOfMonth.IsAny || _dayOfWeek.IsAny - ? dayOfMonthMatches && dayOfWeekMatches - : dayOfMonthMatches || dayOfWeekMatches; - } + ParseCron(expression); } - private sealed class CronFieldSet + /// + /// Parses a 5- or 6-field cron expression using the vendored Cronos parser. Six fields are interpreted as + /// seconds-first (); five fields use the standard format. Cronos + /// supports the full grammar (ranges, steps, lists, L/W/#, named months/days, and macros + /// such as @daily). + /// + private static CronExpression ParseCron(string expression) { - private readonly bool[] _values; - private readonly int _min; - - private CronFieldSet(bool[] values, int min, bool isAny) - { - _values = values; - _min = min; - IsAny = isAny; - } + ArgumentException.ThrowIfNullOrWhiteSpace(expression); - public bool IsAny { get; } + if (expression.StartsWith('@')) + return CronExpression.Parse(expression, CronFormat.IncludeSeconds); - public static CronFieldSet Single(int value, int min, int max) - { - var values = new bool[max - min + 1]; - values[value - min] = true; - return new CronFieldSet(values, min, false); - } - - public static CronFieldSet Parse(string expression, int min, int max, bool allowQuestion = false, bool normalizeDayOfWeek = false) - { - if (expression == "*" || (allowQuestion && expression == "?")) - return Any(min, max); - - var values = new bool[max - min + 1]; - foreach (string segment in expression.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) - AddSegment(values, segment, min, max, allowQuestion, normalizeDayOfWeek); - - return new CronFieldSet(values, min, false); - } - - public bool Contains(int value) - { - int index = value - _min; - return index >= 0 && index < _values.Length && _values[index]; - } - - private static CronFieldSet Any(int min, int max) - { - var values = new bool[max - min + 1]; - Array.Fill(values, true); - return new CronFieldSet(values, min, true); - } - - private static void AddSegment(bool[] values, string segment, int min, int max, bool allowQuestion, bool normalizeDayOfWeek) - { - string[] stepParts = segment.Split('/', StringSplitOptions.TrimEntries); - if (stepParts.Length > 2) - throw new FormatException($"Invalid cron field segment '{segment}'."); - - int step = stepParts.Length == 2 ? ParseNumber(stepParts[1], 1, max) : 1; - string range = stepParts[0]; - - if (range == "*" || (allowQuestion && range == "?")) - { - AddRange(values, min, max, step, min, normalizeDayOfWeek); - return; - } - - string[] rangeParts = range.Split('-', StringSplitOptions.TrimEntries); - if (rangeParts.Length == 1) - { - int value = Normalize(ParseNumber(rangeParts[0], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); - EnsureInRange(value, min, max); - values[value - min] = true; - return; - } - - if (rangeParts.Length != 2) - throw new FormatException($"Invalid cron field segment '{segment}'."); - - int start = Normalize(ParseNumber(rangeParts[0], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); - int end = Normalize(ParseNumber(rangeParts[1], min, normalizeDayOfWeek ? max + 1 : max), normalizeDayOfWeek); - EnsureInRange(start, min, max); - EnsureInRange(end, min, max); - - if (end < start) - throw new FormatException($"Invalid cron range '{segment}'."); - - AddRange(values, start, end, step, min, normalizeDayOfWeek); - } - - private static void AddRange(bool[] values, int start, int end, int step, int min, bool normalizeDayOfWeek) - { - for (int value = start; value <= end; value += step) - { - int normalized = Normalize(value, normalizeDayOfWeek); - values[normalized - min] = true; - } - } - - private static int ParseNumber(string value, int min, int max) - { - if (!Int32.TryParse(value, out int result) || result < min || result > max) - throw new FormatException($"Cron value '{value}' must be between {min} and {max}."); - - return result; - } - - private static int Normalize(int value, bool normalizeDayOfWeek) - { - return normalizeDayOfWeek && value == 7 ? 0 : value; - } - - private static void EnsureInRange(int value, int min, int max) - { - if (value < min || value > max) - throw new FormatException($"Cron value '{value}' must be between {min} and {max}."); - } + int fieldCount = expression.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length; + var format = fieldCount == 6 ? CronFormat.IncludeSeconds : CronFormat.Standard; + return CronExpression.Parse(expression, format); } } diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 717bb0b2b..f134cd9da 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -10,7 +10,7 @@ namespace Foundatio.Messaging; -public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo +public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo { private static readonly IReadOnlySet _supportedRoles = new HashSet { @@ -89,9 +89,13 @@ private async Task> ReceiveAsync(string source, Re ? _timeProvider.GetUtcNow().Add(waitTime) : null; + // Return any messages whose visibility window lapsed (consumer crashed without settling) to the queue so + // they are redelivered with an incremented delivery count — honoring the advertised at-least-once contract. + state.ReclaimExpired(_timeProvider.GetUtcNow()); + while (entries.Count < maxMessages) { - if (TryReceive(source, state, out var entry)) + if (TryReceive(source, state, visibility, out var entry)) { entries.Add(entry); continue; @@ -172,6 +176,35 @@ public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationTo return Task.CompletedTask; } + public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(request); + + if (!_destinations.TryGetValue(destination, out var state)) + return Task.FromResult>([]); + + int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; + var entries = new List(maxMessages); + while (entries.Count < maxMessages && state.TryReadDeadletter(out var message)) + { + entries.Add(new TransportEntry + { + Id = message.Id, + Destination = destination, + Body = message.Body, + Headers = message.Headers, + DeliveryCount = message.DeliveryCount, + EnqueuedUtc = message.EnqueuedUtc, + Receipt = new Receipt { TransportState = null } + }); + } + + return Task.FromResult>(entries); + } + public Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct) { ThrowIfDisposed(); @@ -318,7 +351,20 @@ private async Task RunPushSubscriptionAsync(string source, Func _consumers = new(StringComparer.Ordinal); private int _isDisposed; @@ -110,6 +114,7 @@ public MessageQueue(IMessageTransport transport, QueueOptions? options = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _options = options ?? new QueueOptions(); + _logger = (_options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); } public async Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class @@ -330,21 +335,55 @@ private async Task StartConsumerCoreAsync(string source, strin } } + // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving + // or while processing a single entry (including a poison message that was already dead-lettered) must never tear + // down the consumer loop, otherwise one bad message or a transient transport blip silently stops consumption. private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull, Func onMessage, QueueConsumerOptions options, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - var entries = await pull.ReceiveAsync(source, new ReceiveRequest + IReadOnlyList entries; + try + { + entries = await pull.ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = Math.Max(1, options.MaxConcurrency), + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) { - MaxMessages = Math.Max(1, options.MaxConcurrency), - MaxWaitTime = TimeSpan.FromSeconds(1) - }, cancellationToken).AnyContext(); + _logger.LogError(ex, "Error receiving from \"{Source}\"; retrying: {Message}", source, ex.Message); + await _options.TimeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); + continue; + } - var tasks = entries.Select(entry => onMessage(entry, cancellationToken)).ToArray(); + var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, source, cancellationToken)).ToArray(); await Task.WhenAll(tasks).AnyContext(); } } + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string source, CancellationToken cancellationToken) + { + try + { + await onMessage(entry, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + // The message has already been settled (dead-lettered on deserialize failure, abandoned/dead-lettered on + // handler error); swallowing here keeps the loop alive for the next message. + _logger.LogError(ex, "Error processing message \"{MessageId}\" from \"{Source}\": {Message}", entry.Id, source, ex.Message); + } + } + private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken ct) { return new ReceivedMessage(_transport, entry, ct, _options.RuntimeStore, _options.TimeProvider); @@ -381,8 +420,9 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func(IReceivedMessage message, Func> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct); } public interface ISupportsLockRenewal : IMessageTransport diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 28c9e82da..9d4d9fbb8 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -9,6 +9,8 @@ using Foundatio.Jobs; using Foundatio.Serializer; using Foundatio.Utility; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace Foundatio.Messaging; @@ -42,6 +44,7 @@ public sealed record PubSubOptions public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IJobRuntimeStore? RuntimeStore { get; init; } public TimeProvider TimeProvider { get; init; } = TimeProvider.System; + public ILoggerFactory? LoggerFactory { get; init; } } public interface IPubSub : IAsyncDisposable @@ -66,6 +69,7 @@ public sealed class PubSub : IPubSub { private readonly IMessageTransport _transport; private readonly PubSubOptions _options; + private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); private int _isDisposed; @@ -73,6 +77,7 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _options = options ?? new PubSubOptions(); + _logger = (_options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); } public async Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class @@ -255,21 +260,53 @@ private async Task SubscribeCoreAsync(string topic, string } } + // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving + // or while processing a single entry (including a poison message that was already dead-lettered) must never tear + // down the subscription loop, otherwise one bad message or a transient transport blip silently stops delivery. private async Task RunPullSubscriptionLoopAsync(string subscription, ISupportsPull pull, Func onMessage, PubSubSubscriptionOptions options, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - var entries = await pull.ReceiveAsync(subscription, new ReceiveRequest + IReadOnlyList entries; + try + { + entries = await pull.ReceiveAsync(subscription, new ReceiveRequest + { + MaxMessages = Math.Max(1, options.MaxConcurrency), + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) { - MaxMessages = Math.Max(1, options.MaxConcurrency), - MaxWaitTime = TimeSpan.FromSeconds(1) - }, cancellationToken).AnyContext(); + _logger.LogError(ex, "Error receiving from subscription \"{Subscription}\"; retrying: {Message}", subscription, ex.Message); + await _options.TimeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); + continue; + } - var tasks = entries.Select(entry => onMessage(entry, cancellationToken)).ToArray(); + var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, subscription, cancellationToken)).ToArray(); await Task.WhenAll(tasks).AnyContext(); } } + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string subscription, CancellationToken cancellationToken) + { + try + { + await onMessage(entry, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + _logger.LogError(ex, "Error processing message \"{MessageId}\" from subscription \"{Subscription}\": {Message}", entry.Id, subscription, ex.Message); + } + } + private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) { return new ReceivedMessage(_transport, entry, cancellationToken, _options.RuntimeStore, _options.TimeProvider); @@ -323,8 +360,9 @@ private async Task HandleMessageAsync(IReceivedMessage message, Func(IReceivedMessage message, Func= 5, $"Expected multiple missed occurrences, got {first.Count}"); + + // Deterministic occurrence ids dedupe across overlapping windows: a second pass at the same time adds nothing. + var second = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + Assert.Empty(second); + } + + [Fact] + public async Task JobRuntimeService_RunsQueuedJobsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var probe = new JobSchedulerProbe(); + var serviceProvider = new ServiceCollection().AddSingleton(probe).BuildServiceProvider(); + var store = new InMemoryJobRuntimeStore(); + var scheduler = new InMemoryJobScheduler(); + var registry = new JobTypeRegistry([new JobTypeRegistration("probe", typeof(ScheduledProbeJob))]); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a", jobTypes: registry); + var processor = new JobScheduleProcessor(scheduler, store, worker, nodeId: "node-a", jobTypes: registry); + var client = new JobClient(store, jobTypes: registry); + + var service = new Foundatio.Extensions.Hosting.Jobs.JobRuntimeService(processor, worker, + options: new Foundatio.Extensions.Hosting.Jobs.JobRuntimeServiceOptions { PollInterval = TimeSpan.FromMilliseconds(50) }); + + await ((Microsoft.Extensions.Hosting.IHostedService)service).StartAsync(cancellationToken); + try + { + var handle = await client.EnqueueAsync(cancellationToken: cancellationToken); + + JobState? state = null; + for (int i = 0; i < 100 && (state = await handle.GetStateAsync(cancellationToken))?.Status != JobStatus.Completed; i++) + await Task.Delay(50, cancellationToken); + + Assert.Equal(JobStatus.Completed, state?.Status); + Assert.Equal(1, probe.RunCount); + } + finally + { + await ((Microsoft.Extensions.Hosting.IHostedService)service).StopAsync(cancellationToken); + } + } + [Fact] public async Task RunDueOccurrencesAsync_WhenOccurrenceIsDue_RunsConfiguredJobAsync() { diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs index 9671f6f08..75e6c58d8 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -92,4 +92,22 @@ public override Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() { return base.ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync(); } + + [Fact] + public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() + { + return base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); + } + + [Fact] + public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() + { + return base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); + } + + [Fact] + public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() + { + return base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); + } } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 14962373a..7432df1ad 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -159,6 +159,60 @@ public async Task StartConsumerAsync_WithAutoAck_CompletesMessageAsync() await WaitForCompletedAsync(transport, "preview-work-item", cancellationToken); } + [Fact] + public async Task StartConsumerAsync_WithManualAck_DoesNotAutoCompleteAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var handled = new AsyncCountdownEvent(1); + + await using var consumer = await queue.StartConsumerAsync((message, _) => + { + handled.Signal(); + return Task.CompletedTask; // intentionally does NOT settle the message + }, new QueueConsumerOptions { AckMode = AckMode.Manual }, cts.Token); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "manual" }, cancellationToken: cts.Token); + await handled.WaitAsync(TimeSpan.FromSeconds(2)); + await Task.Delay(200, cts.Token); + + // Manual ack: the handler ran but did not settle, so the message stays in flight and is not auto-completed. + var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + Assert.Equal(0, stats.Completed); + Assert.Equal(1, stats.Working); + } + + [Fact] + public async Task StartConsumerAsync_WithPoisonMessage_DeadLettersAndKeepsConsumingAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + var handled = new AsyncCountdownEvent(1); + + await using var consumer = await queue.StartConsumerAsync((message, _) => + { + Assert.Equal("good", message.Message.Data); + handled.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + // A poison (undeserializable) payload must be dead-lettered without tearing down the consumer loop, so the + // subsequent valid message is still delivered. + await transport.SendAsync("preview-work-item", [ + new TransportMessage { Body = System.Text.Encoding.UTF8.GetBytes("}{ not json"), Headers = MessageHeaders.Empty } + ], new TransportSendOptions(), cts.Token); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "good" }, cancellationToken: cts.Token); + + await handled.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(0, handled.CurrentCount); + } + [Fact] public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() { From 8eada6a9332c17472d294f2f6870d5e203d3c9fc Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 28 Jun 2026 00:12:10 -0500 Subject: [PATCH 12/57] refactor: hoist shared MessageQueue/PubSub behavior into MessageClientCore (M1, M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1 — remove the large duplication between MessageQueue and PubSub: - New internal MessageClientCore owns serialization, header/trace construction, routing-agnostic send, runtime-store scheduled dispatch, received-message creation with poison handling, auto/manual ack settlement, and the resilient consumer/subscription loop. - Unify the two near-identical handle classes into one MessageListenerHandle implementing both IMessageConsumer and IMessageSubscription. - Collapse the duplicated HandleMessageAsync overloads into a single generic method. - MessageQueue and PubSub become thin adapters mapping their option shapes onto the core. Fixes the prior drift: pub/sub subscriptions now also honor RedeliveryBackoff. M2 — enforce ITransportInfo.MaxBatchSize: oversized sends are split into chunks of at most MaxBatchSize (test via a fake transport). Behavior preserved — verified by the existing messaging, queue, and jobs suites plus the added chunking test. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio/Messaging/MessageClientCore.cs | 742 ++++++++++++++++++ src/Foundatio/Messaging/MessageQueue.cs | 737 ++--------------- src/Foundatio/Messaging/PubSub.cs | 524 ++----------- .../Queue/MessageQueueTests.cs | 49 ++ 4 files changed, 896 insertions(+), 1156 deletions(-) create mode 100644 src/Foundatio/Messaging/MessageClientCore.cs diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs new file mode 100644 index 000000000..304a30eac --- /dev/null +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -0,0 +1,742 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Serializer; +using Foundatio.Utility; +using Microsoft.Extensions.Logging; + +namespace Foundatio.Messaging; + +/// +/// Transport-neutral envelope options shared by queue send and pub/sub publish operations. +/// +internal sealed record MessageEnvelopeOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public TimeSpan? Delay { get; init; } + public DateTimeOffset? DeliverAt { get; init; } + public TimeSpan? TimeToLive { get; init; } + public string? CorrelationId { get; init; } + public string? DeduplicationId { get; init; } + public MessageHeaders? Headers { get; init; } +} + +/// +/// Describes a consumer/subscription listener independent of whether it is backed by a queue or a pub/sub subscription. +/// +internal sealed record ListenerConfig +{ + public required string Source { get; init; } + public required string Key { get; init; } + public required Type MessageType { get; init; } + public string Topic { get; init; } = ""; + public string Subscription { get; init; } = ""; + public AckMode AckMode { get; init; } = AckMode.Auto; + public int MaxConcurrency { get; init; } = 1; + public int MaxAttempts { get; init; } = 5; + public Func? RedeliveryBackoff { get; init; } +} + +/// +/// Shared implementation behind and : serialization, header/trace +/// construction, routing-agnostic send (with batch chunking and runtime-store scheduled dispatch), received-message +/// creation with poison handling, auto/manual ack settlement, and the resilient consumer/subscription loop. +/// +internal sealed class MessageClientCore : IAsyncDisposable +{ + private readonly IMessageTransport _transport; + private readonly ISerializer _serializer; + private readonly IMessageRouter _router; + private readonly IJobRuntimeStore? _runtimeStore; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly Func _exceptionFactory; + private readonly ConcurrentDictionary _listeners = new(StringComparer.Ordinal); + private int _isDisposed; + + public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, + IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory) + { + _transport = transport ?? throw new ArgumentNullException(nameof(transport)); + _serializer = serializer; + _router = router; + _runtimeStore = runtimeStore; + _timeProvider = timeProvider; + _logger = logger; + _exceptionFactory = exceptionFactory; + } + + public IMessageRouter Router => _router; + + public Task EnsureAsync(IReadOnlyList declarations, CancellationToken cancellationToken) + { + return _transport is ISupportsProvisioning provisioning + ? provisioning.EnsureAsync(declarations, cancellationToken) + : Task.CompletedTask; + } + + public async Task SendAsync(ScheduledDispatchKind kind, Type messageType, object message, MessageEnvelopeOptions options, string destination, Func? ensureDestination, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + ValidateCapabilities(options.Priority, options.TimeToLive); + + var sendOptions = BuildSendOptions(options); + string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + var transportMessage = CreateTransportMessage(message, messageType, options, messageId); + + if (ensureDestination is not null) + await ensureDestination(destination, cancellationToken).AnyContext(); + + if (await TryScheduleAsync(kind, destination, [transportMessage], sendOptions, cancellationToken).AnyContext()) + return messageId; + + var items = await SendChunkedAsync(destination, [transportMessage], sendOptions, cancellationToken).AnyContext(); + var item = items.Count > 0 ? items[0] : null; + if (item is null || !item.Success) + throw _exceptionFactory($"Unable to send message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}", null); + + return item.MessageId ?? messageId; + } + + public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable messages, Type? declaredType, MessageEnvelopeOptions options, Func resolveDestination, Func? ensureDestination, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + ValidateCapabilities(options.Priority, options.TimeToLive); + + var sendOptions = BuildSendOptions(options); + var grouped = new Dictionary>(StringComparer.Ordinal); + int index = 0; + + foreach (var message in messages) + { + ArgumentNullException.ThrowIfNull(message); + Type messageType = declaredType ?? message.GetType(); + string destination = resolveDestination(messageType); + string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; + index++; + + if (!grouped.TryGetValue(destination, out var transportMessages)) + { + transportMessages = []; + grouped.Add(destination, transportMessages); + } + + transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); + } + + foreach (var group in grouped) + { + if (ensureDestination is not null) + await ensureDestination(group.Key, cancellationToken).AnyContext(); + + if (await TryScheduleAsync(kind, group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) + continue; + + var items = await SendChunkedAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); + int failed = items.Count(i => !i.Success); + if (failed > 0) + throw _exceptionFactory($"Unable to send {failed} of {items.Count} messages to \"{group.Key}\".", null); + } + } + + public async Task ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + var pull = RequirePull(); + var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); + return entries.Count == 0 ? null : CreateReceivedMessage(entries[0], cancellationToken); + } + + public async Task?> ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) where T : class + { + ThrowIfDisposed(); + var pull = RequirePull(); + var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); + return entries.Count == 0 ? null : await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); + } + + public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(handler); + var registration = MessageListenerRegistration.Create(handler, config); + return StartListenerCoreAsync(config, registration, async (entry, token) => + { + var received = CreateReceivedMessage(entry, token); + await HandleMessageAsync(received, config, handler, token).AnyContext(); + }, cancellationToken); + } + + public Task StartListenerAsync(ListenerConfig config, Func, CancellationToken, Task> handler, CancellationToken cancellationToken) where T : class + { + ArgumentNullException.ThrowIfNull(handler); + var registration = MessageListenerRegistration.Create(handler, config); + return StartListenerCoreAsync(config, registration, async (entry, token) => + { + var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + await HandleMessageAsync(received, config, handler, token).AnyContext(); + }, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + foreach (var listener in _listeners.Values.ToArray()) + await listener.DisposeAsync().AnyContext(); + + await _transport.DisposeAsync().AnyContext(); + } + + private async Task StartListenerCoreAsync(ListenerConfig config, MessageListenerRegistration registration, Func onMessage, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + if (_listeners.TryGetValue(config.Key, out var existing) && !existing.IsDisposed) + { + existing.ThrowIfConflicting(registration); + return existing; + } + + var handle = new MessageListenerHandle(config.Topic, config.Subscription, config.Source, config.Key, registration, RemoveListener); + if (!_listeners.TryAdd(config.Key, handle)) + { + await handle.DisposeAsync().AnyContext(); + var current = _listeners[config.Key]; + current.ThrowIfConflicting(registration); + return current; + } + + try + { + if (_transport is ISupportsPush push) + { + var subscription = await push.SubscribeAsync(config.Source, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, config.MaxConcurrency) }, cancellationToken).AnyContext(); + handle.SetPushSubscription(subscription); + return handle; + } + + if (_transport is not ISupportsPull pull) + throw _exceptionFactory($"Transport \"{_transport.GetType().Name}\" does not support receiving messages.", null); + + handle.Start(RunPullLoopAsync(config.Source, pull, onMessage, config.MaxConcurrency, handle.CancellationToken)); + return handle; + } + catch + { + await handle.DisposeAsync().AnyContext(); + throw; + } + } + + // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving + // or while processing a single entry (including a poison message that was already dead-lettered) must never tear + // down the loop, otherwise one bad message or a transient transport blip silently stops consumption. + private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func onMessage, int maxConcurrency, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + IReadOnlyList entries; + try + { + entries = await pull.ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = Math.Max(1, maxConcurrency), + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error receiving from \"{Source}\"; retrying: {Message}", source, ex.Message); + await _timeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); + continue; + } + + var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, source, cancellationToken)).ToArray(); + await Task.WhenAll(tasks).AnyContext(); + } + } + + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string source, CancellationToken cancellationToken) + { + try + { + await onMessage(entry, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + // The message has already been settled (dead-lettered on deserialize failure, abandoned/dead-lettered on + // handler error); swallowing here keeps the loop alive for the next message. + _logger.LogError(ex, "Error processing message \"{MessageId}\" from \"{Source}\": {Message}", entry.Id, source, ex.Message); + } + } + + private async Task HandleMessageAsync(TMessage message, ListenerConfig config, Func handler, CancellationToken cancellationToken) where TMessage : IReceivedMessage + { + try + { + await handler(message, cancellationToken).AnyContext(); + + if (config.AckMode == AckMode.Auto && !message.IsHandled) + await message.CompleteAsync(cancellationToken).AnyContext(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, config.Source, message.Attempts, config.MaxAttempts, ex.Message); + await SettleFailedMessageAsync(message, config, cancellationToken).AnyContext(); + } + } + + private static async Task SettleFailedMessageAsync(IReceivedMessage message, ListenerConfig config, CancellationToken cancellationToken) + { + if (message.IsHandled) + return; + + if (message.Attempts >= config.MaxAttempts) + { + await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); + return; + } + + TimeSpan? redeliveryDelay = config.RedeliveryBackoff?.Invoke(message.Attempts); + if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ISupportsDelayedMessageAbandon received) + await received.AbandonAsync(delay, cancellationToken).AnyContext(); + else + await message.AbandonAsync(cancellationToken).AnyContext(); + } + + private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) + { + return new ReceivedMessage(_transport, entry, cancellationToken, _runtimeStore, _timeProvider); + } + + private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class + { + T? message; + try + { + message = _serializer.Deserialize(entry.Body); + } + catch (Exception ex) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + throw _exceptionFactory($"Unable to deserialize message \"{entry.Id}\".", ex); + } + + if (message is null) + { + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); + } + + return new ReceivedMessage(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider); + } + + private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) + { + return ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken); + } + + public string ResolveMessageType(Type messageType) => _router.ResolveMessageType(messageType); + + private TransportMessage CreateTransportMessage(object message, Type messageType, MessageEnvelopeOptions options, string? messageId) + { + // Content type is intentionally not written as a header: the receive path always uses the single configured + // serializer, so advertising a per-message content type would be misleading until real negotiation exists. + var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() + .Set(KnownHeaders.MessageType, _router.ResolveMessageType(messageType)) + .Set(KnownHeaders.Priority, options.Priority.ToString()); + + if (!String.IsNullOrEmpty(options.CorrelationId)) + headers.Set(KnownHeaders.CorrelationId, options.CorrelationId); + + if (Activity.Current is { } activity) + { + if (!String.IsNullOrEmpty(activity.Id)) + headers.SetIfMissing(KnownHeaders.TraceParent, activity.Id); + + if (!String.IsNullOrEmpty(activity.TraceStateString)) + headers.SetIfMissing(KnownHeaders.TraceState, activity.TraceStateString); + } + + if (options.TimeToLive is { } ttl) + headers.Set(KnownHeaders.Expiration, _timeProvider.GetUtcNow().Add(ttl).ToString("O", CultureInfo.InvariantCulture)); + + return new TransportMessage + { + Body = _serializer.SerializeToBytes(message), + Headers = headers.Build(), + MessageId = messageId + }; + } + + private TransportSendOptions BuildSendOptions(MessageEnvelopeOptions options) + { + return new TransportSendOptions + { + Priority = options.Priority, + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _timeProvider.GetUtcNow().Add(delay) : null), + DeduplicationId = options.DeduplicationId + }; + } + + private void ValidateCapabilities(MessagePriority priority, TimeSpan? timeToLive) + { + if (priority != MessagePriority.Normal && _transport is not ISupportsPriority) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); + + if (timeToLive is not null && _transport is not ISupportsExpiration) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); + } + + private async Task TryScheduleAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + { + if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) + return false; + + foreach (var message in messages) + { + string messageId = message.MessageId ?? Guid.NewGuid().ToString("N"); + await _runtimeStore!.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = messageId, + Kind = kind, + Destination = destination, + Body = message.Body, + Headers = message.Headers, + Options = options with { DeliverAt = null }, + DueUtc = dueUtc + }, cancellationToken).AnyContext(); + } + + return true; + } + + private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) + { + dueUtc = options.DeliverAt.GetValueOrDefault(); + if (options.DeliverAt is null || dueUtc <= _timeProvider.GetUtcNow()) + return false; + + if (_transport is ISupportsDelayedDelivery) + return false; + + if (_runtimeStore is null) + throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or a registered job runtime store.", null); + + return true; + } + + private async Task> SendChunkedAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + { + // Respect a transport-declared maximum batch size by splitting oversized sends into chunks. + int? maxBatchSize = (_transport as ITransportInfo)?.MaxBatchSize; + if (maxBatchSize is not { } limit || limit <= 0 || messages.Count <= limit) + { + var result = await _transport.SendAsync(destination, messages, options, cancellationToken).AnyContext(); + return result.Items; + } + + var items = new List(messages.Count); + for (int offset = 0; offset < messages.Count; offset += limit) + { + var chunk = messages.Skip(offset).Take(limit).ToArray(); + var result = await _transport.SendAsync(destination, chunk, options, cancellationToken).AnyContext(); + items.AddRange(result.Items); + } + + return items; + } + + private ISupportsPull RequirePull() + { + return _transport as ISupportsPull + ?? throw _exceptionFactory($"Transport \"{_transport.GetType().Name}\" does not support pull receive.", null); + } + + private void RemoveListener(string key, MessageListenerHandle handle) + { + _listeners.TryRemove(new KeyValuePair(key, handle)); + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + } +} + +internal interface ISupportsDelayedMessageAbandon +{ + Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default); +} + +internal class ReceivedMessage : IReceivedMessage, ISupportsDelayedMessageAbandon +{ + private readonly IMessageTransport _transport; + private readonly TransportEntry _entry; + private readonly IJobRuntimeStore? _runtimeStore; + private readonly TimeProvider _timeProvider; + private int _isHandled; + + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) + { + _transport = transport; + _entry = entry; + _runtimeStore = runtimeStore; + _timeProvider = timeProvider ?? TimeProvider.System; + CancellationToken = cancellationToken; + } + + public string Id => _entry.Id; + public ReadOnlyMemory Body => _entry.Body; + public MessageHeaders Headers => _entry.Headers; + public string? CorrelationId => Headers.GetValueOrDefault(KnownHeaders.CorrelationId); + public string? MessageType => Headers.GetValueOrDefault(KnownHeaders.MessageType); + public MessagePriority Priority => Enum.TryParse(Headers.GetValueOrDefault(KnownHeaders.Priority), ignoreCase: true, out MessagePriority priority) ? priority : MessagePriority.Normal; + public int Attempts => _entry.DeliveryCount; + public bool IsHandled => Volatile.Read(ref _isHandled) == 1; + public CancellationToken CancellationToken { get; } + + public Task CompleteAsync(CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return Task.CompletedTask; + + return _transport.CompleteAsync(_entry, cancellationToken); + } + + public Task AbandonAsync(CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return Task.CompletedTask; + + return _transport.AbandonAsync(_entry, cancellationToken); + } + + public async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return; + + if (_transport is ISupportsRedeliveryDelay redelivery) + { + await redelivery.AbandonAsync(_entry, redeliveryDelay, cancellationToken).AnyContext(); + return; + } + + if (_runtimeStore is null) + throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or a registered job runtime store."); + + int nextAttempt = _entry.DeliveryCount + 1; + var headers = _entry.Headers.ToBuilder() + .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) + .Build(); + + await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = $"{_entry.Id}:retry:{nextAttempt}", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = _entry.Destination, + Body = _entry.Body, + Headers = headers, + Options = new TransportSendOptions { Priority = Priority }, + DueUtc = _timeProvider.GetUtcNow().Add(redeliveryDelay) + }, cancellationToken).AnyContext(); + + await _transport.CompleteAsync(_entry, cancellationToken).AnyContext(); + } + + public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) + { + if (!TryMarkHandled()) + return; + + await DeadLetterAsync(_transport, _entry, reason, cancellationToken).AnyContext(); + } + + public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) + { + return _transport is ISupportsLockRenewal lockRenewal + ? lockRenewal.RenewLockAsync(_entry, duration, cancellationToken) + : throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support lock renewal."); + } + + public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) + { + throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue or pub/sub messages."); + } + + internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) + { + if (transport is not ISupportsDeadLetter deadLetter) + throw new NotSupportedException($"Transport \"{transport.GetType().Name}\" does not support dead-lettering."); + + await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + } + + private bool TryMarkHandled() + { + return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; + } +} + +internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class +{ + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) + : base(transport, entry, cancellationToken, runtimeStore, timeProvider) + { + Message = message; + } + + public T Message { get; } +} + +internal static class MessageRoutingConventions +{ + public static string ToKebabCase(string value) + { + if (String.IsNullOrEmpty(value)) + return value; + + Span buffer = stackalloc char[value.Length * 2]; + int position = 0; + for (int index = 0; index < value.Length; index++) + { + char current = value[index]; + if (Char.IsUpper(current)) + { + if (index > 0) + buffer[position++] = '-'; + + buffer[position++] = Char.ToLowerInvariant(current); + } + else + { + buffer[position++] = current; + } + } + + return new String(buffer[..position]); + } +} + +/// +/// A started listener handle. A single type backs both the queue consumer and pub/sub subscription surfaces; queue +/// callers observe it as (Source/Key), pub/sub callers as +/// (Topic/Subscription/Key). +/// +internal sealed class MessageListenerHandle : IMessageConsumer, IMessageSubscription +{ + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly Action _remove; + private IPushSubscription? _pushSubscription; + private Task? _worker; + private int _isDisposed; + + public MessageListenerHandle(string topic, string subscription, string source, string key, MessageListenerRegistration registration, Action remove) + { + Topic = topic; + Subscription = subscription; + Source = source; + Key = key; + Registration = registration; + _remove = remove; + } + + public string Topic { get; } + public string Subscription { get; } + public string Source { get; } + public string Key { get; } + public MessageListenerRegistration Registration { get; } + public CancellationToken CancellationToken => _cancellationTokenSource.Token; + public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; + + public void ThrowIfConflicting(MessageListenerRegistration registration) + { + if (!Registration.Matches(registration)) + throw new InvalidOperationException($"A listener with key \"{Key}\" is already registered with a different handler or options."); + } + + public void SetPushSubscription(IPushSubscription subscription) + { + _pushSubscription = subscription; + } + + public void Start(Task worker) + { + _worker = worker; + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + await _cancellationTokenSource.CancelAsync().AnyContext(); + + if (_pushSubscription is not null) + await _pushSubscription.DisposeAsync().AnyContext(); + + if (_worker is not null) + { + try + { + await _worker.AnyContext(); + } + catch (OperationCanceledException) { } + } + + _cancellationTokenSource.Dispose(); + _remove(Key, this); + } +} + +internal sealed record MessageListenerRegistration +{ + public required Type MessageType { get; init; } + public required string Source { get; init; } + public required Delegate Handler { get; init; } + public required AckMode AckMode { get; init; } + public required int MaxConcurrency { get; init; } + public required int MaxAttempts { get; init; } + public required bool HasRedeliveryBackoff { get; init; } + + public static MessageListenerRegistration Create(Delegate handler, ListenerConfig config) + { + return new MessageListenerRegistration + { + MessageType = config.MessageType, + Source = config.Source, + Handler = handler, + AckMode = config.AckMode, + MaxConcurrency = Math.Max(1, config.MaxConcurrency), + MaxAttempts = config.MaxAttempts, + HasRedeliveryBackoff = config.RedeliveryBackoff is not null + }; + } + + public bool Matches(MessageListenerRegistration other) + { + return MessageType == other.MessageType + && String.Equals(Source, other.Source, StringComparison.Ordinal) + && Handler == other.Handler + && AckMode == other.AckMode + && MaxConcurrency == other.MaxConcurrency + && MaxAttempts == other.MaxAttempts + && HasRedeliveryBackoff == other.HasRedeliveryBackoff; + } +} diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index 16d5ccd5e..0da6a6aff 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -1,8 +1,5 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -102,131 +99,68 @@ public interface IReceivedMessage : IReceivedMessage where T : class T Message { get; } } +/// +/// App-facing durable competing-consumer queue. Routing, serialization, settlement, scheduling, and the consumer loop +/// live in ; this type maps queue-shaped options onto that shared core. +/// public sealed class MessageQueue : IQueue { - private readonly IMessageTransport _transport; - private readonly QueueOptions _options; - private readonly ILogger _logger; - private readonly ConcurrentDictionary _consumers = new(StringComparer.Ordinal); - private int _isDisposed; + private readonly MessageClientCore _core; public MessageQueue(IMessageTransport transport, QueueOptions? options = null) { - _transport = transport ?? throw new ArgumentNullException(nameof(transport)); - _options = options ?? new QueueOptions(); - _logger = (_options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + ArgumentNullException.ThrowIfNull(transport); + options ??= new QueueOptions(); + var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, + static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner)); } - public async Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); - ThrowIfDisposed(); - options ??= new QueueMessageOptions(); - ValidateSendOptions(options); - - string destination = GetDestination(typeof(T), options.Destination); - var sendOptions = CreateSendOptions(options); - string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); - var transportMessage = CreateTransportMessage(message, typeof(T), options, messageId); - - if (await TryScheduleDispatchAsync(ScheduledDispatchKind.QueueMessage, destination, transportMessage, sendOptions, cancellationToken).AnyContext()) - return messageId; - - var result = await _transport.SendAsync(destination, [transportMessage], sendOptions, cancellationToken).AnyContext(); - var item = result.Items.Count > 0 ? result.Items[0] : null; - - if (item is null || !item.Success) - throw new MessageQueueException($"Unable to enqueue message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}"); - - return item.MessageId ?? messageId; + return _core.SendAsync(ScheduledDispatchKind.QueueMessage, typeof(T), message, ToEnvelope(options), GetDestination(typeof(T), options.Destination), ensureDestination: null, cancellationToken); } - public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); - await EnqueueBatchCoreAsync(messages.Cast(), typeof(T), options, cancellationToken).AnyContext(); + options ??= new QueueMessageOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); } - public async Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) + public Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(messages); - await EnqueueBatchCoreAsync(messages, null, options, cancellationToken).AnyContext(); + options ??= new QueueMessageOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages, null, ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); } - public async Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) + public Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) { - ThrowIfDisposed(); - - if (_transport is not ISupportsPull pull) - throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); - options ??= new QueueReceiveOptions(); - Type routeType = options.RouteType ?? typeof(object); - string source = GetDestination(routeType, options.Source); - var entries = await pull.ReceiveAsync(source, new ReceiveRequest - { - MaxMessages = 1, - MaxWaitTime = options.MaxWaitTime - }, cancellationToken).AnyContext(); - - if (entries.Count == 0) - return null; - - return CreateReceivedMessage(entries[0], cancellationToken); + return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(object), options.Source), options.MaxWaitTime, cancellationToken); } - public async Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class { - ThrowIfDisposed(); - - if (_transport is not ISupportsPull pull) - throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support pull receive."); - options ??= new QueueReceiveOptions(); - string source = GetDestination(options.RouteType ?? typeof(T), options.Source); - var entries = await pull.ReceiveAsync(source, new ReceiveRequest - { - MaxMessages = 1, - MaxWaitTime = options.MaxWaitTime - }, cancellationToken).AnyContext(); - - if (entries.Count == 0) - return null; - - return await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); + return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(T), options.Source), options.MaxWaitTime, cancellationToken); } public async Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(handler); options ??= new QueueConsumerOptions(); - Type routeType = options.RouteType ?? typeof(object); - string source = GetDestination(routeType, options.Source); - string key = GetConsumerKey(routeType, source, options.Key); - var registration = MessageListenerRegistration.Create(handler, routeType, source, options); - - return await StartConsumerCoreAsync(source, key, registration, options, async (entry, token) => - { - var received = CreateReceivedMessage(entry, token); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, cancellationToken).AnyContext(); + return await _core.StartListenerAsync(BuildConfig(options.RouteType ?? typeof(object), options), handler, cancellationToken).AnyContext(); } public async Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); options ??= new QueueConsumerOptions(); - Type routeType = options.RouteType ?? typeof(T); - string source = GetDestination(routeType, options.Source); - string key = GetConsumerKey(routeType, source, options.Key); - var registration = MessageListenerRegistration.Create(handler, routeType, source, options); - - return await StartConsumerCoreAsync(source, key, registration, options, async (entry, token) => - { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, cancellationToken).AnyContext(); + return await _core.StartListenerAsync(BuildConfig(options.RouteType ?? typeof(T), options), handler, cancellationToken).AnyContext(); } public async Task RunConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) @@ -241,328 +175,29 @@ public async Task RunConsumerAsync(Func, CancellationToke await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - var consumers = _consumers.Values.ToArray(); - foreach (var consumer in consumers) - await consumer.DisposeAsync().AnyContext(); - - await _transport.DisposeAsync().AnyContext(); + return _core.DisposeAsync(); } - private async Task EnqueueBatchCoreAsync(IEnumerable messages, Type? declaredType, QueueMessageOptions? options, CancellationToken cancellationToken) + private ListenerConfig BuildConfig(Type routeType, QueueConsumerOptions options) { - ThrowIfDisposed(); - - options ??= new QueueMessageOptions(); - ValidateSendOptions(options); - - var sendOptions = CreateSendOptions(options); - var grouped = new Dictionary>(StringComparer.Ordinal); - int index = 0; - - foreach (var message in messages) - { - ArgumentNullException.ThrowIfNull(message); - Type messageType = declaredType ?? message.GetType(); - string destination = GetDestination(messageType, options.Destination); - string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; - index++; - - if (!grouped.TryGetValue(destination, out var transportMessages)) - { - transportMessages = []; - grouped.Add(destination, transportMessages); - } - - transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); - } - - foreach (var group in grouped) - { - if (await TryScheduleDispatchesAsync(ScheduledDispatchKind.QueueMessage, group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) - continue; - - var result = await _transport.SendAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); - if (!result.AllSucceeded) - throw new MessageQueueException($"Unable to enqueue {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{group.Key}\"."); - } - } - - private async Task StartConsumerCoreAsync(string source, string key, MessageListenerRegistration registration, QueueConsumerOptions options, Func onMessage, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - if (_consumers.TryGetValue(key, out var existing) && !existing.IsDisposed) - { - existing.ThrowIfConflicting(registration); - return existing; - } - - var handle = new MessageConsumerHandle(source, key, registration, RemoveConsumer); - if (!_consumers.TryAdd(key, handle)) - { - await handle.DisposeAsync().AnyContext(); - var current = _consumers[key]; - current.ThrowIfConflicting(registration); - return current; - } - - try - { - if (_transport is ISupportsPush push) - { - var subscription = await push.SubscribeAsync(source, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); - - handle.SetPushSubscription(subscription); - return handle; - } - - if (_transport is not ISupportsPull pull) - throw new MessageQueueException($"Transport \"{_transport.GetType().Name}\" does not support receiving messages."); - - handle.Start(RunPullConsumerLoopAsync(source, pull, onMessage, options, handle.CancellationToken)); - return handle; - } - catch - { - await handle.DisposeAsync().AnyContext(); - throw; - } - } - - // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving - // or while processing a single entry (including a poison message that was already dead-lettered) must never tear - // down the consumer loop, otherwise one bad message or a transient transport blip silently stops consumption. - private async Task RunPullConsumerLoopAsync(string source, ISupportsPull pull, Func onMessage, QueueConsumerOptions options, CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - IReadOnlyList entries; - try - { - entries = await pull.ReceiveAsync(source, new ReceiveRequest - { - MaxMessages = Math.Max(1, options.MaxConcurrency), - MaxWaitTime = TimeSpan.FromSeconds(1) - }, cancellationToken).AnyContext(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error receiving from \"{Source}\"; retrying: {Message}", source, ex.Message); - await _options.TimeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); - continue; - } - - var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, source, cancellationToken)).ToArray(); - await Task.WhenAll(tasks).AnyContext(); - } - } - - private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string source, CancellationToken cancellationToken) - { - try - { - await onMessage(entry, cancellationToken).AnyContext(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - } - catch (Exception ex) - { - // The message has already been settled (dead-lettered on deserialize failure, abandoned/dead-lettered on - // handler error); swallowing here keeps the loop alive for the next message. - _logger.LogError(ex, "Error processing message \"{MessageId}\" from \"{Source}\": {Message}", entry.Id, source, ex.Message); - } - } - - private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken ct) - { - return new ReceivedMessage(_transport, entry, ct, _options.RuntimeStore, _options.TimeProvider); - } - - private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken ct) where T : class - { - try - { - var message = _options.Serializer.Deserialize(entry.Body); - if (message is null) - throw new MessageQueueException($"Message \"{entry.Id}\" deserialized to null."); - - return new ReceivedMessage(_transport, entry, message, ct, _options.RuntimeStore, _options.TimeProvider); - } - catch (Exception ex) when (ex is not MessageQueueException) - { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); - throw new MessageQueueException($"Unable to deserialize message \"{entry.Id}\".", ex); - } - catch (MessageQueueException) - { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ct).AnyContext(); - throw; - } - } - - private async Task HandleMessageAsync(IReceivedMessage message, Func handler, QueueConsumerOptions options, CancellationToken ct) - { - try - { - await handler(message, ct).AnyContext(); - - if (options.AckMode == AckMode.Auto && !message.IsHandled) - await message.CompleteAsync(ct).AnyContext(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, message.MessageType, message.Attempts, options.MaxAttempts, ex.Message); - await SettleFailedMessageAsync(message, options, ct).AnyContext(); - } - } - - private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, QueueConsumerOptions options, CancellationToken ct) where T : class - { - try - { - await handler(message, ct).AnyContext(); - - if (options.AckMode == AckMode.Auto && !message.IsHandled) - await message.CompleteAsync(ct).AnyContext(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, message.MessageType, message.Attempts, options.MaxAttempts, ex.Message); - await SettleFailedMessageAsync(message, options, ct).AnyContext(); - } - } - - private static async Task SettleFailedMessageAsync(IReceivedMessage message, QueueConsumerOptions options, CancellationToken ct) - { - if (message.IsHandled) - return; - - if (message.Attempts >= options.MaxAttempts) - { - await message.DeadLetterAsync("handler-error", ct).AnyContext(); - return; - } - - TimeSpan? redeliveryDelay = options.RedeliveryBackoff?.Invoke(message.Attempts); - if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ISupportsDelayedMessageAbandon received) - await received.AbandonAsync(delay, ct).AnyContext(); - else - await message.AbandonAsync(ct).AnyContext(); - } - - private TransportMessage CreateTransportMessage(object message, Type messageType, QueueMessageOptions options, string? messageId = null) - { - // Content type is intentionally not written as a header: the receive path always uses the single configured - // serializer, so advertising a per-message content type would be misleading until real negotiation exists. - var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() - .Set(KnownHeaders.MessageType, GetMessageType(messageType)) - .Set(KnownHeaders.Priority, options.Priority.ToString()); - - if (!String.IsNullOrEmpty(options.CorrelationId)) - headers.Set(KnownHeaders.CorrelationId, options.CorrelationId); - - if (Activity.Current is { } activity) - { - if (!String.IsNullOrEmpty(activity.Id)) - headers.SetIfMissing(KnownHeaders.TraceParent, activity.Id); - - if (!String.IsNullOrEmpty(activity.TraceStateString)) - headers.SetIfMissing(KnownHeaders.TraceState, activity.TraceStateString); - } - - if (options.TimeToLive is { } ttl) - headers.Set(KnownHeaders.Expiration, _options.TimeProvider.GetUtcNow().Add(ttl).ToString("O", CultureInfo.InvariantCulture)); - - return new TransportMessage - { - Body = _options.Serializer.SerializeToBytes(message), - Headers = headers.Build(), - MessageId = messageId - }; - } - - private TransportSendOptions CreateSendOptions(QueueMessageOptions options) - { - return new TransportSendOptions + string source = GetDestination(routeType, options.Source); + return new ListenerConfig { - Priority = options.Priority, - DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _options.TimeProvider.GetUtcNow().Add(delay) : null), - DeduplicationId = options.DeduplicationId + Source = source, + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{source}:{routeType.FullName ?? routeType.Name}", + MessageType = routeType, + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff }; } - private void ValidateSendOptions(QueueMessageOptions options) - { - if (options.Priority != MessagePriority.Normal && _transport is not ISupportsPriority) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); - - if (options.TimeToLive is not null && _transport is not ISupportsExpiration) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); - } - - private async Task TryScheduleDispatchesAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) - { - if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) - return false; - - for (int index = 0; index < messages.Count; index++) - { - var message = messages[index]; - string messageId = message.MessageId ?? Guid.NewGuid().ToString("N"); - await ScheduleDispatchAsync(kind, destination, message with { MessageId = messageId }, options, dueUtc, cancellationToken).AnyContext(); - } - - return true; - } - - private Task TryScheduleDispatchAsync(ScheduledDispatchKind kind, string destination, TransportMessage message, TransportSendOptions options, CancellationToken cancellationToken) - { - return TryScheduleDispatchesAsync(kind, destination, [message], options, cancellationToken); - } - - private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) - { - dueUtc = options.DeliverAt.GetValueOrDefault(); - if (options.DeliverAt is null || dueUtc <= _options.TimeProvider.GetUtcNow()) - return false; - - if (_transport is ISupportsDelayedDelivery) - return false; - - if (_options.RuntimeStore is null) - throw new MessageQueueException($"Delayed queue delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(QueueOptions)}.{nameof(QueueOptions.RuntimeStore)}."); - - return true; - } - - private Task ScheduleDispatchAsync(ScheduledDispatchKind kind, string destination, TransportMessage message, TransportSendOptions options, DateTimeOffset dueUtc, CancellationToken cancellationToken) - { - return _options.RuntimeStore!.ScheduleDispatchAsync(new ScheduledDispatchState - { - DispatchId = message.MessageId!, - Kind = kind, - Destination = destination, - Body = message.Body, - Headers = message.Headers, - Options = options with { DeliverAt = null }, - DueUtc = dueUtc - }, cancellationToken); - } - private string GetDestination(Type messageType, string? destination) { - return _options.Router.ResolveRoute(new MessageRouteContext + return _core.Router.ResolveRoute(new MessageRouteContext { MessageType = messageType, Role = MessageRouteRole.QueueDestination, @@ -570,297 +205,17 @@ private string GetDestination(Type messageType, string? destination) }); } - private string GetMessageType(Type messageType) - { - return _options.Router.ResolveMessageType(messageType); - } - - private static string GetConsumerKey(Type messageType, string source, string? key) - { - return !String.IsNullOrEmpty(key) - ? key - : $"{source}:{messageType.FullName ?? messageType.Name}"; - } - - private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken ct) - { - await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, ct).AnyContext(); - } - - private void RemoveConsumer(string key, MessageConsumerHandle handle) - { - _consumers.TryRemove(new KeyValuePair(key, handle)); - } - - private void ThrowIfDisposed() - { - ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); - } -} - -internal interface ISupportsDelayedMessageAbandon -{ - Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default); -} - -internal class ReceivedMessage : IReceivedMessage, ISupportsDelayedMessageAbandon -{ - private readonly IMessageTransport _transport; - private readonly TransportEntry _entry; - private readonly IJobRuntimeStore? _runtimeStore; - private readonly TimeProvider _timeProvider; - private int _isHandled; - - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) - { - _transport = transport; - _entry = entry; - _runtimeStore = runtimeStore; - _timeProvider = timeProvider ?? TimeProvider.System; - CancellationToken = cancellationToken; - } - - public string Id => _entry.Id; - public ReadOnlyMemory Body => _entry.Body; - public MessageHeaders Headers => _entry.Headers; - public string? CorrelationId => Headers.GetValueOrDefault(KnownHeaders.CorrelationId); - public string? MessageType => Headers.GetValueOrDefault(KnownHeaders.MessageType); - public MessagePriority Priority => Enum.TryParse(Headers.GetValueOrDefault(KnownHeaders.Priority), ignoreCase: true, out MessagePriority priority) ? priority : MessagePriority.Normal; - public int Attempts => _entry.DeliveryCount; - public bool IsHandled => Volatile.Read(ref _isHandled) == 1; - public CancellationToken CancellationToken { get; } - - public Task CompleteAsync(CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) - return Task.CompletedTask; - - return _transport.CompleteAsync(_entry, cancellationToken); - } - - public Task AbandonAsync(CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) - return Task.CompletedTask; - - return _transport.AbandonAsync(_entry, cancellationToken); - } - - public async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) - return; - - if (_transport is ISupportsRedeliveryDelay redelivery) - { - await redelivery.AbandonAsync(_entry, redeliveryDelay, cancellationToken).AnyContext(); - return; - } - - if (_runtimeStore is null) - throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or {nameof(QueueOptions)}.{nameof(QueueOptions.RuntimeStore)}."); - - int nextAttempt = _entry.DeliveryCount + 1; - var headers = _entry.Headers.ToBuilder() - .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) - .Build(); - - await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState - { - DispatchId = $"{_entry.Id}:retry:{nextAttempt}", - Kind = ScheduledDispatchKind.QueueMessage, - Destination = _entry.Destination, - Body = _entry.Body, - Headers = headers, - Options = new TransportSendOptions { Priority = Priority }, - DueUtc = _timeProvider.GetUtcNow().Add(redeliveryDelay) - }, cancellationToken).AnyContext(); - - await _transport.CompleteAsync(_entry, cancellationToken).AnyContext(); - } - - public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) - return; - - await DeadLetterAsync(_transport, _entry, reason, cancellationToken).AnyContext(); - } - - public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) - { - return _transport is ISupportsLockRenewal lockRenewal - ? lockRenewal.RenewLockAsync(_entry, duration, cancellationToken) - : throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support lock renewal."); - } - - public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) - { - throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue or pub/sub messages."); - } - - internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) - { - if (transport is not ISupportsDeadLetter deadLetter) - throw new NotSupportedException($"Transport \"{transport.GetType().Name}\" does not support dead-lettering."); - - await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); - } - - private bool TryMarkHandled() - { - return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; - } -} - -internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class -{ - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) - : base(transport, entry, cancellationToken, runtimeStore, timeProvider) - { - Message = message; - } - - public T Message { get; } -} - -internal static class MessageRoutingConventions -{ - public static string ToKebabCase(string value) - { - if (String.IsNullOrEmpty(value)) - return value; - - Span buffer = stackalloc char[value.Length * 2]; - int position = 0; - for (int index = 0; index < value.Length; index++) - { - char current = value[index]; - if (Char.IsUpper(current)) - { - if (index > 0) - buffer[position++] = '-'; - - buffer[position++] = Char.ToLowerInvariant(current); - } - else - { - buffer[position++] = current; - } - } - - return new String(buffer[..position]); - } -} - -internal sealed class MessageConsumerHandle : IMessageConsumer -{ - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly Action _remove; - private IPushSubscription? _pushSubscription; - private Task? _worker; - private int _isDisposed; - - public MessageConsumerHandle(string source, string key, MessageListenerRegistration registration, Action remove) - { - Source = source; - Key = key; - Registration = registration; - _remove = remove; - } - - public string Source { get; } - public string Key { get; } - public MessageListenerRegistration Registration { get; } - public CancellationToken CancellationToken => _cancellationTokenSource.Token; - public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; - - public void ThrowIfConflicting(MessageListenerRegistration registration) - { - if (!Registration.Matches(registration)) - throw new InvalidOperationException($"A consumer with key \"{Key}\" is already registered with different handler or options."); - } - - public void SetPushSubscription(IPushSubscription subscription) - { - _pushSubscription = subscription; - } - - public void Start(Task worker) - { - _worker = worker; - } - - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - await _cancellationTokenSource.CancelAsync().AnyContext(); - - if (_pushSubscription is not null) - await _pushSubscription.DisposeAsync().AnyContext(); - - if (_worker is not null) - { - try - { - await _worker.AnyContext(); - } - catch (OperationCanceledException) { } - } - - _cancellationTokenSource.Dispose(); - _remove(Key, this); - } -} - -internal sealed record MessageListenerRegistration -{ - public required Type MessageType { get; init; } - public required string Source { get; init; } - public required Delegate Handler { get; init; } - public required AckMode AckMode { get; init; } - public required int MaxConcurrency { get; init; } - public required int MaxAttempts { get; init; } - public required bool HasRedeliveryBackoff { get; init; } - - public static MessageListenerRegistration Create(Delegate handler, Type messageType, string source, QueueConsumerOptions options) + private static MessageEnvelopeOptions ToEnvelope(QueueMessageOptions options) { - return new MessageListenerRegistration + return new MessageEnvelopeOptions { - MessageType = messageType, - Source = source, - Handler = handler, - AckMode = options.AckMode, - MaxConcurrency = Math.Max(1, options.MaxConcurrency), - MaxAttempts = options.MaxAttempts, - HasRedeliveryBackoff = options.RedeliveryBackoff is not null - }; - } - - public static MessageListenerRegistration Create(Delegate handler, Type messageType, string topic, string subscription, PubSubSubscriptionOptions options) - { - return new MessageListenerRegistration - { - MessageType = messageType, - Source = $"{topic}:{subscription}", - Handler = handler, - AckMode = options.AckMode, - MaxConcurrency = Math.Max(1, options.MaxConcurrency), - MaxAttempts = options.MaxAttempts, - HasRedeliveryBackoff = false + Priority = options.Priority, + Delay = options.Delay, + DeliverAt = options.DeliverAt, + TimeToLive = options.TimeToLive, + CorrelationId = options.CorrelationId, + DeduplicationId = options.DeduplicationId, + Headers = options.Headers }; } - - public bool Matches(MessageListenerRegistration other) - { - return MessageType == other.MessageType - && String.Equals(Source, other.Source, StringComparison.Ordinal) - && Handler == other.Handler - && AckMode == other.AckMode - && MaxConcurrency == other.MaxConcurrency - && MaxAttempts == other.MaxAttempts - && HasRedeliveryBackoff == other.HasRedeliveryBackoff; - } } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 9d4d9fbb8..4e0479c93 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -1,8 +1,5 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -35,6 +32,7 @@ public sealed record PubSubSubscriptionOptions public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; public int MaxAttempts { get; init; } = 5; + public Func? RedeliveryBackoff { get; init; } } public sealed record PubSubOptions @@ -65,91 +63,60 @@ public interface IMessageSubscription : IAsyncDisposable string Key { get; } } +/// +/// App-facing fan-out pub/sub. Routing, serialization, settlement, scheduling, and the subscription loop live in +/// ; this type maps topic/subscription-shaped options onto that shared core. +/// public sealed class PubSub : IPubSub { - private readonly IMessageTransport _transport; - private readonly PubSubOptions _options; - private readonly ILogger _logger; - private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); - private int _isDisposed; + private readonly MessageClientCore _core; public PubSub(IMessageTransport transport, PubSubOptions? options = null) { - _transport = transport ?? throw new ArgumentNullException(nameof(transport)); - _options = options ?? new PubSubOptions(); - _logger = (_options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + ArgumentNullException.ThrowIfNull(transport); + options ??= new PubSubOptions(); + var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner)); } - public async Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); - ThrowIfDisposed(); - options ??= new PubSubMessageOptions(); - ValidateSendOptions(options); - - string topic = GetTopic(typeof(T), options.Topic); - await EnsureTopicAsync(topic, cancellationToken).AnyContext(); - - var sendOptions = CreateSendOptions(options); - string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); - var transportMessage = CreateTransportMessage(message, typeof(T), options, messageId); - - if (await TryScheduleDispatchAsync(topic, transportMessage, sendOptions, cancellationToken).AnyContext()) - return; - - var result = await _transport.SendAsync(topic, [transportMessage], sendOptions, cancellationToken).AnyContext(); - var item = result.Items.Count > 0 ? result.Items[0] : null; - if (item is null || !item.Success) - throw new MessageBusException($"Unable to publish message to \"{topic}\": {item?.ErrorCode ?? "unknown error"}"); + return _core.SendAsync(ScheduledDispatchKind.PubSubMessage, typeof(T), message, ToEnvelope(options), GetTopic(typeof(T), options.Topic), EnsureTopicAsync, cancellationToken); } - public async Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); - await PublishBatchCoreAsync(messages.Cast(), typeof(T), options, cancellationToken).AnyContext(); + options ??= new PubSubMessageOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } - public async Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) + public Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(messages); - await PublishBatchCoreAsync(messages, null, options, cancellationToken).AnyContext(); + options ??= new PubSubMessageOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } public async Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(handler); options ??= new PubSubSubscriptionOptions(); - Type routeType = options.RouteType ?? typeof(object); - string topic = GetTopic(routeType, options.Topic); - string subscription = GetSubscription(routeType, topic, options.Subscription); - string key = GetSubscriptionKey(routeType, topic, subscription, options.Key); - var registration = MessageListenerRegistration.Create(handler, routeType, topic, subscription, options); - await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); - - return await SubscribeCoreAsync(topic, subscription, key, registration, options, async (entry, token) => - { - var received = CreateReceivedMessage(entry, token); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, cancellationToken).AnyContext(); + var config = BuildConfig(options.RouteType ?? typeof(object), options); + await EnsureSubscriptionAsync(config, cancellationToken).AnyContext(); + return await _core.StartListenerAsync(config, handler, cancellationToken).AnyContext(); } public async Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); options ??= new PubSubSubscriptionOptions(); - Type routeType = options.RouteType ?? typeof(T); - string topic = GetTopic(routeType, options.Topic); - string subscription = GetSubscription(routeType, topic, options.Subscription); - string key = GetSubscriptionKey(routeType, topic, subscription, options.Key); - var registration = MessageListenerRegistration.Create(handler, routeType, topic, subscription, options); - await EnsureSubscriptionAsync(topic, subscription, cancellationToken).AnyContext(); - - return await SubscribeCoreAsync(topic, subscription, key, registration, options, async (entry, token) => - { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); - await HandleMessageAsync(received, handler, options, token).AnyContext(); - }, cancellationToken).AnyContext(); + var config = BuildConfig(options.RouteType ?? typeof(T), options); + await EnsureSubscriptionAsync(config, cancellationToken).AnyContext(); + return await _core.StartListenerAsync(config, handler, cancellationToken).AnyContext(); } public async Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) @@ -164,341 +131,45 @@ public async Task RunSubscriptionAsync(Func, Cancellation await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); } - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - var subscriptions = _subscriptions.Values.ToArray(); - foreach (var subscription in subscriptions) - await subscription.DisposeAsync().AnyContext(); - - await _transport.DisposeAsync().AnyContext(); - } - - private async Task PublishBatchCoreAsync(IEnumerable messages, Type? declaredType, PubSubMessageOptions? options, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - - options ??= new PubSubMessageOptions(); - ValidateSendOptions(options); - - var sendOptions = CreateSendOptions(options); - var grouped = new Dictionary>(StringComparer.Ordinal); - int index = 0; - - foreach (var message in messages) - { - ArgumentNullException.ThrowIfNull(message); - Type messageType = declaredType ?? message.GetType(); - string topic = GetTopic(messageType, options.Topic); - string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; - index++; - - if (!grouped.TryGetValue(topic, out var transportMessages)) - { - transportMessages = []; - grouped.Add(topic, transportMessages); - } - - transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); - } - - foreach (var group in grouped) - { - await EnsureTopicAsync(group.Key, cancellationToken).AnyContext(); - - if (await TryScheduleDispatchesAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) - continue; - - var result = await _transport.SendAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); - if (!result.AllSucceeded) - throw new MessageBusException($"Unable to publish {result.Items.Count(i => !i.Success)} of {result.Items.Count} messages to \"{group.Key}\"."); - } - } - - private async Task SubscribeCoreAsync(string topic, string subscription, string key, MessageListenerRegistration registration, PubSubSubscriptionOptions options, Func onMessage, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - if (_subscriptions.TryGetValue(key, out var existing) && !existing.IsDisposed) - { - existing.ThrowIfConflicting(registration); - return existing; - } - - var handle = new MessageSubscriptionHandle(topic, subscription, key, registration, RemoveSubscription); - if (!_subscriptions.TryAdd(key, handle)) - { - await handle.DisposeAsync().AnyContext(); - var current = _subscriptions[key]; - current.ThrowIfConflicting(registration); - return current; - } - - try - { - if (_transport is ISupportsPush push) - { - var pushSubscription = await push.SubscribeAsync(subscription, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, options.MaxConcurrency) }, cancellationToken).AnyContext(); - - handle.SetPushSubscription(pushSubscription); - return handle; - } - - if (_transport is not ISupportsPull pull) - throw new MessageBusException($"Transport \"{_transport.GetType().Name}\" does not support subscriptions."); - - handle.Start(RunPullSubscriptionLoopAsync(subscription, pull, onMessage, options, handle.CancellationToken)); - return handle; - } - catch - { - await handle.DisposeAsync().AnyContext(); - throw; - } - } - - // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving - // or while processing a single entry (including a poison message that was already dead-lettered) must never tear - // down the subscription loop, otherwise one bad message or a transient transport blip silently stops delivery. - private async Task RunPullSubscriptionLoopAsync(string subscription, ISupportsPull pull, Func onMessage, PubSubSubscriptionOptions options, CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - IReadOnlyList entries; - try - { - entries = await pull.ReceiveAsync(subscription, new ReceiveRequest - { - MaxMessages = Math.Max(1, options.MaxConcurrency), - MaxWaitTime = TimeSpan.FromSeconds(1) - }, cancellationToken).AnyContext(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error receiving from subscription \"{Subscription}\"; retrying: {Message}", subscription, ex.Message); - await _options.TimeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); - continue; - } - - var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, subscription, cancellationToken)).ToArray(); - await Task.WhenAll(tasks).AnyContext(); - } - } - - private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string subscription, CancellationToken cancellationToken) - { - try - { - await onMessage(entry, cancellationToken).AnyContext(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - } - catch (Exception ex) - { - _logger.LogError(ex, "Error processing message \"{MessageId}\" from subscription \"{Subscription}\": {Message}", entry.Id, subscription, ex.Message); - } - } - - private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) - { - return new ReceivedMessage(_transport, entry, cancellationToken, _options.RuntimeStore, _options.TimeProvider); - } - - private async Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) - { - if (_transport is ISupportsProvisioning provisioning) - await provisioning.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken).AnyContext(); - } - - private async Task EnsureSubscriptionAsync(string topic, string subscription, CancellationToken cancellationToken) - { - if (_transport is ISupportsProvisioning provisioning) - { - await provisioning.EnsureAsync([ - new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = subscription, Role = DestinationRole.Subscription, Source = topic } - ], cancellationToken).AnyContext(); - } - } - - private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class - { - try - { - var message = _options.Serializer.Deserialize(entry.Body); - if (message is null) - throw new MessageBusException($"Message \"{entry.Id}\" deserialized to null."); - - return new ReceivedMessage(_transport, entry, message, cancellationToken, _options.RuntimeStore, _options.TimeProvider); - } - catch (Exception ex) when (ex is not MessageBusException) - { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); - throw new MessageBusException($"Unable to deserialize message \"{entry.Id}\".", ex); - } - catch (MessageBusException) - { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); - throw; - } - } - - private async Task HandleMessageAsync(IReceivedMessage message, Func handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) + public ValueTask DisposeAsync() { - try - { - await handler(message, cancellationToken).AnyContext(); - - if (options.AckMode == AckMode.Auto && !message.IsHandled) - await message.CompleteAsync(cancellationToken).AnyContext(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Subscriber failed for message \"{MessageId}\" from \"{Subscription}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, message.MessageType, message.Attempts, options.MaxAttempts, ex.Message); - await SettleFailedMessageAsync(message, options, cancellationToken).AnyContext(); - } + return _core.DisposeAsync(); } - private async Task HandleMessageAsync(IReceivedMessage message, Func, CancellationToken, Task> handler, PubSubSubscriptionOptions options, CancellationToken cancellationToken) where T : class + private ListenerConfig BuildConfig(Type routeType, PubSubSubscriptionOptions options) { - try - { - await handler(message, cancellationToken).AnyContext(); - - if (options.AckMode == AckMode.Auto && !message.IsHandled) - await message.CompleteAsync(cancellationToken).AnyContext(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Subscriber failed for message \"{MessageId}\" from \"{Subscription}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, message.MessageType, message.Attempts, options.MaxAttempts, ex.Message); - await SettleFailedMessageAsync(message, options, cancellationToken).AnyContext(); - } - } - - private static async Task SettleFailedMessageAsync(IReceivedMessage message, PubSubSubscriptionOptions options, CancellationToken cancellationToken) - { - if (message.IsHandled) - return; - - if (message.Attempts >= options.MaxAttempts) - { - await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); - return; - } - - await message.AbandonAsync(cancellationToken).AnyContext(); - } - - private TransportMessage CreateTransportMessage(object message, Type messageType, PubSubMessageOptions options, string? messageId = null) - { - // Content type is intentionally not written as a header: the receive path always uses the single configured - // serializer, so advertising a per-message content type would be misleading until real negotiation exists. - var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() - .Set(KnownHeaders.MessageType, GetMessageType(messageType)) - .Set(KnownHeaders.Priority, options.Priority.ToString()); - - if (!String.IsNullOrEmpty(options.CorrelationId)) - headers.Set(KnownHeaders.CorrelationId, options.CorrelationId); - - if (Activity.Current is { } activity) - { - if (!String.IsNullOrEmpty(activity.Id)) - headers.SetIfMissing(KnownHeaders.TraceParent, activity.Id); - - if (!String.IsNullOrEmpty(activity.TraceStateString)) - headers.SetIfMissing(KnownHeaders.TraceState, activity.TraceStateString); - } - - if (options.TimeToLive is { } ttl) - headers.Set(KnownHeaders.Expiration, _options.TimeProvider.GetUtcNow().Add(ttl).ToString("O", CultureInfo.InvariantCulture)); - - return new TransportMessage - { - Body = _options.Serializer.SerializeToBytes(message), - Headers = headers.Build(), - MessageId = messageId - }; - } - - private TransportSendOptions CreateSendOptions(PubSubMessageOptions options) - { - return new TransportSendOptions + string topic = GetTopic(routeType, options.Topic); + string subscription = GetSubscription(routeType, topic, options.Subscription); + return new ListenerConfig { - Priority = options.Priority, - DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _options.TimeProvider.GetUtcNow().Add(delay) : null), - DeduplicationId = options.DeduplicationId + Topic = topic, + Subscription = subscription, + Source = subscription, // a pub/sub consumer receives from its subscription destination + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{routeType.FullName ?? routeType.Name}", + MessageType = routeType, + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff }; } - private void ValidateSendOptions(PubSubMessageOptions options) - { - if (options.Priority != MessagePriority.Normal && _transport is not ISupportsPriority) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); - - if (options.TimeToLive is not null && _transport is not ISupportsExpiration) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); - } - - private async Task TryScheduleDispatchesAsync(string topic, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) - { - if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) - return false; - - for (int index = 0; index < messages.Count; index++) - { - var message = messages[index]; - string messageId = message.MessageId ?? Guid.NewGuid().ToString("N"); - await ScheduleDispatchAsync(topic, message with { MessageId = messageId }, options, dueUtc, cancellationToken).AnyContext(); - } - - return true; - } - - private Task TryScheduleDispatchAsync(string topic, TransportMessage message, TransportSendOptions options, CancellationToken cancellationToken) + private Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) { - return TryScheduleDispatchesAsync(topic, [message], options, cancellationToken); + return _core.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken); } - private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) + private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken cancellationToken) { - dueUtc = options.DeliverAt.GetValueOrDefault(); - if (options.DeliverAt is null || dueUtc <= _options.TimeProvider.GetUtcNow()) - return false; - - if (_transport is ISupportsDelayedDelivery) - return false; - - if (_options.RuntimeStore is null) - throw new MessageBusException($"Delayed publish requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or {nameof(PubSubOptions)}.{nameof(PubSubOptions.RuntimeStore)}."); - - return true; - } - - private Task ScheduleDispatchAsync(string topic, TransportMessage message, TransportSendOptions options, DateTimeOffset dueUtc, CancellationToken cancellationToken) - { - return _options.RuntimeStore!.ScheduleDispatchAsync(new ScheduledDispatchState - { - DispatchId = message.MessageId!, - Kind = ScheduledDispatchKind.PubSubMessage, - Destination = topic, - Body = message.Body, - Headers = message.Headers, - Options = options with { DeliverAt = null }, - DueUtc = dueUtc - }, cancellationToken); + return _core.EnsureAsync([ + new DestinationDeclaration { Name = config.Topic, Role = DestinationRole.Topic }, + new DestinationDeclaration { Name = config.Subscription, Role = DestinationRole.Subscription, Source = config.Topic } + ], cancellationToken); } private string GetTopic(Type messageType, string? topic) { - return _options.Router.ResolveRoute(new MessageRouteContext + return _core.Router.ResolveRoute(new MessageRouteContext { MessageType = messageType, Role = MessageRouteRole.PubSubTopic, @@ -508,7 +179,7 @@ private string GetTopic(Type messageType, string? topic) private string GetSubscription(Type messageType, string topic, string? subscription) { - return _options.Router.ResolveSubscription(new MessageSubscriptionContext + return _core.Router.ResolveSubscription(new MessageSubscriptionContext { MessageType = messageType, Topic = topic, @@ -516,94 +187,17 @@ private string GetSubscription(Type messageType, string topic, string? subscript }); } - private static string GetSubscriptionKey(Type messageType, string topic, string subscription, string? key) - { - return !String.IsNullOrEmpty(key) - ? key - : $"{topic}:{subscription}:{messageType.FullName ?? messageType.Name}"; - } - - private string GetMessageType(Type messageType) - { - return _options.Router.ResolveMessageType(messageType); - } - - private async Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) - { - await ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken).AnyContext(); - } - - private void RemoveSubscription(string key, MessageSubscriptionHandle handle) - { - _subscriptions.TryRemove(new KeyValuePair(key, handle)); - } - - private void ThrowIfDisposed() - { - ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); - } -} - -internal sealed class MessageSubscriptionHandle : IMessageSubscription -{ - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly Action _remove; - private IPushSubscription? _pushSubscription; - private Task? _worker; - private int _isDisposed; - - public MessageSubscriptionHandle(string topic, string subscription, string key, MessageListenerRegistration registration, Action remove) + private static MessageEnvelopeOptions ToEnvelope(PubSubMessageOptions options) { - Topic = topic; - Subscription = subscription; - Key = key; - Registration = registration; - _remove = remove; - } - - public string Topic { get; } - public string Subscription { get; } - public string Key { get; } - public MessageListenerRegistration Registration { get; } - public CancellationToken CancellationToken => _cancellationTokenSource.Token; - public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; - - public void ThrowIfConflicting(MessageListenerRegistration registration) - { - if (!Registration.Matches(registration)) - throw new InvalidOperationException($"A subscription with key \"{Key}\" is already registered with different handler or options."); - } - - public void SetPushSubscription(IPushSubscription subscription) - { - _pushSubscription = subscription; - } - - public void Start(Task worker) - { - _worker = worker; - } - - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 1) - return; - - await _cancellationTokenSource.CancelAsync().AnyContext(); - - if (_pushSubscription is not null) - await _pushSubscription.DisposeAsync().AnyContext(); - - if (_worker is not null) + return new MessageEnvelopeOptions { - try - { - await _worker.AnyContext(); - } - catch (OperationCanceledException) { } - } - - _cancellationTokenSource.Dispose(); - _remove(Key, this); + Priority = options.Priority, + Delay = options.Delay, + DeliverAt = options.DeliverAt, + TimeToLive = options.TimeToLive, + CorrelationId = options.CorrelationId, + DeduplicationId = options.DeduplicationId, + Headers = options.Headers + }; } } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 7432df1ad..168865a45 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -213,6 +213,26 @@ await transport.SendAsync("preview-work-item", [ Assert.Equal(0, handled.CurrentCount); } + [Fact] + public async Task EnqueueBatchAsync_RespectsTransportMaxBatchSizeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var transport = new BatchLimitTransport(maxBatchSize: 2); + await using var queue = new MessageQueue(transport); + + await queue.EnqueueBatchAsync(new[] + { + new PreviewWorkItem { Data = "1" }, + new PreviewWorkItem { Data = "2" }, + new PreviewWorkItem { Data = "3" }, + new PreviewWorkItem { Data = "4" }, + new PreviewWorkItem { Data = "5" } + }, cancellationToken: cancellationToken); + + // Five messages to one destination with MaxBatchSize=2 must be split into chunks of 2, 2, 1. + Assert.Equal(new[] { 2, 2, 1 }, transport.SendBatchSizes); + } + [Fact] public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() { @@ -515,4 +535,33 @@ private sealed class OtherWorkItem : IGroupedWorkItem { public string? Data { get; set; } } + + private sealed class BatchLimitTransport : IMessageTransport, ITransportInfo + { + public BatchLimitTransport(int maxBatchSize) + { + MaxBatchSize = maxBatchSize; + } + + public List SendBatchSizes { get; } = new(); + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; + public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; + public int? MaxBatchSize { get; } + public long? MaxMessageBytes => null; + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + SendBatchSizes.Add(messages.Count); + var items = new SendItemResult[messages.Count]; + for (int i = 0; i < messages.Count; i++) + items[i] = new SendItemResult { MessageId = messages[i].MessageId ?? Guid.NewGuid().ToString("N"), Success = true }; + + return Task.FromResult(new SendResult { Items = items }); + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } } \ No newline at end of file From f4c38a809337967600ed259ef47039727bd9958e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 28 Jun 2026 09:32:41 -0500 Subject: [PATCH 13/57] fix: address messaging/jobs design-review feedback (round 2) Close gaps surfaced in design review of the messaging/jobs redesign: - Prove redelivery-delay and lock-renewal: InMemoryMessageTransport now implements ISupportsRedeliveryDelay (timer-based re-enqueue that wakes a blocked receiver) and ISupportsLockRenewal (extends the in-flight visibility window), with conformance tests for both. - Make the attempt counter transport-independent: ReceivedMessage.Attempts reconciles DeliveryCount with the message.attempts header so store-backed redelivery can't reset the count and loop forever. - Real back-pressure: the pull loop is now a SemaphoreSlim-gated continuous dispatcher (per-message slot release, opportunistic batch claim) instead of a Task.WhenAll batch barrier, eliminating head-of-line blocking. - Core-owned metrics: foundatio.messaging.* and foundatio.jobs.* counters and histograms emitted on FoundatioDiagnostics.Meter. - Receive-side trace continuity: handlers run inside a Consumer Activity linked to the producer's traceparent/tracestate. - Configurable job cancellation polling (default 1s instead of fixed 50ms). - Document the IQueue namespace collision and the using-alias remedy. Add BasicQueueTransport test double (pull-only, opaque headers, no time-based capabilities) and repoint the unsupported-lock and redelivery-fallback tests at it, keeping fallback coverage and proving the attempt reconciliation end-to-end. Co-Authored-By: Claude Opus 4.8 --- docs/guide/messaging-jobs-redesign.md | 17 ++ .../MessageTransportConformanceTests.cs | 72 +++++++ src/Foundatio/Jobs/JobRuntime.cs | 42 +++- .../Messaging/InMemoryMessageTransport.cs | 87 +++++++- src/Foundatio/Messaging/MessageClientCore.cs | 190 +++++++++++++++--- .../InMemoryMessageTransportTests.cs | 12 ++ .../Queue/BasicQueueTransport.cs | 170 ++++++++++++++++ .../Queue/MessageQueueTests.cs | 9 +- 8 files changed, 570 insertions(+), 29 deletions(-) create mode 100644 tests/Foundatio.Tests/Queue/BasicQueueTransport.cs diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index dd7e2f784..7aebdbe71 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -194,6 +194,23 @@ await using var subscription = await pubsub.SubscribeAsync(Handl For per-type routing, register each type. For grouped routing, map an interface or base type. For default/global-style routing, set one default queue destination or topic for otherwise unmapped messages. Operation-level overrides should be reserved for exceptional paths such as replays or priority lanes. +### `IQueue` name collision during migration + +Two public `IQueue` types coexist while the legacy queue is still shipped: + +- `Foundatio.Queues.IQueue` / `IQueue` — the legacy one-type-per-queue API. +- `Foundatio.Messaging.IQueue` — the new app-facing queue. + +A file that has `using` directives for both namespaces will get a `CS0104` ambiguous-reference error on the bare name `IQueue`. Until the legacy API is removed, disambiguate per file with a `using` alias rather than fully qualifying every usage: + +```csharp +using IQueue = Foundatio.Messaging.IQueue; // new code +// or, while finishing a migration: +// using LegacyQueue = Foundatio.Queues.IQueue; +``` + +New application code should depend on `Foundatio.Messaging.IQueue`; the alias keeps call sites clean without dropping the legacy namespace a file may still need mid-migration. + ## Rollout Notes The in-memory transport proves the API shape and conformance coverage for local development. Before locking this as a stable public API, validate at least one external provider against the same routing, topic/subscription, delayed delivery, dead-letter, TTL, priority, and batch constraints. diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 393267a79..f52c3d535 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -371,6 +371,78 @@ public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() } } + public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull || transport is not ISupportsRedeliveryDelay redelivery) + { + Assert.Skip("Transport does not support pull receive with redelivery delay (ISupportsPull + ISupportsRedeliveryDelay)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "redelivery-delay", Role = DestinationRole.Queue }); + await transport.SendAsync("redelivery-delay", [CreateMessage("delay-me")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + Assert.Equal(1, first.DeliveryCount); + + await redelivery.AbandonAsync(first, TimeSpan.FromMilliseconds(300), TestCancellationToken); + + // Within the delay window the message must not be visible again. + var early = await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + Assert.Empty(early); + + // After the delay lapses it is redelivered with an incremented delivery count. + var second = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + Assert.Equal(first.Id, second.Id); + Assert.Equal(2, second.DeliveryCount); + Assert.Equal("delay-me", ReadBody(second)); + + await transport.CompleteAsync(second, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + + public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsVisibilityTimeout visibility || transport is not ISupportsLockRenewal lockRenewal) + { + Assert.Skip("Transport does not support visibility timeout with lock renewal (ISupportsVisibilityTimeout + ISupportsLockRenewal)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "lock-renewal", Role = DestinationRole.Queue }); + await transport.SendAsync("lock-renewal", [CreateMessage("hold")], new TransportSendOptions(), TestCancellationToken); + + var first = Assert.Single(await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(300), TestCancellationToken)); + Assert.Equal(1, first.DeliveryCount); + + // Renew before the original window lapses, extending it well past the original expiry. + await Task.Delay(TimeSpan.FromMilliseconds(150), TestCancellationToken); + await lockRenewal.RenewLockAsync(first, TimeSpan.FromSeconds(2), TestCancellationToken); + + // Past the original 300ms window but inside the renewed window: the message must still be held, so a + // competing receive sees nothing rather than a premature redelivery. + await Task.Delay(TimeSpan.FromMilliseconds(300), TestCancellationToken); + var held = await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TimeSpan.FromMilliseconds(300), TestCancellationToken); + Assert.Empty(held); + + await transport.CompleteAsync(first, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + public virtual async Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() { var transport = CreateTransport(); diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index a3100e5cc..65c40636e 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -9,6 +10,19 @@ namespace Foundatio.Jobs; +/// +/// Core-owned durable-job instruments, shared by every so job throughput and run latency are +/// observable independent of the runtime store implementation. +/// +internal static class JobInstruments +{ + public static readonly Counter Started = FoundatioDiagnostics.Meter.CreateCounter("foundatio.jobs.started", description: "Number of durable jobs started"); + public static readonly Counter Completed = FoundatioDiagnostics.Meter.CreateCounter("foundatio.jobs.completed", description: "Number of durable jobs completed successfully"); + public static readonly Counter Failed = FoundatioDiagnostics.Meter.CreateCounter("foundatio.jobs.failed", description: "Number of durable jobs that failed"); + public static readonly Counter Cancelled = FoundatioDiagnostics.Meter.CreateCounter("foundatio.jobs.cancelled", description: "Number of durable jobs that were cancelled"); + public static readonly Histogram RunTime = FoundatioDiagnostics.Meter.CreateHistogram("foundatio.jobs.runtime", unit: "ms", description: "Durable job execution time"); +} + public enum JobStatus { Queued, @@ -577,6 +591,7 @@ private static string Resolve() public sealed class JobWorker : IJobWorker { private static readonly TimeSpan DefaultLease = TimeSpan.FromMinutes(5); + private static readonly TimeSpan DefaultCancellationPollInterval = TimeSpan.FromSeconds(1); private readonly IJobRuntimeStore _store; private readonly IServiceProvider _serviceProvider; @@ -584,8 +599,9 @@ public sealed class JobWorker : IJobWorker private readonly IJobTypeRegistry _jobTypes; private readonly string _nodeId; private readonly TimeSpan _lease; + private readonly TimeSpan _cancellationPollInterval; - public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null) + public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null, TimeSpan? cancellationPollInterval = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); @@ -593,6 +609,12 @@ public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeP _jobTypes = jobTypes ?? new JobTypeRegistry(); _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _lease = lease ?? DefaultLease; + + // Cooperative cancellation is observed by polling the runtime store. The default is intentionally + // conservative (one poll per second per running job) so a real store isn't hammered when many jobs run + // concurrently; callers that need snappier cancellation can opt into a tighter interval. + var pollInterval = cancellationPollInterval ?? DefaultCancellationPollInterval; + _cancellationPollInterval = pollInterval > TimeSpan.Zero ? pollInterval : DefaultCancellationPollInterval; } public async Task RunQueuedAsync(int limit = 100, CancellationToken cancellationToken = default) @@ -638,6 +660,9 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc return false; } + var jobTag = new KeyValuePair("job", state.Name); + JobInstruments.Started.Add(1, jobTag); + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); using var cancellationWatcher = WatchCancellation(state.JobId, linkedCancellationTokenSource); using var leaseRenewer = RenewLeasePeriodically(state.JobId, linkedCancellationTokenSource); @@ -680,17 +705,28 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); } + if (result.IsCancelled) + JobInstruments.Cancelled.Add(1, jobTag); + else if (result.IsSuccess) + JobInstruments.Completed.Add(1, jobTag); + else + JobInstruments.Failed.Add(1, jobTag); + + JobInstruments.RunTime.Record((completedAt - now).TotalMilliseconds, jobTag); return true; } catch (Exception ex) { + var failedAt = _timeProvider.GetUtcNow(); await _store.TryTransitionAsync(state.JobId, JobStatus.Processing, JobStatus.Failed, new JobStatePatch { Error = ex.Message, - CompletedUtc = _timeProvider.GetUtcNow(), + CompletedUtc = failedAt, ClearNodeId = true, ClearLeaseExpiresUtc = true }, expectedNodeId: _nodeId, cancellationToken: CancellationToken.None).ConfigureAwait(false); + JobInstruments.Failed.Add(1, jobTag); + JobInstruments.RunTime.Record((failedAt - now).TotalMilliseconds, jobTag); throw; } } @@ -712,7 +748,7 @@ private Type ResolveJobType(JobState state) private IDisposable WatchCancellation(string jobId, CancellationTokenSource cancellationTokenSource) { - return new Timer(_ => _ = PollCancellationAsync(jobId, cancellationTokenSource), null, TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50)); + return new Timer(_ => _ = PollCancellationAsync(jobId, cancellationTokenSource), null, _cancellationPollInterval, _cancellationPollInterval); } private IDisposable RenewLeasePeriodically(string jobId, CancellationTokenSource cancellationTokenSource) diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index f134cd9da..3e97102ab 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -10,8 +10,10 @@ namespace Foundatio.Messaging; -public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo +public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo { + private static readonly TimeSpan _defaultLockRenewal = TimeSpan.FromMinutes(1); + private static readonly IReadOnlySet _supportedRoles = new HashSet { DestinationRole.Queue, @@ -23,6 +25,7 @@ public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, private readonly ConcurrentDictionary _destinations = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _roles = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary> _topicSubscriptions = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _redeliveryTimers = new(); private readonly TimeProvider _timeProvider; private readonly CancellationTokenSource _disposeCancellationTokenSource = new(); private int _isDisposed; @@ -160,6 +163,52 @@ public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) return Task.CompletedTask; } + public Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(entry); + + if (redeliveryDelay <= TimeSpan.Zero) + return AbandonAsync(entry, ct); + + var receipt = GetReceipt(entry); + var state = GetExistingDestination(receipt.Destination); + + if (!state.InFlight.TryRemove(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) + throw new ReceiptExpiredException(); + + Interlocked.Increment(ref state.Abandoned); + var redelivered = inFlight.Message with { DeliveryCount = entry.DeliveryCount + 1 }; + ScheduleRedelivery(receipt.Destination, redelivered, redeliveryDelay); + + return Task.CompletedTask; + } + + public Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + ArgumentNullException.ThrowIfNull(entry); + + var receipt = GetReceipt(entry); + var state = GetExistingDestination(receipt.Destination); + + if (!state.InFlight.TryGetValue(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) + throw new ReceiptExpiredException(); + + // Renewal only extends a finite visibility window. A message received without a window holds an indefinite + // lock, so there is nothing to extend — leave it as-is rather than imposing a window that could reclaim it. + if (inFlight.VisibilityExpiresUtc is null) + return Task.CompletedTask; + + var renewed = inFlight with { VisibilityExpiresUtc = _timeProvider.GetUtcNow().Add(duration ?? _defaultLockRenewal) }; + if (!state.InFlight.TryUpdate(receipt.LockToken, renewed, inFlight)) + throw new ReceiptExpiredException(); + + return Task.CompletedTask; + } + public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct) { ThrowIfDisposed(); @@ -311,6 +360,13 @@ public ValueTask DisposeAsync() _disposeCancellationTokenSource.Cancel(); _disposeCancellationTokenSource.Dispose(); + + foreach (var timer in _redeliveryTimers.Keys) + { + if (_redeliveryTimers.TryRemove(timer, out _)) + timer.Dispose(); + } + _destinations.Clear(); _roles.Clear(); _topicSubscriptions.Clear(); @@ -397,6 +453,35 @@ private void EnqueueStoredMessage(string destination, StoredMessage message) state.Enqueue(message with { Destination = destination }); } + // Make an abandoned message invisible for the redelivery delay, then re-enqueue it. Re-enqueueing releases the + // destination's availability semaphore, so a consumer blocked in a long receive wait wakes immediately when the + // message becomes due. The one-shot timer is tracked so it can be disposed if the transport is torn down first. + private void ScheduleRedelivery(string destination, StoredMessage message, TimeSpan delay) + { + ITimer? timer = null; + timer = _timeProvider.CreateTimer(timerState => + { + if (timer is not null && _redeliveryTimers.TryRemove(timer, out _)) + timer.Dispose(); + + if (Volatile.Read(ref _isDisposed) == 1) + return; + + try + { + EnqueueStoredMessage(destination, message); + } + catch (ObjectDisposedException) { } + catch (InvalidOperationException) { } // destination was deleted / completed between scheduling and firing + }, null, delay, Timeout.InfiniteTimeSpan); + + _redeliveryTimers[timer] = 0; + + // A redelivery scheduled right as the transport disposes could otherwise leak its timer; clean up the race. + if (Volatile.Read(ref _isDisposed) == 1 && _redeliveryTimers.TryRemove(timer, out _)) + timer.Dispose(); + } + private bool TryReceive(string source, DestinationState state, TimeSpan? visibility, out TransportEntry entry) { while (state.TryDequeue(out var message)) diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 304a30eac..079534c30 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Globalization; using System.Linq; using System.Threading; @@ -13,6 +14,21 @@ namespace Foundatio.Messaging; +/// +/// Core-owned messaging instruments. Counters and histograms are transport-agnostic and shared by every +/// and instance so that send/receive/settlement volume and handler +/// latency are observable regardless of which transport is plugged in. +/// +internal static class MessagingInstruments +{ + public static readonly Counter Sent = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.sent", description: "Number of messages sent to a destination"); + public static readonly Counter Received = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.received", description: "Number of messages received from a source"); + public static readonly Counter Completed = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.completed", description: "Number of messages completed"); + public static readonly Counter Abandoned = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.abandoned", description: "Number of messages abandoned"); + public static readonly Counter DeadLettered = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.deadlettered", description: "Number of messages dead-lettered"); + public static readonly Histogram HandlerTime = FoundatioDiagnostics.Meter.CreateHistogram("foundatio.messaging.handlertime", unit: "ms", description: "Message handler execution time"); +} + /// /// Transport-neutral envelope options shared by queue send and pub/sub publish operations. /// @@ -236,38 +252,99 @@ private async Task StartListenerCoreAsync(ListenerConfig } } - // MaxConcurrency bounds the number of in-flight messages processed per receive batch. A failure while receiving - // or while processing a single entry (including a poison message that was already dead-lettered) must never tear - // down the loop, otherwise one bad message or a transient transport blip silently stops consumption. + // MaxConcurrency bounds the number of in-flight messages. A slot is held from receive until the message settles + // and is released the instant that one message finishes — so a single slow message never stalls the other slots + // (no head-of-line blocking) and steady-state utilization stays at the configured concurrency. A failure while + // receiving or while processing a single entry (including a poison message that was already dead-lettered) must + // never tear down the loop, otherwise one bad message or a transient transport blip silently stops consumption. private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func onMessage, int maxConcurrency, CancellationToken cancellationToken) { - while (!cancellationToken.IsCancellationRequested) + maxConcurrency = Math.Max(1, maxConcurrency); + var slots = new SemaphoreSlim(maxConcurrency, maxConcurrency); + var inFlight = new ConcurrentDictionary(); + + try { - IReadOnlyList entries; - try + while (!cancellationToken.IsCancellationRequested) { - entries = await pull.ReceiveAsync(source, new ReceiveRequest + // Block for a free slot before receiving so we never pull more than we can process concurrently. + try { - MaxMessages = Math.Max(1, maxConcurrency), - MaxWaitTime = TimeSpan.FromSeconds(1) - }, cancellationToken).AnyContext(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error receiving from \"{Source}\"; retrying: {Message}", source, ex.Message); - await _timeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); - continue; + await slots.WaitAsync(cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + // Opportunistically claim any other idle slots so a transport that supports batch receive can still + // pull a batch while keeping per-message slot release. WaitAsync(Zero) is a non-blocking try-acquire. + int claimed = 1; + while (claimed < maxConcurrency && await slots.WaitAsync(TimeSpan.Zero).AnyContext()) + claimed++; + + IReadOnlyList entries; + try + { + entries = await pull.ReceiveAsync(source, new ReceiveRequest + { + MaxMessages = claimed, + MaxWaitTime = TimeSpan.FromSeconds(1) + }, cancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + ReleaseSlots(slots, claimed); + break; + } + catch (Exception ex) + { + ReleaseSlots(slots, claimed); + _logger.LogError(ex, "Error receiving from \"{Source}\"; retrying: {Message}", source, ex.Message); + await _timeProvider.SafeDelay(TimeSpan.FromSeconds(1), cancellationToken).AnyContext(); + continue; + } + + // Return any slots we claimed but didn't fill (empty receive or a partial batch). + ReleaseSlots(slots, claimed - entries.Count); + + foreach (var entry in entries) + { + var task = ProcessAndReleaseSlotAsync(entry, onMessage, source, slots, cancellationToken); + if (!task.IsCompleted) + { + inFlight[task] = 0; + _ = task.ContinueWith(static (t, state) => ((ConcurrentDictionary)state!).TryRemove(t, out _), inFlight, TaskScheduler.Default); + } + } } + } + finally + { + // Drain in-flight handlers before the semaphore is disposed so their slot releases never hit a disposed handle. + await Task.WhenAll(inFlight.Keys.ToArray()).AnyContext(); + slots.Dispose(); + } + } - var tasks = entries.Select(entry => SafeProcessAsync(entry, onMessage, source, cancellationToken)).ToArray(); - await Task.WhenAll(tasks).AnyContext(); + private async Task ProcessAndReleaseSlotAsync(TransportEntry entry, Func onMessage, string source, SemaphoreSlim slots, CancellationToken cancellationToken) + { + try + { + await SafeProcessAsync(entry, onMessage, source, cancellationToken).AnyContext(); + } + finally + { + slots.Release(); } } + private static void ReleaseSlots(SemaphoreSlim slots, int count) + { + if (count > 0) + slots.Release(count); + } + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string source, CancellationToken cancellationToken) { try @@ -287,6 +364,10 @@ private async Task SafeProcessAsync(TransportEntry entry, Func(TMessage message, ListenerConfig config, Func handler, CancellationToken cancellationToken) where TMessage : IReceivedMessage { + // Re-establish the producer's trace context on the consumer side so a cross-process trace continues here + // instead of breaking at the transport boundary. + using var activity = StartProcessActivity(message, config); + long startTimestamp = Stopwatch.GetTimestamp(); try { await handler(message, cancellationToken).AnyContext(); @@ -296,9 +377,36 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig } catch (Exception ex) { + activity?.SetErrorStatus(ex); _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, config.Source, message.Attempts, config.MaxAttempts, ex.Message); await SettleFailedMessageAsync(message, config, cancellationToken).AnyContext(); } + finally + { + MessagingInstruments.HandlerTime.Record(Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds, new KeyValuePair("source", config.Source)); + } + } + + private static Activity? StartProcessActivity(IReceivedMessage message, ListenerConfig config) + { + string? traceParent = message.Headers.GetValueOrDefault(KnownHeaders.TraceParent); + var activity = FoundatioDiagnostics.ActivitySource.StartActivity("ProcessMessage", ActivityKind.Consumer, traceParent); + if (activity is null) + return null; + + string? traceState = message.Headers.GetValueOrDefault(KnownHeaders.TraceState); + if (!String.IsNullOrEmpty(traceState)) + activity.TraceStateString = traceState; + + activity.DisplayName = $"Process: {message.MessageType ?? config.MessageType.Name}"; + + if (activity.IsAllDataRequested) + { + activity.SetTag("messaging.source", config.Source); + activity.SetTag("messaging.message.id", message.Id); + } + + return activity; } private static async Task SettleFailedMessageAsync(IReceivedMessage message, ListenerConfig config, CancellationToken cancellationToken) @@ -321,11 +429,14 @@ private static async Task SettleFailedMessageAsync(IReceivedMessage message, Lis private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) { + MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); return new ReceivedMessage(_transport, entry, cancellationToken, _runtimeStore, _timeProvider); } private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class { + MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); + T? message; try { @@ -448,6 +559,7 @@ private async Task> SendChunkedAsync(string destin if (maxBatchSize is not { } limit || limit <= 0 || messages.Count <= limit) { var result = await _transport.SendAsync(destination, messages, options, cancellationToken).AnyContext(); + RecordSent(destination, result.Items); return result.Items; } @@ -456,12 +568,26 @@ private async Task> SendChunkedAsync(string destin { var chunk = messages.Skip(offset).Take(limit).ToArray(); var result = await _transport.SendAsync(destination, chunk, options, cancellationToken).AnyContext(); + RecordSent(destination, result.Items); items.AddRange(result.Items); } return items; } + private static void RecordSent(string destination, IReadOnlyList items) + { + int sent = 0; + for (int index = 0; index < items.Count; index++) + { + if (items[index].Success) + sent++; + } + + if (sent > 0) + MessagingInstruments.Sent.Add(sent, new KeyValuePair("destination", destination)); + } + private ISupportsPull RequirePull() { return _transport as ISupportsPull @@ -507,7 +633,13 @@ public ReceivedMessage(IMessageTransport transport, TransportEntry entry, Cancel public string? CorrelationId => Headers.GetValueOrDefault(KnownHeaders.CorrelationId); public string? MessageType => Headers.GetValueOrDefault(KnownHeaders.MessageType); public MessagePriority Priority => Enum.TryParse(Headers.GetValueOrDefault(KnownHeaders.Priority), ignoreCase: true, out MessagePriority priority) ? priority : MessagePriority.Normal; - public int Attempts => _entry.DeliveryCount; + + // Reconcile the transport-reported delivery count with the message.attempts header. When redelivery-delay is + // served through the runtime-store fallback (transports without native ISupportsRedeliveryDelay), the message is + // re-sent as a brand-new transport message, so its DeliveryCount resets to 1; the carried-over attempt count + // lives in the header. Taking the max keeps MaxAttempts/dead-letter correct regardless of whether the transport + // honors the header, so the counter never silently resets and redelivery can't loop forever. + public int Attempts => Math.Max(_entry.DeliveryCount, ParseAttemptsHeader(_entry.Headers)); public bool IsHandled => Volatile.Read(ref _isHandled) == 1; public CancellationToken CancellationToken { get; } @@ -516,6 +648,7 @@ public Task CompleteAsync(CancellationToken cancellationToken = default) if (!TryMarkHandled()) return Task.CompletedTask; + MessagingInstruments.Completed.Add(1, new KeyValuePair("source", _entry.Destination)); return _transport.CompleteAsync(_entry, cancellationToken); } @@ -524,6 +657,7 @@ public Task AbandonAsync(CancellationToken cancellationToken = default) if (!TryMarkHandled()) return Task.CompletedTask; + MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination)); return _transport.AbandonAsync(_entry, cancellationToken); } @@ -532,6 +666,8 @@ public async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cance if (!TryMarkHandled()) return; + MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination)); + if (_transport is ISupportsRedeliveryDelay redelivery) { await redelivery.AbandonAsync(_entry, redeliveryDelay, cancellationToken).AnyContext(); @@ -565,6 +701,7 @@ public async Task DeadLetterAsync(string? reason = null, CancellationToken cance if (!TryMarkHandled()) return; + MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); await DeadLetterAsync(_transport, _entry, reason, cancellationToken).AnyContext(); } @@ -592,6 +729,13 @@ private bool TryMarkHandled() { return Interlocked.CompareExchange(ref _isHandled, 1, 0) == 0; } + + private static int ParseAttemptsHeader(MessageHeaders headers) + { + return Int32.TryParse(headers.GetValueOrDefault(KnownHeaders.Attempts), NumberStyles.Integer, CultureInfo.InvariantCulture, out int attempts) && attempts > 0 + ? attempts + : 0; + } } internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs index 75e6c58d8..fccdf77cf 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -99,6 +99,18 @@ public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() return base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); } + [Fact] + public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() + { + return base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); + } + + [Fact] + public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() + { + return base.RenewLockAsync_ExtendsVisibilityWindowAsync(); + } + [Fact] public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() { diff --git a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs new file mode 100644 index 000000000..4fd7ed484 --- /dev/null +++ b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; + +namespace Foundatio.Tests.Queue; + +/// +/// A deliberately minimal transport for tests: basic competing-consumer pull semantics only — no native redelivery +/// delay, lock renewal, visibility timeout, delayed delivery, priority, or expiration. Headers are treated as opaque +/// (preserved but never interpreted) and the delivery count is owned solely by the transport, so it never seeds the +/// count from the message.attempts header. This models a real provider that lacks time-based capabilities, +/// which exercises the runtime-store fallbacks and proves the core reconciles the attempt count itself rather than +/// relying on the transport to honor the header. +/// +internal sealed class BasicQueueTransport : IMessageTransport, ISupportsPull, ISupportsDeadLetter, ISupportsStats +{ + private readonly ConcurrentDictionary _destinations = new(StringComparer.OrdinalIgnoreCase); + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + var dest = _destinations.GetOrAdd(destination, static _ => new Destination()); + var results = new SendItemResult[messages.Count]; + for (int index = 0; index < messages.Count; index++) + { + var message = messages[index]; + string id = message.MessageId ?? options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + dest.Ready.Enqueue(new StoredEntry(id, message.Body, message.Headers, DeliveryCount: 1)); + Interlocked.Increment(ref dest.Enqueued); + results[index] = new SendItemResult { MessageId = id, Success = true }; + } + + return Task.FromResult(new SendResult { Items = results }); + } + + public async Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + { + var dest = _destinations.GetOrAdd(source, static _ => new Destination()); + int max = request.MaxMessages <= 0 ? 1 : request.MaxMessages; + DateTimeOffset? deadline = request.MaxWaitTime is { } wait && wait > TimeSpan.Zero ? DateTimeOffset.UtcNow.Add(wait) : null; + var entries = new List(max); + + while (true) + { + while (entries.Count < max && dest.Ready.TryDequeue(out var stored)) + { + string token = Guid.NewGuid().ToString("N"); + dest.InFlight[token] = stored; + Interlocked.Increment(ref dest.Dequeued); + entries.Add(new TransportEntry + { + Id = stored.Id, + Destination = source, + Body = stored.Body, + Headers = stored.Headers, + DeliveryCount = stored.DeliveryCount, + Receipt = new Receipt { TransportState = new BasicReceipt(source, token) } + }); + } + + if (entries.Count > 0 || deadline is null || DateTimeOffset.UtcNow >= deadline) + return entries; + + await Task.Delay(TimeSpan.FromMilliseconds(15), ct).ConfigureAwait(false); + } + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + var (dest, token) = Locate(entry); + if (!dest.InFlight.TryRemove(token, out _)) + throw new ReceiptExpiredException(); + + Interlocked.Increment(ref dest.Completed); + return Task.CompletedTask; + } + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + var (dest, token) = Locate(entry); + if (!dest.InFlight.TryRemove(token, out var stored)) + throw new ReceiptExpiredException(); + + Interlocked.Increment(ref dest.Abandoned); + dest.Ready.Enqueue(stored with { DeliveryCount = stored.DeliveryCount + 1 }); + return Task.CompletedTask; + } + + public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct) + { + var (dest, token) = Locate(entry); + if (!dest.InFlight.TryRemove(token, out var stored)) + throw new ReceiptExpiredException(); + + var headers = String.IsNullOrEmpty(reason) ? stored.Headers : stored.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); + dest.Dead.Enqueue(stored with { Headers = headers }); + return Task.CompletedTask; + } + + public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + { + var entries = new List(); + if (_destinations.TryGetValue(destination, out var dest)) + { + int max = request.MaxMessages <= 0 ? 1 : request.MaxMessages; + while (entries.Count < max && dest.Dead.TryDequeue(out var stored)) + { + entries.Add(new TransportEntry + { + Id = stored.Id, + Destination = destination, + Body = stored.Body, + Headers = stored.Headers, + DeliveryCount = stored.DeliveryCount, + Receipt = new Receipt { TransportState = null } + }); + } + } + + return Task.FromResult>(entries); + } + + public Task GetStatsAsync(string destination, CancellationToken ct) + { + if (!_destinations.TryGetValue(destination, out var dest)) + return Task.FromResult(new MessageDestinationStats()); + + return Task.FromResult(new MessageDestinationStats + { + Queued = dest.Ready.Count, + Working = dest.InFlight.Count, + Deadletter = dest.Dead.Count, + Enqueued = Interlocked.Read(ref dest.Enqueued), + Dequeued = Interlocked.Read(ref dest.Dequeued), + Completed = Interlocked.Read(ref dest.Completed), + Abandoned = Interlocked.Read(ref dest.Abandoned) + }); + } + + public ValueTask DisposeAsync() + { + _destinations.Clear(); + return ValueTask.CompletedTask; + } + + private (Destination Destination, string Token) Locate(TransportEntry entry) + { + if (entry.Receipt.TransportState is not BasicReceipt receipt || !_destinations.TryGetValue(receipt.Destination, out var dest)) + throw new ReceiptExpiredException(); + + return (dest, receipt.Token); + } + + private sealed record StoredEntry(string Id, ReadOnlyMemory Body, MessageHeaders Headers, int DeliveryCount); + + private sealed record BasicReceipt(string Destination, string Token); + + private sealed class Destination + { + public readonly ConcurrentQueue Ready = new(); + public readonly ConcurrentQueue Dead = new(); + public readonly ConcurrentDictionary InFlight = new(StringComparer.Ordinal); + public long Enqueued; + public long Dequeued; + public long Completed; + public long Abandoned; + } +} diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 168865a45..997e9fa40 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -97,7 +97,9 @@ public async Task AbandonAsync_RedeliversAsync() public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + // BasicQueueTransport intentionally does not implement ISupportsLockRenewal, so the core must surface the + // unsupported capability rather than silently no-op. + await using var queue = new MessageQueue(new BasicQueueTransport()); await queue.EnqueueAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); @@ -270,7 +272,10 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough { var cancellationToken = TestContext.Current.CancellationToken; var store = new InMemoryJobRuntimeStore(); - await using var transport = new InMemoryMessageTransport(); + // BasicQueueTransport lacks native redelivery delay, so the backoff routes through the runtime store. It also + // never seeds the delivery count from the message.attempts header, proving the core reconciles the attempt + // count from the header itself (second attempt must observe Attempts == 2, not a reset-to-1 loop). + await using var transport = new BasicQueueTransport(); await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); From a921f6753719d92919927b02e7e7a0a42eeb0014 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 28 Jun 2026 22:13:00 -0500 Subject: [PATCH 14/57] feat: core-owned retry/dead-letter, Reject settlement, and per-source type demux Address messaging/jobs design-review feedback on the transport-backed messaging API. The transport stays a thin set of primitives; the core owns serialization, routing, retry, and dead-lettering so behavior is identical across transports. Settlement: replace IReceivedMessage.AbandonAsync/DeadLetterAsync with a single RejectAsync(RejectOptions) verb (Terminal/Reason/RedeliveryDelay). Terminal reject moves the message to the transport's native dead-letter sink, else a configured RetryPolicy.DeadLetterDestination, else drops (honest at-most-once) instead of throwing. Consumers: one receive loop per source that demultiplexes by message type, so multiple typed consumers can share a destination without mis-dispatch; same-type consumers compete round-robin. An unmatched type increments foundatio.messaging.unhandled, throws UnhandledMessageTypeException (isolated per message so the loop and other handlers survive), retries, and dead-letters as "no-handler" after a lenient budget. Retry policy: add RetryPolicy (MaxAttempts/Backoff/DeadLetterDestination/UnmatchedMaxAttempts/UnmatchedBackoff), configurable via Messaging.ConfigureRetry and overridable per consumer. The broker delivery count is the crash-safe attempt counter; no broker-native redrive config. Capabilities: add MaxDeliveryDelay/MaxRedeliveryDelay/MaxVisibilityTimeout to the delay/visibility capability interfaces so an over-limit delay routes through the durable runtime store instead of being silently truncated by the broker. Docs: rewrite settlement section and add 'core owns behavior' + 'retry and dead-lettering' guidance. Add 7 tests covering cap-routing, Reject, multi-type demux, unmatched dead-letter, core-managed DLQ, and default-tier MaxAttempts. Co-Authored-By: Claude Opus 4.8 --- docs/guide/messaging-jobs-redesign.md | 69 ++- src/Foundatio/FoundatioServicesExtensions.cs | 17 + .../Messaging/InMemoryMessageTransport.cs | 4 + src/Foundatio/Messaging/MessageClientCore.cs | 494 +++++++++++++----- src/Foundatio/Messaging/MessageQueue.cs | 74 ++- src/Foundatio/Messaging/MessageTransport.cs | 22 +- src/Foundatio/Messaging/PubSub.cs | 6 +- .../Queue/MessageQueueTests.cs | 257 ++++++++- 8 files changed, 794 insertions(+), 149 deletions(-) diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index 7aebdbe71..ddf8b7867 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -4,6 +4,12 @@ The new messaging API is app-facing and type-driven. Queue, pub/sub, received-me Provider-facing transport contracts such as `IMessageTransport`, `ISupportsPull`, `ISupportsPush`, and `ISupportsDeadLetter` are still public so external providers can implement them. They are infrastructure contracts, not the primary application surface. +## The core owns behavior; transports stay simple + +The division of responsibility is deliberate: **the core owns behavior, transports stay thin.** A transport is bytes in, bytes out plus a few primitives — send, receive, complete, abandon, and (optionally) a dead-letter sink. Everything that defines *how messaging behaves* — serialization, content types, routing, multi-type dispatch, priority, back-pressure, tracing, metrics, and especially **retry and dead-lettering** — lives in the core and is therefore identical across every transport. A provider only advertises which primitives it supports through small capability interfaces (`ISupportsPull`, `ISupportsPush`, `ISupportsDeadLetter`, `ISupportsDelayedDelivery`, `ISupportsRedeliveryDelay`, …); it never owns policy. + +This keeps providers small and hard to get subtly wrong, and keeps behavior portable: code verified against the in-memory transport behaves the same on a real broker. It also avoids a split-brain retry model — there is exactly one authority (the core), never a tug-of-war between the core's `MaxAttempts` and a broker-native redrive policy. See [Retry and dead-lettering](#retry-and-dead-lettering). + ## Setup Register the in-memory messaging transport, central routing policy, and durable job runtime through DI: @@ -64,6 +70,18 @@ await using IMessageConsumer consumer = await queue.StartConsumerAsync(HandleSubmittedAsync); +await using var cancelled = await queue.StartConsumerAsync(HandleCancelledAsync); +// One loop on the shared destination. OrderSubmitted is dispatched to the first handler, OrderCancelled to the second. +``` + +Consumers that share a message type compete: each message is dispatched to one of them, round-robin. The non-generic `StartConsumerAsync` (or a consumer whose route type is an interface/base type) is a catch-all that receives any type no exact-typed consumer claimed — the grouped/raw-envelope path. All consumers on one destination must agree on `MaxConcurrency` (it is a property of the shared loop). A message whose type has **no** registered consumer on this node is handled loudly — see [Unmatched message types](#unmatched-message-types). + ## Pub/Sub Pub/sub follows the same type-driven publishing pattern: @@ -135,17 +153,56 @@ await topology.ValidateAsync(); // app startup check without creating destinatio ## Delivery Settlement -Received messages use explicit settlement verbs for both queue and pub/sub: +Received messages settle with two verbs — the same for queue and pub/sub: ```csharp -await message.CompleteAsync(); -await message.AbandonAsync(); -await message.DeadLetterAsync("validation"); +await message.CompleteAsync(); // handled successfully +await message.RejectAsync(); // retry, transport-timed redelivery +await message.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.FromSeconds(30) }); // retry after a delay +await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }); // do not retry await message.RenewLockAsync(); -await message.ReportProgressAsync(50, "half"); ``` -Unsupported capabilities fail clearly with `NotSupportedException` or a validation exception. There are no silent no-ops for dead-lettering, lock renewal, progress, priority, expiration, or delayed delivery. +`RejectAsync` replaces the separate abandon and dead-letter verbs. A non-terminal reject returns the message for redelivery (optionally after `RedeliveryDelay`); `Terminal = true` means "never redeliver" and routes the message to the dead-letter sink, falling back to a configured destination or a drop (see below). + +Auto-ack is the default: a handler that returns without settling is completed automatically, and a handler that throws is rejected according to the [retry policy](#retry-and-dead-lettering). Manual ack is opt-in with `AckMode.Manual` on the consumer options. + +Unsupported capabilities fail clearly with `NotSupportedException` or a validation exception — there are no silent no-ops for lock renewal, priority, expiration, or delayed delivery. Terminal reject is the one deliberate exception: a transport with no dead-letter sink does not throw, it drops the message (at-most-once for terminal messages), which is the honest behavior for an ack-less/broadcast transport. + +## Retry and dead-lettering + +The core owns retry and dead-lettering, so behavior is identical on every transport (see [The core owns behavior](#the-core-owns-behavior-transports-stay-simple)). A transport only has to redeliver an abandoned message and, optionally, expose a dead-letter sink; the core decides how many times to retry, how long to wait between attempts, and when to give up. The broker's own delivery count is used as a crash-safe attempt counter, so the core owns the *policy* without owning durable retry *state*. + +Configure a default policy and override it per consumer: + +```csharp +services.AddFoundatio() + .Messaging.ConfigureRetry(r => r with { + MaxAttempts = 5, + Backoff = attempt => TimeSpan.FromSeconds(Math.Min(30, Math.Pow(2, attempt))), + DeadLetterDestination = "orders-dead-letter" + }); + +await queue.StartConsumerAsync(HandleAsync, new QueueConsumerOptions { + MaxAttempts = 10 // per-consumer override; null inherits the default policy +}); +``` + +When a handler throws, the message is retried (abandoned for redelivery, with the configured backoff) until `MaxAttempts` is reached, then dead-lettered. Where a dead-lettered message lands, in order of preference: + +1. the transport's native dead-letter sink, when it has one (`ISupportsDeadLetter`) — preserving native DLQ tooling; +2. otherwise the configured `RetryPolicy.DeadLetterDestination`, which the core writes to directly (a normal queue on the same transport), recording the reason in the `message.dead_letter.reason` header; +3. otherwise the message is dropped (at-most-once) — the honest outcome when there is nowhere durable to park it. + +We deliberately do **not** configure broker-native redrive policies (SQS `maxReceiveCount`, Azure Service Bus `MaxDeliveryCount`, RabbitMQ DLX). That would split authority between the broker and the core and make behavior transport-specific. The core is always authoritative; transports stay simple. A destination's structural creation knobs, if any, are limited to `DestinationDeclaration.ProviderArguments`. + +### Delayed redelivery and capability bounds + +An explicit `RedeliveryDelay` (or a configured `Backoff`) is served natively when the transport supports it within its advertised limit — `ISupportsRedeliveryDelay.MaxRedeliveryDelay` and `ISupportsDelayedDelivery.MaxDeliveryDelay`. A delay longer than the broker can honor — for example beyond SQS's 15-minute delivery delay or 12-hour visibility window — is routed through the durable job runtime store instead of being silently truncated. If neither native support nor a runtime store is available, the operation fails loudly rather than dropping the delay. + +### Unmatched message types + +A message that arrives on a destination but whose type has no registered consumer on this node — for example a newer message type during a rolling deploy, before every node has been updated — is surfaced loudly rather than quietly swallowed. It increments the `foundatio.messaging.unhandled` metric and throws `UnhandledMessageTypeException`, isolated to that one message so the receive loop and the other type handlers keep running. The message is retried so a node that *does* handle the type can pick it up, and is finally dead-lettered as `"no-handler"` once `RetryPolicy.UnmatchedMaxAttempts` (default 50) is exhausted — so a genuinely orphaned type cannot loop forever. ## Jobs diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 85c5893be..24c87c85c 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -281,6 +281,21 @@ public MessagingBuilder ConfigureRouting(Action co return this; } + // The core owns retry and dead-letter behavior so it is identical across transports. This configures the + // default policy applied to queue and pub/sub consumers; a consumer can still override MaxAttempts/backoff. + public MessagingBuilder ConfigureRetry(RetryPolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + _services.ReplaceSingleton(_ => policy); + return this; + } + + public MessagingBuilder ConfigureRetry(Func configure) + { + ArgumentNullException.ThrowIfNull(configure); + return ConfigureRetry(configure(new RetryPolicy())); + } + public FoundatioBuilder UseInMemory(InMemoryMessageBusOptions? options = null) { _services.ReplaceSingleton(sp => new InMemoryMessageBus(options.UseServices(sp))); @@ -364,6 +379,7 @@ private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), + RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, LoggerFactory = serviceProvider.GetService() }; @@ -376,6 +392,7 @@ private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvide Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, RuntimeStore = serviceProvider.GetService(), + RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, LoggerFactory = serviceProvider.GetService() }; diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 3e97102ab..f481e1a40 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -41,6 +41,10 @@ public InMemoryMessageTransport(TimeProvider? timeProvider = null) public int? MaxBatchSize => null; public long? MaxMessageBytes => null; + // The in-memory transport has no broker-imposed ceiling on visibility or redelivery delay. + public TimeSpan? MaxVisibilityTimeout => null; + public TimeSpan? MaxRedeliveryDelay => null; + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { ThrowIfDisposed(); diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 079534c30..164e0bd8c 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -26,6 +26,7 @@ internal static class MessagingInstruments public static readonly Counter Completed = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.completed", description: "Number of messages completed"); public static readonly Counter Abandoned = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.abandoned", description: "Number of messages abandoned"); public static readonly Counter DeadLettered = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.deadlettered", description: "Number of messages dead-lettered"); + public static readonly Counter Unhandled = FoundatioDiagnostics.Meter.CreateCounter("foundatio.messaging.unhandled", description: "Number of received messages with no registered consumer for their type"); public static readonly Histogram HandlerTime = FoundatioDiagnostics.Meter.CreateHistogram("foundatio.messaging.handlertime", unit: "ms", description: "Message handler execution time"); } @@ -55,7 +56,8 @@ internal sealed record ListenerConfig public string Subscription { get; init; } = ""; public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; - public int MaxAttempts { get; init; } = 5; + // Null falls back to the client's default RetryPolicy. + public int? MaxAttempts { get; init; } public Func? RedeliveryBackoff { get; init; } } @@ -73,11 +75,12 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly Func _exceptionFactory; - private readonly ConcurrentDictionary _listeners = new(StringComparer.Ordinal); + private readonly RetryPolicy _retryPolicy; + private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, - IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory) + IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _serializer = serializer; @@ -86,6 +89,7 @@ public MessageClientCore(IMessageTransport transport, ISerializer serializer, IM _timeProvider = timeProvider; _logger = logger; _exceptionFactory = exceptionFactory; + _retryPolicy = retryPolicy ?? new RetryPolicy(); } public IMessageRouter Router => _router; @@ -180,8 +184,7 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(handler); - var registration = MessageListenerRegistration.Create(handler, config); - return StartListenerCoreAsync(config, registration, async (entry, token) => + return RegisterConsumerAsync(config, handler, async (entry, token) => { var received = CreateReceivedMessage(entry, token); await HandleMessageAsync(received, config, handler, token).AnyContext(); @@ -191,8 +194,7 @@ public Task StartListenerAsync(ListenerConfig config, Fun public Task StartListenerAsync(ListenerConfig config, Func, CancellationToken, Task> handler, CancellationToken cancellationToken) where T : class { ArgumentNullException.ThrowIfNull(handler); - var registration = MessageListenerRegistration.Create(handler, config); - return StartListenerCoreAsync(config, registration, async (entry, token) => + return RegisterConsumerAsync(config, handler, async (entry, token) => { var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); await HandleMessageAsync(received, config, handler, token).AnyContext(); @@ -204,54 +206,79 @@ public async ValueTask DisposeAsync() if (Interlocked.Exchange(ref _isDisposed, 1) == 1) return; - foreach (var listener in _listeners.Values.ToArray()) + foreach (var listener in _sources.Values.ToArray()) await listener.DisposeAsync().AnyContext(); await _transport.DisposeAsync().AnyContext(); } - private async Task StartListenerCoreAsync(ListenerConfig config, MessageListenerRegistration registration, Func onMessage, CancellationToken cancellationToken) + // Multiple typed consumers can share one destination. They attach to a single per-source listener whose loop + // demultiplexes each message to the consumer registered for its type; a type with no registered consumer is + // handled by HandleUnmatchedAsync. Starting the same consumer key with a matching handler/options is idempotent. + private async Task RegisterConsumerAsync(ListenerConfig config, Delegate handler, Func dispatch, CancellationToken cancellationToken) { ThrowIfDisposed(); cancellationToken.ThrowIfCancellationRequested(); - if (_listeners.TryGetValue(config.Key, out var existing) && !existing.IsDisposed) + bool catchAll = IsCatchAll(config.MessageType); + var registration = new ConsumerRegistration { - existing.ThrowIfConflicting(registration); - return existing; - } - - var handle = new MessageListenerHandle(config.Topic, config.Subscription, config.Source, config.Key, registration, RemoveListener); - if (!_listeners.TryAdd(config.Key, handle)) - { - await handle.DisposeAsync().AnyContext(); - var current = _listeners[config.Key]; - current.ThrowIfConflicting(registration); - return current; - } + Key = config.Key, + Config = config, + Dispatch = dispatch, + Info = MessageListenerRegistration.Create(handler, config), + IsCatchAll = catchAll, + TypeName = catchAll ? null : _router.ResolveMessageType(config.MessageType) + }; - try + while (true) { - if (_transport is ISupportsPush push) + var listener = _sources.GetOrAdd(config.Source, source => new SourceListener(this, source)); + if (listener.TryAddConsumer(registration, out var handle, out bool created)) { - var subscription = await push.SubscribeAsync(config.Source, onMessage, new PushOptions { MaxConcurrentMessages = Math.Max(1, config.MaxConcurrency) }, cancellationToken).AnyContext(); - handle.SetPushSubscription(subscription); + if (created) + { + try + { + await listener.StartAsync(cancellationToken).AnyContext(); + } + catch + { + await listener.DisposeAsync().AnyContext(); + throw; + } + } + return handle; } - if (_transport is not ISupportsPull pull) - throw _exceptionFactory($"Transport \"{_transport.GetType().Name}\" does not support receiving messages.", null); - - handle.Start(RunPullLoopAsync(config.Source, pull, onMessage, config.MaxConcurrency, handle.CancellationToken)); - return handle; - } - catch - { - await handle.DisposeAsync().AnyContext(); - throw; + // The listener was disposing as its last consumer detached; drop our stale reference and retry. + _sources.TryRemove(new KeyValuePair(config.Source, listener)); } } + // A concrete message type binds an exact-type consumer; object/interface/abstract route types are catch-alls that + // receive every message a more specific typed consumer did not claim (the grouped/raw-envelope path). + private static bool IsCatchAll(Type messageType) + { + return messageType == typeof(object) || messageType.IsInterface || messageType.IsAbstract; + } + + private async Task HandleUnmatchedAsync(TransportEntry entry, string source, CancellationToken cancellationToken) + { + MessagingInstruments.Unhandled.Add(1, new KeyValuePair("source", source)); + + var message = CreateReceivedMessage(entry, cancellationToken); + + // Retry so a node that does handle this type can pick it up; dead-letter as "no-handler" once the lenient + // budget is exhausted so a genuinely orphaned type cannot loop forever. + await SettleFailedMessageAsync(message, _retryPolicy.UnmatchedMaxAttempts, _retryPolicy.UnmatchedBackoff, deadLetterReason: "no-handler", cancellationToken).AnyContext(); + + // Surface loudly. The throw is caught by the loop's per-message handling (SafeProcessAsync), so it never tears + // down the receive loop or the other type handlers sharing this source. + throw new UnhandledMessageTypeException(message.MessageType, source); + } + // MaxConcurrency bounds the number of in-flight messages. A slot is held from receive until the message settles // and is released the instant that one message finishes — so a single slow message never stalls the other slots // (no head-of-line blocking) and steady-state utilization stays at the configured concurrency. A failure while @@ -378,8 +405,10 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig catch (Exception ex) { activity?.SetErrorStatus(ex); - _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, config.Source, message.Attempts, config.MaxAttempts, ex.Message); - await SettleFailedMessageAsync(message, config, cancellationToken).AnyContext(); + int maxAttempts = config.MaxAttempts ?? _retryPolicy.MaxAttempts; + var backoff = config.RedeliveryBackoff ?? _retryPolicy.Backoff; + _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); + await SettleFailedMessageAsync(message, maxAttempts, backoff, "handler-error", cancellationToken).AnyContext(); } finally { @@ -409,28 +438,21 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig return activity; } - private static async Task SettleFailedMessageAsync(IReceivedMessage message, ListenerConfig config, CancellationToken cancellationToken) + private static Task SettleFailedMessageAsync(IReceivedMessage message, int maxAttempts, Func? backoff, string deadLetterReason, CancellationToken cancellationToken) { if (message.IsHandled) - return; + return Task.CompletedTask; - if (message.Attempts >= config.MaxAttempts) - { - await message.DeadLetterAsync("handler-error", cancellationToken).AnyContext(); - return; - } + if (message.Attempts >= maxAttempts) + return message.RejectAsync(new RejectOptions { Terminal = true, Reason = deadLetterReason }, cancellationToken); - TimeSpan? redeliveryDelay = config.RedeliveryBackoff?.Invoke(message.Attempts); - if (redeliveryDelay is { } delay && delay > TimeSpan.Zero && message is ISupportsDelayedMessageAbandon received) - await received.AbandonAsync(delay, cancellationToken).AnyContext(); - else - await message.AbandonAsync(cancellationToken).AnyContext(); + return message.RejectAsync(new RejectOptions { RedeliveryDelay = backoff?.Invoke(message.Attempts) }, cancellationToken); } private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); - return new ReceivedMessage(_transport, entry, cancellationToken, _runtimeStore, _timeProvider); + return new ReceivedMessage(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); } private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class @@ -454,12 +476,13 @@ private async Task> CreateReceivedMessageAsync(TransportE throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); } - return new ReceivedMessage(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider); + return new ReceivedMessage(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); } private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) { - return ReceivedMessage.DeadLetterAsync(_transport, entry, reason, cancellationToken); + MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination)); + return ReceivedMessage.DeadLetterOrDropAsync(_transport, entry, reason, _retryPolicy.DeadLetterDestination, cancellationToken); } public string ResolveMessageType(Type messageType) => _router.ResolveMessageType(messageType); @@ -540,14 +563,18 @@ private async Task TryScheduleAsync(ScheduledDispatchKind kind, string des private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) { dueUtc = options.DeliverAt.GetValueOrDefault(); - if (options.DeliverAt is null || dueUtc <= _timeProvider.GetUtcNow()) + var now = _timeProvider.GetUtcNow(); + if (options.DeliverAt is null || dueUtc <= now) return false; - if (_transport is ISupportsDelayedDelivery) + // A transport can deliver natively only up to its advertised maximum; a delay longer than the broker supports + // (e.g. SQS caps DelaySeconds at 15 minutes) must route through the durable runtime store rather than be + // silently truncated to the broker's ceiling. + if (_transport is ISupportsDelayedDelivery delayed && (delayed.MaxDeliveryDelay is not { } max || dueUtc - now <= max)) return false; if (_runtimeStore is null) - throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" or a registered job runtime store.", null); + throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store.", null); return true; } @@ -594,36 +621,258 @@ private ISupportsPull RequirePull() ?? throw _exceptionFactory($"Transport \"{_transport.GetType().Name}\" does not support pull receive.", null); } - private void RemoveListener(string key, MessageListenerHandle handle) + private void RemoveSource(string source, SourceListener listener) { - _listeners.TryRemove(new KeyValuePair(key, handle)); + _sources.TryRemove(new KeyValuePair(source, listener)); } private void ThrowIfDisposed() { ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); } -} -internal interface ISupportsDelayedMessageAbandon -{ - Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default); + private sealed class ConsumerRegistration + { + public required string Key { get; init; } + public required ListenerConfig Config { get; init; } + public required Func Dispatch { get; init; } + public required MessageListenerRegistration Info { get; init; } + public required bool IsCatchAll { get; init; } + public required string? TypeName { get; init; } + } + + // One receive loop per source. Consumers register by message type; the loop reads the message-type header and + // dispatches each entry to a consumer for that type (round-robin when several share a type, so same-type consumers + // compete), to the catch-all group for unmapped types, or to HandleUnmatchedAsync when nothing claims the type. + // The loop runs while at least one consumer is attached and shuts down when the last one detaches. + private sealed class SourceListener + { + private readonly MessageClientCore _core; + private readonly string _source; + private readonly object _lock = new(); + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly ConcurrentDictionary _consumers = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _byType = new(StringComparer.Ordinal); + private readonly ConsumerGroup _catchAll = new(); + private int _maxConcurrency = 1; + private IPushSubscription? _pushSubscription; + private Task? _loop; + private bool _isDisposed; + + public SourceListener(MessageClientCore core, string source) + { + _core = core; + _source = source; + } + + public bool TryAddConsumer(ConsumerRegistration registration, out MessageListenerHandle handle, out bool created) + { + handle = null!; + created = false; + + lock (_lock) + { + if (_isDisposed) + return false; + + if (_consumers.TryGetValue(registration.Key, out var existing)) + { + if (!existing.Registration.Info.Matches(registration.Info)) + throw new InvalidOperationException($"A consumer with key \"{registration.Key}\" is already registered with a different handler or options."); + + handle = existing.Handle; // idempotent re-registration + return true; + } + + int desired = Math.Max(1, registration.Config.MaxConcurrency); + if (_consumers.IsEmpty) + { + _maxConcurrency = desired; + created = true; + } + else if (desired != _maxConcurrency) + { + throw new InvalidOperationException($"Source \"{_source}\" is already consumed with MaxConcurrency {_maxConcurrency}; a conflicting MaxConcurrency {desired} was requested. Consumers sharing a destination must use the same MaxConcurrency."); + } + + handle = new MessageListenerHandle(registration.Config.Topic, registration.Config.Subscription, _source, registration.Key, () => RemoveConsumerAsync(registration.Key)); + _consumers[registration.Key] = new Registered(registration, handle); + GroupFor(registration).Add(registration); + + return true; + } + } + + private ConsumerGroup GroupFor(ConsumerRegistration registration) + { + return registration.IsCatchAll ? _catchAll : _byType.GetOrAdd(registration.TypeName!, _ => new ConsumerGroup()); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (_core._transport is ISupportsPush push) + { + // Route the push callback through SafeProcessAsync so a throw (including an unmatched-type throw) is + // isolated to the message and never tears down the subscription. + _pushSubscription = await push.SubscribeAsync(_source, (entry, token) => _core.SafeProcessAsync(entry, DispatchAsync, _source, token), new PushOptions { MaxConcurrentMessages = Math.Max(1, _maxConcurrency) }, cancellationToken).AnyContext(); + return; + } + + if (_core._transport is not ISupportsPull pull) + throw _core._exceptionFactory($"Transport \"{_core._transport.GetType().Name}\" does not support receiving messages.", null); + + _loop = _core.RunPullLoopAsync(_source, pull, DispatchAsync, _maxConcurrency, _cancellationTokenSource.Token); + } + + public async ValueTask DisposeAsync() + { + lock (_lock) + { + if (_isDisposed) + return; + + _isDisposed = true; + } + + await ShutdownAsync().AnyContext(); + } + + private async ValueTask RemoveConsumerAsync(string key) + { + bool shutdown = false; + lock (_lock) + { + if (!_consumers.TryRemove(key, out var registered)) + return; + + var registration = registered.Registration; + if (registration.IsCatchAll) + { + _catchAll.Remove(registration); + } + else if (registration.TypeName is { } typeName && _byType.TryGetValue(typeName, out var group)) + { + group.Remove(registration); + if (group.IsEmpty) + _byType.TryRemove(new KeyValuePair(typeName, group)); + } + + if (_consumers.IsEmpty && !_isDisposed) + { + _isDisposed = true; + shutdown = true; + } + } + + if (shutdown) + await ShutdownAsync().AnyContext(); + } + + private async Task ShutdownAsync() + { + await _cancellationTokenSource.CancelAsync().AnyContext(); + + if (_pushSubscription is not null) + await _pushSubscription.DisposeAsync().AnyContext(); + + if (_loop is not null) + { + try + { + await _loop.AnyContext(); + } + catch (OperationCanceledException) { } + } + + _cancellationTokenSource.Dispose(); + _core.RemoveSource(_source, this); + } + + private async Task DispatchAsync(TransportEntry entry, CancellationToken token) + { + var registration = Resolve(entry); + if (registration is null) + { + await _core.HandleUnmatchedAsync(entry, _source, token).AnyContext(); + return; + } + + await registration.Dispatch(entry, token).AnyContext(); + } + + private ConsumerRegistration? Resolve(TransportEntry entry) + { + string? typeName = entry.Headers.GetValueOrDefault(KnownHeaders.MessageType); + if (typeName is not null && _byType.TryGetValue(typeName, out var group) && group.Next() is { } typed) + return typed; + + return _catchAll.Next(); + } + + private sealed record Registered(ConsumerRegistration Registration, MessageListenerHandle Handle); + + // Consumers sharing a message type (or the catch-all) on one source compete: each message is dispatched to one + // of them, round-robin. The registration array is swapped under the listener lock; Next() reads it lock-free. + private sealed class ConsumerGroup + { + private ConsumerRegistration[] _registrations = []; + private int _next; + + public bool IsEmpty => Volatile.Read(ref _registrations).Length == 0; + + public void Add(ConsumerRegistration registration) + { + var current = _registrations; + var updated = new ConsumerRegistration[current.Length + 1]; + Array.Copy(current, updated, current.Length); + updated[^1] = registration; + Volatile.Write(ref _registrations, updated); + } + + public void Remove(ConsumerRegistration registration) + { + var current = _registrations; + int index = Array.IndexOf(current, registration); + if (index < 0) + return; + + var updated = new ConsumerRegistration[current.Length - 1]; + Array.Copy(current, 0, updated, 0, index); + Array.Copy(current, index + 1, updated, index, current.Length - index - 1); + Volatile.Write(ref _registrations, updated); + } + + public ConsumerRegistration? Next() + { + var snapshot = Volatile.Read(ref _registrations); + if (snapshot.Length == 0) + return null; + if (snapshot.Length == 1) + return snapshot[0]; + + int index = (int)((uint)Interlocked.Increment(ref _next) % (uint)snapshot.Length); + return snapshot[index]; + } + } + } } -internal class ReceivedMessage : IReceivedMessage, ISupportsDelayedMessageAbandon +internal class ReceivedMessage : IReceivedMessage { private readonly IMessageTransport _transport; private readonly TransportEntry _entry; private readonly IJobRuntimeStore? _runtimeStore; private readonly TimeProvider _timeProvider; + private readonly string? _deadLetterDestination; private int _isHandled; - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) { _transport = transport; _entry = entry; _runtimeStore = runtimeStore; _timeProvider = timeProvider ?? TimeProvider.System; + _deadLetterDestination = deadLetterDestination; CancellationToken = cancellationToken; } @@ -652,30 +901,39 @@ public Task CompleteAsync(CancellationToken cancellationToken = default) return _transport.CompleteAsync(_entry, cancellationToken); } - public Task AbandonAsync(CancellationToken cancellationToken = default) + public async Task RejectAsync(RejectOptions? options = null, CancellationToken cancellationToken = default) { if (!TryMarkHandled()) - return Task.CompletedTask; + return; - MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination)); - return _transport.AbandonAsync(_entry, cancellationToken); - } + options ??= new RejectOptions(); - public async Task AbandonAsync(TimeSpan redeliveryDelay, CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) + if (options.Terminal) + { + MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); + await DeadLetterOrDropAsync(_transport, _entry, options.Reason, _deadLetterDestination, cancellationToken).AnyContext(); return; + } MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination)); - if (_transport is ISupportsRedeliveryDelay redelivery) + if (options.RedeliveryDelay is not { } redeliveryDelay || redeliveryDelay <= TimeSpan.Zero) + { + await _transport.AbandonAsync(_entry, cancellationToken).AnyContext(); + return; + } + + // Honor an explicit redelivery delay natively when the transport can (within its advertised maximum); otherwise + // re-schedule the message through the runtime store and complete the original so the delay survives transports + // without native delayed redelivery. + if (_transport is ISupportsRedeliveryDelay redelivery && (redelivery.MaxRedeliveryDelay is not { } max || redeliveryDelay <= max)) { await redelivery.AbandonAsync(_entry, redeliveryDelay, cancellationToken).AnyContext(); return; } if (_runtimeStore is null) - throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" or a registered job runtime store."); + throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); int nextAttempt = _entry.DeliveryCount + 1; var headers = _entry.Headers.ToBuilder() @@ -696,15 +954,6 @@ await _runtimeStore.ScheduleDispatchAsync(new ScheduledDispatchState await _transport.CompleteAsync(_entry, cancellationToken).AnyContext(); } - public async Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default) - { - if (!TryMarkHandled()) - return; - - MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); - await DeadLetterAsync(_transport, _entry, reason, cancellationToken).AnyContext(); - } - public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default) { return _transport is ISupportsLockRenewal lockRenewal @@ -717,12 +966,29 @@ public Task ReportProgressAsync(int? percent = null, string? message = null, Can throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue or pub/sub messages."); } - internal static async Task DeadLetterAsync(IMessageTransport transport, TransportEntry entry, string? reason, CancellationToken cancellationToken) + // Terminal settlement. Prefer the transport's native dead-letter sink (preserves native DLQ tooling). When the + // transport has none, fall back to a configured core-managed dead-letter destination: copy the raw entry there + // (recording the reason) and complete the original. With neither, the message can't be parked, so it is completed + // (dropped) rather than throwing and stalling the consumer. + internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, TransportEntry entry, string? reason, string? deadLetterDestination, CancellationToken cancellationToken) { - if (transport is not ISupportsDeadLetter deadLetter) - throw new NotSupportedException($"Transport \"{transport.GetType().Name}\" does not support dead-lettering."); + if (transport is ISupportsDeadLetter deadLetter) + { + await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + return; + } - await deadLetter.DeadLetterAsync(entry, reason, cancellationToken).AnyContext(); + if (!String.IsNullOrEmpty(deadLetterDestination)) + { + var headers = String.IsNullOrEmpty(reason) + ? entry.Headers + : entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); + await transport.SendAsync(deadLetterDestination, [new TransportMessage { Body = entry.Body, Headers = headers, MessageId = entry.Id }], new TransportSendOptions(), cancellationToken).AnyContext(); + await transport.CompleteAsync(entry, cancellationToken).AnyContext(); + return; + } + + await transport.CompleteAsync(entry, cancellationToken).AnyContext(); } private bool TryMarkHandled() @@ -740,8 +1006,8 @@ private static int ParseAttemptsHeader(MessageHeaders headers) internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class { - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null) - : base(transport, entry, cancellationToken, runtimeStore, timeProvider) + public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) + : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination) { Message = message; } @@ -785,67 +1051,31 @@ public static string ToKebabCase(string value) /// internal sealed class MessageListenerHandle : IMessageConsumer, IMessageSubscription { - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly Action _remove; - private IPushSubscription? _pushSubscription; - private Task? _worker; + private readonly Func _dispose; private int _isDisposed; - public MessageListenerHandle(string topic, string subscription, string source, string key, MessageListenerRegistration registration, Action remove) + public MessageListenerHandle(string topic, string subscription, string source, string key, Func dispose) { Topic = topic; Subscription = subscription; Source = source; Key = key; - Registration = registration; - _remove = remove; + _dispose = dispose; } public string Topic { get; } public string Subscription { get; } public string Source { get; } public string Key { get; } - public MessageListenerRegistration Registration { get; } - public CancellationToken CancellationToken => _cancellationTokenSource.Token; - public bool IsDisposed => Volatile.Read(ref _isDisposed) == 1; - - public void ThrowIfConflicting(MessageListenerRegistration registration) - { - if (!Registration.Matches(registration)) - throw new InvalidOperationException($"A listener with key \"{Key}\" is already registered with a different handler or options."); - } - - public void SetPushSubscription(IPushSubscription subscription) - { - _pushSubscription = subscription; - } - - public void Start(Task worker) - { - _worker = worker; - } + // Disposing a single consumer handle detaches just that consumer from its source listener; the underlying receive + // loop keeps running until its last consumer detaches. public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _isDisposed, 1) == 1) return; - await _cancellationTokenSource.CancelAsync().AnyContext(); - - if (_pushSubscription is not null) - await _pushSubscription.DisposeAsync().AnyContext(); - - if (_worker is not null) - { - try - { - await _worker.AnyContext(); - } - catch (OperationCanceledException) { } - } - - _cancellationTokenSource.Dispose(); - _remove(Key, this); + await _dispose().AnyContext(); } } @@ -856,7 +1086,7 @@ internal sealed record MessageListenerRegistration public required Delegate Handler { get; init; } public required AckMode AckMode { get; init; } public required int MaxConcurrency { get; init; } - public required int MaxAttempts { get; init; } + public required int? MaxAttempts { get; init; } public required bool HasRedeliveryBackoff { get; init; } public static MessageListenerRegistration Create(Delegate handler, ListenerConfig config) diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index 0da6a6aff..65e69078e 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -36,6 +36,33 @@ public sealed record QueueReceiveOptions public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); } +/// +/// Core-owned retry and dead-letter policy. Foundatio always owns redelivery and dead-lettering so the behavior is +/// identical across transports; transports stay simple and only provide the underlying primitives (redelivery and an +/// optional dead-letter sink). Configure a default on /; a +/// consumer can override /backoff per consumer. +/// +public sealed record RetryPolicy +{ + /// Maximum delivery attempts for a failing handler before the message is dead-lettered. Default 5. + public int MaxAttempts { get; init; } = 5; + + /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's own redelivery timing. + public Func? Backoff { get; init; } + + /// + /// Destination terminal (dead-lettered) messages are sent to when the transport has no native dead-letter sink. + /// Null drops terminal messages on such transports. Ignored when the transport supports native dead-lettering. + /// + public string? DeadLetterDestination { get; init; } + + /// Maximum attempts for a message whose type has no registered consumer before it is dead-lettered as "no-handler". Default 50. + public int UnmatchedMaxAttempts { get; init; } = 50; + + /// Delay before redelivering an unmatched-type message. Null defers to the transport's own redelivery timing. + public Func? UnmatchedBackoff { get; init; } +} + public sealed record QueueConsumerOptions { public AckMode AckMode { get; init; } = AckMode.Auto; @@ -43,7 +70,8 @@ public sealed record QueueConsumerOptions public Type? RouteType { get; init; } public string? Key { get; init; } public int MaxConcurrency { get; init; } = 1; - public int MaxAttempts { get; init; } = 5; + // Null falls back to the queue's default RetryPolicy. + public int? MaxAttempts { get; init; } public Func? RedeliveryBackoff { get; init; } } @@ -53,6 +81,7 @@ public sealed record QueueOptions public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IJobRuntimeStore? RuntimeStore { get; init; } + public RetryPolicy RetryPolicy { get; init; } = new(); public TimeProvider TimeProvider { get; init; } = TimeProvider.System; public ILoggerFactory? LoggerFactory { get; init; } } @@ -76,6 +105,44 @@ public interface IMessageConsumer : IAsyncDisposable string Key { get; } } +/// +/// Thrown by the consumer loop when a message arrives on a shared destination whose type has no registered consumer +/// on this node (for example a newer message type mid rolling-deploy, or a misconfiguration). It is surfaced loudly +/// per message and isolated to that message — the receive loop and the other type handlers keep running. +/// +public sealed class UnhandledMessageTypeException : Exception +{ + public UnhandledMessageTypeException(string? messageType, string source) + : base($"No consumer is registered for message type \"{messageType ?? "(unknown)"}\" received on source \"{source}\".") + { + MessageType = messageType; + SourceName = source; + } + + public string? MessageType { get; } + public string SourceName { get; } +} + +public sealed record RejectOptions +{ + /// + /// When false (default) the message is returned for redelivery (a retry). When true the message is terminal: it + /// is moved to the transport's dead-letter sink where one exists, otherwise dropped. Terminal messages are never + /// redelivered. + /// + public bool Terminal { get; init; } + + /// Reason carried to the dead-letter sink (where the transport supports one) for a terminal reject. + public string? Reason { get; init; } + + /// + /// An explicit delay before the message is redelivered. Honored only for a non-terminal reject, served natively + /// when the transport supports redelivery delay within its advertised maximum, otherwise through the runtime store. + /// When null the transport's own redelivery timing applies. + /// + public TimeSpan? RedeliveryDelay { get; init; } +} + public interface IReceivedMessage { string Id { get; } @@ -88,8 +155,7 @@ public interface IReceivedMessage bool IsHandled { get; } CancellationToken CancellationToken { get; } Task CompleteAsync(CancellationToken cancellationToken = default); - Task AbandonAsync(CancellationToken cancellationToken = default); - Task DeadLetterAsync(string? reason = null, CancellationToken cancellationToken = default); + Task RejectAsync(RejectOptions? options = null, CancellationToken cancellationToken = default); Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default); } @@ -113,7 +179,7 @@ public MessageQueue(IMessageTransport transport, QueueOptions? options = null) options ??= new QueueOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner)); + static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy); } public Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index d2f88ceeb..adc665b95 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -112,6 +112,10 @@ public sealed record DestinationDeclaration public required string Name { get; init; } public DestinationRole Role { get; init; } = DestinationRole.Queue; public string? Source { get; init; } + + // Provider-specific creation arguments for transports that provision destinations (e.g. RabbitMQ queue arguments). + // Retry and dead-letter behavior is owned by the core RetryPolicy, not declared here, so destinations stay simple. + public IReadOnlyDictionary? ProviderArguments { get; init; } } public sealed record PushOptions @@ -148,6 +152,11 @@ public interface ISupportsPush : IMessageTransport public interface ISupportsRedeliveryDelay : IMessageTransport { + // The longest redelivery delay the transport can honor natively (e.g. SQS serves this via ChangeMessageVisibility, + // capped at 12 hours). Null means unbounded. A requested delay longer than this is routed through the runtime-store + // fallback instead of being silently clamped by the broker. + TimeSpan? MaxRedeliveryDelay { get; } + Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct); } @@ -167,6 +176,11 @@ public interface ISupportsLockRenewal : IMessageTransport public interface ISupportsVisibilityTimeout : IMessageTransport { + // The longest receive visibility timeout the transport can honor natively (e.g. SQS caps visibility at 12 hours). + // Null means unbounded. Callers requesting a longer visibility than the broker supports should treat that as + // unsatisfiable rather than relying on a silently clamped value. + TimeSpan? MaxVisibilityTimeout { get; } + Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct); } @@ -177,7 +191,13 @@ public interface ISupportsStats : IMessageTransport public interface ISupportsPriority : IMessageTransport { } -public interface ISupportsDelayedDelivery : IMessageTransport { } +public interface ISupportsDelayedDelivery : IMessageTransport +{ + // The longest delivery delay the transport can honor natively (e.g. SQS caps DelaySeconds at 15 minutes). + // Null means unbounded. A send scheduled further out than this is routed through the runtime-store fallback + // instead of being silently truncated to the broker's maximum. + TimeSpan? MaxDeliveryDelay { get; } +} public interface ISupportsExpiration : IMessageTransport { } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 4e0479c93..9b76c4c07 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -31,7 +31,8 @@ public sealed record PubSubSubscriptionOptions public string? Key { get; init; } public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; - public int MaxAttempts { get; init; } = 5; + // Null falls back to the pub/sub default RetryPolicy. + public int? MaxAttempts { get; init; } public Func? RedeliveryBackoff { get; init; } } @@ -41,6 +42,7 @@ public sealed record PubSubOptions public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IJobRuntimeStore? RuntimeStore { get; init; } + public RetryPolicy RetryPolicy { get; init; } = new(); public TimeProvider TimeProvider { get; init; } = TimeProvider.System; public ILoggerFactory? LoggerFactory { get; init; } } @@ -77,7 +79,7 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) options ??= new PubSubOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner)); + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy); } public Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 997e9fa40..dc0909ed1 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -73,7 +74,7 @@ await queue.EnqueueBatchAsync([ } [Fact] - public async Task AbandonAsync_RedeliversAsync() + public async Task RejectAsync_NonTerminal_RedeliversAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var queue = new MessageQueue(new InMemoryMessageTransport()); @@ -82,7 +83,7 @@ public async Task AbandonAsync_RedeliversAsync() var first = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); - await first.AbandonAsync(cancellationToken); + await first.RejectAsync(cancellationToken: cancellationToken); var second = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(second); @@ -122,7 +123,7 @@ public async Task ReportProgressAsync_WhenUntracked_ThrowsAsync() } [Fact] - public async Task DeadLetterAsync_DeadLettersAsync() + public async Task RejectAsync_Terminal_DeadLettersAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); @@ -132,7 +133,7 @@ public async Task DeadLetterAsync_DeadLettersAsync() var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); - await message.DeadLetterAsync("validation", cancellationToken); + await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Deadletter); @@ -267,6 +268,41 @@ await Assert.ThrowsAsync(async () => await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); } + [Fact] + public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Within the transport's advertised maximum: delivered natively, never touches the runtime store. + var nativeStore = new InMemoryJobRuntimeStore(); + await using var nativeTransport = new CappedDelayTransport(maxDeliveryDelay: TimeSpan.FromMinutes(15)); + await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); + var nativeProcessor = CreateDispatchProcessor(nativeStore, nativeTransport); + + await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + + Assert.Equal(1, nativeTransport.SendCount); + Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); + Assert.Equal(0, await nativeProcessor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddYears(1), cancellationToken: cancellationToken)); + + // Beyond the transport's maximum: routed through the runtime store instead of being silently truncated. + var fallbackStore = new InMemoryJobRuntimeStore(); + await using var fallbackTransport = new CappedDelayTransport(maxDeliveryDelay: TimeSpan.FromMinutes(15)); + await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); + var fallbackProcessor = CreateDispatchProcessor(fallbackStore, fallbackTransport); + + await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + + Assert.Equal(0, fallbackTransport.SendCount); + Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddHours(2), cancellationToken: cancellationToken)); + Assert.Equal(1, fallbackTransport.SendCount); + + var delayed = await fallbackQueue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + Assert.NotNull(delayed); + Assert.Equal("later", delayed.Message.Data); + await delayed.CompleteAsync(cancellationToken); + } + [Fact] public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThroughRuntimeStoreAsync() { @@ -514,6 +550,130 @@ private static async Task WaitForCompletedAsync(InMemoryMessageTransport transpo Assert.Equal(1, finalStats.Completed); } + [Fact] + public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTypeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var queue = new MessageQueue(new InMemoryMessageTransport()); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var aReceived = new List(); + var bReceived = new List(); + var aSignal = new AsyncCountdownEvent(1); + var bSignal = new AsyncCountdownEvent(1); + + await using var consumerA = await queue.StartConsumerAsync((message, _) => + { + lock (aReceived) + aReceived.Add(message.Message.Data); + aSignal.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + await using var consumerB = await queue.StartConsumerAsync((message, _) => + { + lock (bReceived) + bReceived.Add(message.Message.Data); + bSignal.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + // Both types route to the same destination, so they share one underlying receive loop that dispatches by type. + Assert.Equal(consumerA.Source, consumerB.Source); + + await queue.EnqueueAsync(new SharedAWorkItem { Data = "a" }, cancellationToken: cts.Token); + await queue.EnqueueAsync(new SharedBWorkItem { Data = "b" }, cancellationToken: cts.Token); + + await aSignal.WaitAsync(TimeSpan.FromSeconds(2)); + await bSignal.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(new[] { "a" }, aReceived); + Assert.Equal(new[] { "b" }, bReceived); + } + + [Fact] + public async Task StartConsumerAsync_UnmatchedType_DeadLettersAndKeepsConsumingAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { UnmatchedMaxAttempts = 3 } }); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(20)); + + var aSignal = new AsyncCountdownEvent(1); + await using var consumerA = await queue.StartConsumerAsync((_, _) => + { + aSignal.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + // SharedBWorkItem routes to the same destination but has no registered consumer on this node. + await queue.EnqueueAsync(new SharedBWorkItem { Data = "orphan" }, cancellationToken: cts.Token); + + // It is retried and finally dead-lettered as "no-handler" once the configured unmatched budget is exhausted. + for (int i = 0; i < 400; i++) + { + if ((await transport.GetStatsAsync("shared-demux", cts.Token)).Deadletter == 1) + break; + await Task.Delay(TimeSpan.FromMilliseconds(25), cts.Token); + } + + Assert.Equal(1, (await transport.GetStatsAsync("shared-demux", cts.Token)).Deadletter); + + // The loop survived the unmatched message and keeps consuming the type it does handle. + await queue.EnqueueAsync(new SharedAWorkItem { Data = "ok" }, cancellationToken: cts.Token); + await aSignal.WaitAsync(TimeSpan.FromSeconds(2)); + } + + [Fact] + public async Task RejectAsync_Terminal_WithoutNativeDeadLetter_SendsToConfiguredDestinationAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new NoDeadLetterTransport(); + await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { DeadLetterDestination = "preview-dead-letter" } }); + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(message); + + await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); + + // The transport has no native dead-letter sink, so core routes the terminal message to the configured destination. + var dead = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "preview-dead-letter", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(dead); + Assert.Equal("validation", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterReason)); + } + + [Fact] + public async Task StartConsumerAsync_UsesDefaultRetryPolicyMaxAttempts_WhenConsumerDoesNotOverrideAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { MaxAttempts = 2 } }); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(20)); + + int attempts = 0; + await using var consumer = await queue.StartConsumerAsync((_, _) => + { + Interlocked.Increment(ref attempts); + throw new InvalidOperationException("always fails"); + }, cancellationToken: cts.Token); // no per-consumer MaxAttempts -> default RetryPolicy (2) + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "x" }, cancellationToken: cts.Token); + + for (int i = 0; i < 400; i++) + { + if ((await transport.GetStatsAsync("preview-work-item", cts.Token)).Deadletter == 1) + break; + await Task.Delay(TimeSpan.FromMilliseconds(25), cts.Token); + } + + Assert.Equal(1, (await transport.GetStatsAsync("preview-work-item", cts.Token)).Deadletter); + Assert.Equal(2, attempts); + } + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { var serviceProvider = new ServiceCollection().BuildServiceProvider(); @@ -527,6 +687,18 @@ private sealed class RoutedWorkItem public string? Data { get; set; } } + [MessageRoute("shared-demux")] + private sealed class SharedAWorkItem + { + public string? Data { get; set; } + } + + [MessageRoute("shared-demux")] + private sealed class SharedBWorkItem + { + public string? Data { get; set; } + } + private interface IGroupedWorkItem { } @@ -569,4 +741,81 @@ public Task SendAsync(string destination, IReadOnlyList Task.CompletedTask; public ValueTask DisposeAsync() => ValueTask.CompletedTask; } + + private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ISupportsDelayedDelivery + { + private readonly Queue _entries = new(); + + public CappedDelayTransport(TimeSpan? maxDeliveryDelay) + { + MaxDeliveryDelay = maxDeliveryDelay; + } + + public TimeSpan? MaxDeliveryDelay { get; } + public int SendCount { get; private set; } + public TransportSendOptions? LastSendOptions { get; private set; } + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + SendCount += messages.Count; + LastSendOptions = options; + var items = new SendItemResult[messages.Count]; + for (int i = 0; i < messages.Count; i++) + { + string id = messages[i].MessageId ?? Guid.NewGuid().ToString("N"); + _entries.Enqueue(new TransportEntry { Id = id, Destination = destination, Body = messages[i].Body, Headers = messages[i].Headers, Receipt = new Receipt() }); + items[i] = new SendItemResult { MessageId = id, Success = true }; + } + + return Task.FromResult(new SendResult { Items = items }); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + { + return Task.FromResult>(_entries.Count > 0 ? [_entries.Dequeue()] : []); + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + // A minimal multi-destination pull transport with NO native dead-letter sink, used to prove core-managed + // dead-lettering routes terminal messages to the configured RetryPolicy.DeadLetterDestination. + private sealed class NoDeadLetterTransport : IMessageTransport, ISupportsPull + { + private readonly ConcurrentDictionary> _queues = new(StringComparer.Ordinal); + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + var queue = _queues.GetOrAdd(destination, _ => new ConcurrentQueue()); + var items = new SendItemResult[messages.Count]; + for (int i = 0; i < messages.Count; i++) + { + string id = messages[i].MessageId ?? Guid.NewGuid().ToString("N"); + queue.Enqueue(new TransportEntry { Id = id, Destination = destination, Body = messages[i].Body, Headers = messages[i].Headers, Receipt = new Receipt() }); + items[i] = new SendItemResult { MessageId = id, Success = true }; + } + + return Task.FromResult(new SendResult { Items = items }); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + { + if (_queues.TryGetValue(source, out var queue) && queue.TryDequeue(out var entry)) + return Task.FromResult>([entry]); + + return Task.FromResult>([]); + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + _queues.GetOrAdd(entry.Destination, _ => new ConcurrentQueue()).Enqueue(entry with { DeliveryCount = entry.DeliveryCount + 1 }); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } } \ No newline at end of file From 99d06e1e586473ba6d9db43ac5f2c752381a8090 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 29 Jun 2026 00:04:43 -0500 Subject: [PATCH 15/57] feat: address messaging/jobs review feedback (pub/sub addressing, multi-type dispatch, job context, recovery, ownership, CRON) Pub/sub addressing: the transport receive source for a subscription is now the topic-qualified "{topic}/{subscription}" composite (exposed as IMessageSubscription.Source), so the same subscription identity on two topics stays isolated instead of colliding on a bare name. Multi-type dispatch: add IMessageTypeRegistry (stable name<->type, Type.FullName fallback, RegisterMessageType) as the single wire-discriminator authority; interface/base-routed consumers now resolve the concrete payload type from the message.type header and deserialize the actual type (assignable to the route type) instead of raw-envelope-only. Removes the orphaned MessageTypeResolver/UseMessageTypeName router API. Job execution context: IJobWithExecutionContext receives a JobExecutionContext (job id, attempt, store-backed progress, lease heartbeat, cancellation checks); remove the always-throwing ReportProgressAsync from IReceivedMessage. Non-CRON job recovery: the runtime pump reclaims plain jobs stuck in Processing past their lease via IJobRuntimeStore.GetExpiredProcessingAsync (excludes CRON occurrences) + a lease+owner-aware TryReclaimExpiredAsync (re-queue while attempts remain, else dead-letter), closing the renew race that could double-run a live job. Transport ownership: OwnsTransport flag so DI-built queue and pub/sub clients do not both dispose a shared singleton transport (the container disposes it once); direct construction still owns it. CRON: mark the legacy in-process AddCronJob/AddJobScheduler/ScheduledJobService path as legacy/compat with docs pointing to the durable runtime (full reroute deferred). Adds 8 tests (pub/sub isolation, interface concrete-deserialize, job context, stale recovery + reclaim guard + occurrence exclusion, DI dispose-once, delay cap routing) and updates the redesign guide. All messaging/queue/jobs tests pass; solution builds on net8 + net10. Co-Authored-By: Claude Opus 4.8 --- docs/guide/messaging-jobs-redesign.md | 30 +++- .../Jobs/JobHostExtensions.cs | 12 ++ .../Jobs/JobRuntimeService.cs | 10 ++ .../Jobs/ScheduledJobService.cs | 5 + src/Foundatio/FoundatioServicesExtensions.cs | 17 +++ src/Foundatio/Jobs/IJob.cs | 11 ++ src/Foundatio/Jobs/JobRuntime.cs | 138 ++++++++++++++++++ src/Foundatio/Messaging/MessageClientCore.cs | 41 ++++-- src/Foundatio/Messaging/MessageQueue.cs | 11 +- src/Foundatio/Messaging/MessageRouting.cs | 12 -- .../Messaging/MessageTypeRegistry.cs | 72 +++++++++ src/Foundatio/Messaging/PubSub.cs | 29 +++- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 93 ++++++++++++ .../Foundatio.Tests/Messaging/PubSubTests.cs | 63 +++++++- .../Queue/MessageQueueTests.cs | 110 ++++++++++++-- 15 files changed, 606 insertions(+), 48 deletions(-) create mode 100644 src/Foundatio/Messaging/MessageTypeRegistry.cs diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index ddf8b7867..ce0e1b169 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -80,7 +80,23 @@ await using var cancelled = await queue.StartConsumerAsync(Handl // One loop on the shared destination. OrderSubmitted is dispatched to the first handler, OrderCancelled to the second. ``` -Consumers that share a message type compete: each message is dispatched to one of them, round-robin. The non-generic `StartConsumerAsync` (or a consumer whose route type is an interface/base type) is a catch-all that receives any type no exact-typed consumer claimed — the grouped/raw-envelope path. All consumers on one destination must agree on `MaxConcurrency` (it is a property of the shared loop). A message whose type has **no** registered consumer on this node is handled loudly — see [Unmatched message types](#unmatched-message-types). +Consumers that share a message type compete: each message is dispatched to one of them, round-robin. All consumers on one destination must agree on `MaxConcurrency` (it is a property of the shared loop). A message whose type has **no** registered consumer on this node is handled loudly — see [Unmatched message types](#unmatched-message-types). + +A consumer whose route type is an interface or base type is a **grouped** consumer and receives the concrete payload (assignable to that type), not raw bytes: + +```csharp +await using var all = await queue.StartConsumerAsync(HandleAnyAsync); +// HandleAnyAsync receives IReceivedMessage whose Message is the concrete OrderSubmitted / OrderCancelled. +``` + +The concrete type is resolved from the `message.type` header through `IMessageTypeRegistry` and deserialized as the actual payload type. The registry is the stable wire discriminator in both directions — register stable names for types that may move between assemblies/namespaces; unregistered types fall back to `Type.FullName` (never `AssemblyQualifiedName`): + +```csharp +services.AddFoundatio() + .Messaging.RegisterMessageType("order.submitted"); +``` + +The raw-envelope path (non-generic `ReceiveAsync(new QueueReceiveOptions { RouteType = ... })`) remains for callers that want the bytes without deserialization. ## Pub/Sub @@ -225,6 +241,18 @@ services.AddFoundatio() Unregistered jobs fall back to `Type.FullName`, not `AssemblyQualifiedName`. +### Execution context + +A job that wants its runtime identity and store-backed operations implements `IJobWithExecutionContext`; the runtime sets `ExecutionContext` before invoking it. The context exposes `JobId`, `Attempt`, the cancellation token, and `ReportProgressAsync`, `RenewLeaseAsync` (heartbeat for long runs), and `IsCancellationRequestedAsync` — the parts of `IJobRuntimeStore` useful from inside job code. Jobs that use it should be registered transient (the context is per-run state). Untracked queue/pub-sub messages have no progress concept, so `IReceivedMessage` has no `ReportProgressAsync`. + +### Recovery + +The runtime pump reclaims jobs stuck in `Processing` past their lease (a worker that crashed mid-run), not just CRON occurrences: `IJobRuntimeStore.GetExpiredProcessingAsync` surfaces them and the worker re-queues them while attempts remain (`JobRuntimeServiceOptions.MaxJobAttempts`), otherwise dead-letters them. The status CAS serializes concurrent reclaimers. + +### CRON + +The redesigned durable CRON path materializes durable, recoverable occurrences through `IJobScheduler` → `JobScheduleProcessor` → the runtime store and pump. The legacy hosted `AddCronJob`/`AddJobScheduler` API still wires the in-process `ScheduledJobService` and is retained as **legacy/compat only**; routing the default hosted CRON API onto the durable scheduler is a planned follow-up (best validated alongside a real provider, since durable distributed CRON leans on the runtime store's transition semantics). + ## Migration Legacy queue code usually moves from one queue instance per payload type to one app-facing queue plus routing: diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index 3de0ac8dc..fc709bbfb 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -61,6 +61,13 @@ public static IServiceCollection AddJob(this IServiceCollection services, string return services.AddJob(jobOptionsBuilder.Target); } + /// + /// Legacy/compat. This registers the in-process , which runs CRON occurrences + /// in-process and does not materialize durable, recoverable occurrences. The forward path in the redesigned runtime + /// is the durable scheduler — register it with services.AddFoundatio().Jobs.UseInMemoryRuntime() plus + /// , which materializes durable occurrences with retry, recovery, and + /// dead-lettering. Routing this default API onto the durable scheduler is a planned follow-up. + /// public static IServiceCollection AddCronJob(this IServiceCollection services, ScheduledJobOptions jobOptions) { if (jobOptions.JobFactory == null) @@ -180,6 +187,11 @@ public static IServiceCollection AddDistributedCronJob(this IServiceCollection s }))); } + /// + /// Legacy/compat: registers the in-process CRON scheduler. For durable, + /// recoverable CRON occurrences use the redesigned runtime (AddFoundatio().Jobs.UseInMemoryRuntime() + + /// ) instead. + /// public static IServiceCollection AddJobScheduler(this IServiceCollection services) { if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(ScheduledJobService))) diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs index 328faacf0..0253c890a 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs @@ -25,6 +25,12 @@ public class JobRuntimeServiceOptions /// Maximum number of due dispatches and queued jobs claimed per pump iteration. /// public int BatchSize { get; set; } = 100; + + /// + /// Maximum number of processing attempts for a durable job before a stale (lease-expired) instance is + /// dead-lettered instead of re-queued. Defaults to 3. + /// + public int MaxJobAttempts { get; set; } = 3; } /// @@ -66,6 +72,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // recovers occurrences whose processing lease expired (crash mid-run) and applies retry/dead-letter. await _processor.RunDueOccurrencesAsync(now, _options.BatchSize, lease: null, stoppingToken).AnyContext(); + // Recover plain (non-CRON) jobs whose processing lease expired (a worker crash mid-run): re-queue them + // while attempts remain, otherwise dead-letter them. Without this they would strand in Processing. + await _worker.RecoverStaleAsync(_options.MaxJobAttempts, _options.BatchSize, stoppingToken).AnyContext(); + // Run jobs submitted via IJobClient that are sitting in the Queued state. await _worker.RunQueuedAsync(_options.BatchSize, stoppingToken).AnyContext(); } diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs index 39bcdfbd9..b7b4ea619 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs @@ -13,6 +13,11 @@ namespace Foundatio.Extensions.Hosting.Jobs; +/// +/// Legacy/compat in-process CRON scheduler used by . +/// It runs occurrences in-process and does not materialize durable, recoverable occurrences. The redesigned runtime's +/// durable scheduler (JobScheduleProcessor driven by ) is the forward path. +/// public class ScheduledJobService : BackgroundService { private readonly IServiceProvider _serviceProvider; diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 24c87c85c..184b45574 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -296,6 +296,15 @@ public MessagingBuilder ConfigureRetry(Func configure) return ConfigureRetry(configure(new RetryPolicy())); } + // Registers a stable wire name for a message type so the discriminator survives assembly/namespace moves and + // grouped/interface consumers can resolve and deserialize the concrete payload type. + public MessagingBuilder RegisterMessageType(string name) where T : class + { + ArgumentException.ThrowIfNullOrEmpty(name); + _services.AddSingleton(new MessageTypeRegistration(name, typeof(T))); + return this; + } + public FoundatioBuilder UseInMemory(InMemoryMessageBusOptions? options = null) { _services.ReplaceSingleton(sp => new InMemoryMessageBus(options.UseServices(sp))); @@ -368,6 +377,7 @@ private void RegisterMessageTopology() private void RegisterMessageClients() { RegisterRoutingServices(); + _services.ReplaceSingleton(sp => new MessageTypeRegistry(sp.GetServices())); _services.ReplaceSingleton(sp => new MessageQueue(sp.GetRequiredService(), CreateQueueOptions(sp))); _services.ReplaceSingleton(sp => new PubSub(sp.GetRequiredService(), CreatePubSubOptions(sp))); } @@ -378,8 +388,12 @@ private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) { Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, + MessageTypes = serviceProvider.GetService() ?? new MessageTypeRegistry(), RuntimeStore = serviceProvider.GetService(), RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), + // The transport is a shared DI singleton owned by the container; the queue must not dispose it (the + // pub/sub client uses the same instance). + OwnsTransport = false, TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, LoggerFactory = serviceProvider.GetService() }; @@ -391,8 +405,11 @@ private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvide { Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, + MessageTypes = serviceProvider.GetService() ?? new MessageTypeRegistry(), RuntimeStore = serviceProvider.GetService(), RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), + // Shared DI singleton transport; disposed once by the container, not by this client. + OwnsTransport = false, TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, LoggerFactory = serviceProvider.GetService() }; diff --git a/src/Foundatio/Jobs/IJob.cs b/src/Foundatio/Jobs/IJob.cs index ac5cefd10..5d222322d 100644 --- a/src/Foundatio/Jobs/IJob.cs +++ b/src/Foundatio/Jobs/IJob.cs @@ -32,6 +32,17 @@ public interface IJobWithOptions : IJob JobOptions? Options { get; set; } } +/// +/// A durable job that wants its — job id, attempt number, and store-backed progress, +/// lease heartbeat, and cooperative cancellation checks. The durable runtime sets on the +/// job instance before invoking it. Jobs that use the context should be registered as transient (a fresh instance per +/// run), since the context is per-run state, matching . +/// +public interface IJobWithExecutionContext : IJob +{ + JobExecutionContext? ExecutionContext { get; set; } +} + public static class JobExtensions { public static async Task TryRunAsync(this IJob job, CancellationToken cancellationToken = default) diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 65c40636e..fba07b1cd 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -206,6 +206,42 @@ public Task RequestCancellationAsync(CancellationToken cancellationToken = } } +/// +/// Passed to a durable job that implements . Gives the running job its identity +/// and attempt number, plus store-backed progress reporting, lease heartbeat (for long runs), and cooperative +/// cancellation checks — the parts of that are useful from inside job code. +/// +public sealed class JobExecutionContext +{ + private readonly IJobRuntimeStore _store; + private readonly string _nodeId; + private readonly TimeSpan _lease; + + internal JobExecutionContext(string jobId, int attempt, CancellationToken cancellationToken, IJobRuntimeStore store, string nodeId, TimeSpan lease) + { + JobId = jobId; + Attempt = attempt; + CancellationToken = cancellationToken; + _store = store; + _nodeId = nodeId; + _lease = lease; + } + + public string JobId { get; } + public int Attempt { get; } + public CancellationToken CancellationToken { get; } + + public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) + => _store.SetProgressAsync(JobId, percent, message, cancellationToken); + + // Extends the worker's lease so a long-but-alive run is not reclaimed as stale. + public Task RenewLeaseAsync(CancellationToken cancellationToken = default) + => _store.RenewClaimAsync(JobId, _nodeId, _lease, cancellationToken); + + public Task IsCancellationRequestedAsync(CancellationToken cancellationToken = default) + => _store.IsCancellationRequestedAsync(JobId, cancellationToken); +} + public interface IJobMonitor { Task GetAsync(string jobId, CancellationToken cancellationToken = default); @@ -223,6 +259,9 @@ public interface IJobWorker { Task RunAsync(string jobId, CancellationToken cancellationToken = default); Task RunQueuedAsync(int limit = 100, CancellationToken cancellationToken = default); + // Reclaims jobs stuck in Processing past their lease (a worker that crashed mid-run): re-queues them while attempts + // remain, otherwise dead-letters them. Returns the number recovered. + Task RecoverStaleAsync(int maxAttempts, int limit = 100, CancellationToken cancellationToken = default); } public interface IJobRuntimeStore : IJobMonitor @@ -235,6 +274,15 @@ public interface IJobRuntimeStore : IJobMonitor Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken cancellationToken = default); + // Returns plain (non-CRON-occurrence) jobs in Processing whose lease has expired as of + // (their owning worker is presumed dead), so the runtime can reclaim them. CRON occurrences are excluded — the + // scheduler recovers those with its own per-definition retry budget. + Task> GetExpiredProcessingAsync(DateTimeOffset now, int limit, CancellationToken cancellationToken = default); + // Atomically reclaims a stale Processing job: the transition applies only if the job is STILL owned by + // and its lease is STILL expired as of . This closes the + // race where the owning worker renews its lease between a stale scan and the reclaim (which would otherwise + // re-queue a live job and double-run it). + Task TryReclaimExpiredAsync(string jobId, DateTimeOffset now, string expectedNodeId, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default); Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken cancellationToken = default); Task IncrementAttemptAsync(string jobId, CancellationToken cancellationToken = default); Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); @@ -380,6 +428,52 @@ public Task ReleaseClaimAsync(string jobId, string nodeId, CancellationTok } } + public Task> GetExpiredProcessingAsync(DateTimeOffset now, int limit, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + lock (_lock) + { + var expired = _jobs.Values + // Exclude CRON occurrences (ScheduledForUtc set): the scheduler owns their recovery via its own + // per-definition retry budget. This path only recovers plain IJobClient-submitted jobs. + .Where(s => s.Status == JobStatus.Processing && s.ScheduledForUtc is null && s.LeaseExpiresUtc is { } lease && lease <= now) + .OrderBy(s => s.LeaseExpiresUtc) + .Take(Math.Max(1, limit)) + .ToArray(); + + return Task.FromResult>(expired); + } + } + + public Task TryReclaimExpiredAsync(string jobId, DateTimeOffset now, string expectedNodeId, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(expectedNodeId); + + lock (_lock) + { + if (!_jobs.TryGetValue(jobId, out var current)) + return Task.FromResult(false); + + // Re-check (atomically, under the lock) the conditions the stale scan saw: still Processing, still owned by + // the same node, and the lease is still expired. A renewal or re-claim that landed since the scan fails one + // of these and the reclaim is skipped. + if (current.Status != JobStatus.Processing || !String.Equals(current.NodeId, expectedNodeId, StringComparison.Ordinal)) + return Task.FromResult(false); + + if (current.LeaseExpiresUtc is not { } lease || lease > now) + return Task.FromResult(false); + + _jobs[jobId] = ApplyPatch(current, patch) with + { + Status = newStatus, + LastUpdatedUtc = patch?.LastUpdatedUtc ?? _timeProvider.GetUtcNow() + }; + return Task.FromResult(true); + } + } + public Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -643,6 +737,44 @@ public async Task RunAsync(string jobId, CancellationToken cancellationTok return state is not null && await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false); } + public async Task RecoverStaleAsync(int maxAttempts, int limit = 100, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var stale = await _store.GetExpiredProcessingAsync(now, limit, cancellationToken).ConfigureAwait(false); + + int recovered = 0; + foreach (var state in stale) + { + if (String.IsNullOrEmpty(state.NodeId)) + continue; + + // TryReclaimExpiredAsync re-verifies (atomically) that the job is still owned by the same presumed-dead + // node and its lease is still expired, so a worker that renewed between the scan and here is not yanked out + // from under itself (no double-run). Attempts are incremented per run, so a job that keeps crashing is + // dead-lettered once it has consumed its attempt budget instead of being re-queued forever. + bool transitioned = state.Attempt >= maxAttempts + ? await _store.TryReclaimExpiredAsync(state.JobId, now, state.NodeId, JobStatus.DeadLettered, new JobStatePatch + { + Error = $"Lease expired after {state.Attempt} attempt(s) without completion.", + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + CompletedUtc = now, + LastUpdatedUtc = now + }, cancellationToken).ConfigureAwait(false) + : await _store.TryReclaimExpiredAsync(state.JobId, now, state.NodeId, JobStatus.Queued, new JobStatePatch + { + ClearNodeId = true, + ClearLeaseExpiresUtc = true, + LastUpdatedUtc = now + }, cancellationToken).ConfigureAwait(false); + + if (transitioned) + recovered++; + } + + return recovered; + } + private async Task RunJobStateAsync(JobState state, CancellationToken cancellationToken) { if (state.Status != JobStatus.Queued) @@ -671,6 +803,12 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc { var jobType = ResolveJobType(state); var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); + + // Hand the job its execution context (progress, heartbeat, cancellation, identity) when it opts in. The + // store was already incremented to this attempt by the Queued -> Processing transition above. + if (job is IJobWithExecutionContext contextual) + contextual.ExecutionContext = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease); + var result = await job.TryRunAsync(linkedCancellationTokenSource.Token).ConfigureAwait(false); var completedAt = _timeProvider.GetUtcNow(); diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 164e0bd8c..f8642495e 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -76,11 +76,13 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly ILogger _logger; private readonly Func _exceptionFactory; private readonly RetryPolicy _retryPolicy; + private readonly IMessageTypeRegistry _typeRegistry; + private readonly bool _ownsTransport; private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, - IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null) + IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _serializer = serializer; @@ -90,6 +92,8 @@ public MessageClientCore(IMessageTransport transport, ISerializer serializer, IM _logger = logger; _exceptionFactory = exceptionFactory; _retryPolicy = retryPolicy ?? new RetryPolicy(); + _typeRegistry = typeRegistry ?? new MessageTypeRegistry(); + _ownsTransport = ownsTransport; } public IMessageRouter Router => _router; @@ -209,7 +213,11 @@ public async ValueTask DisposeAsync() foreach (var listener in _sources.Values.ToArray()) await listener.DisposeAsync().AnyContext(); - await _transport.DisposeAsync().AnyContext(); + // Only dispose the transport when this client owns it. In DI the transport is a shared singleton owned by the + // container, so neither the queue nor the pub/sub client should dispose it (that would double-dispose the one + // the other still depends on). + if (_ownsTransport) + await _transport.DisposeAsync().AnyContext(); } // Multiple typed consumers can share one destination. They attach to a single per-source listener whose loop @@ -228,7 +236,7 @@ private async Task RegisterConsumerAsync(ListenerConfig c Dispatch = dispatch, Info = MessageListenerRegistration.Create(handler, config), IsCatchAll = catchAll, - TypeName = catchAll ? null : _router.ResolveMessageType(config.MessageType) + TypeName = catchAll ? null : _typeRegistry.GetName(config.MessageType) }; while (true) @@ -459,10 +467,27 @@ private async Task> CreateReceivedMessageAsync(TransportE { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); + // For an interface/base route the body cannot be deserialized as T directly. Resolve the concrete payload type + // from the message-type header via the registry and deserialize that, then hand it back as T (the concrete + // instance is assignable to T). Exact concrete routes deserialize as T directly. + Type targetType = typeof(T); + if (typeof(T).IsInterface || typeof(T).IsAbstract) + { + string? typeName = entry.Headers.GetValueOrDefault(KnownHeaders.MessageType); + var resolved = String.IsNullOrEmpty(typeName) ? null : _typeRegistry.Resolve(typeName); + if (resolved is null || !typeof(T).IsAssignableFrom(resolved)) + { + await DeadLetterPoisonMessageAsync(entry, "unresolved-type", cancellationToken).AnyContext(); + throw _exceptionFactory($"Unable to resolve a concrete type \"{typeName}\" assignable to \"{typeof(T).Name}\" for message \"{entry.Id}\".", null); + } + + targetType = resolved; + } + T? message; try { - message = _serializer.Deserialize(entry.Body); + message = _serializer.Deserialize(entry.Body, targetType) as T; } catch (Exception ex) { @@ -485,14 +510,13 @@ private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, C return ReceivedMessage.DeadLetterOrDropAsync(_transport, entry, reason, _retryPolicy.DeadLetterDestination, cancellationToken); } - public string ResolveMessageType(Type messageType) => _router.ResolveMessageType(messageType); private TransportMessage CreateTransportMessage(object message, Type messageType, MessageEnvelopeOptions options, string? messageId) { // Content type is intentionally not written as a header: the receive path always uses the single configured // serializer, so advertising a per-message content type would be misleading until real negotiation exists. var headers = (options.Headers ?? MessageHeaders.Empty).ToBuilder() - .Set(KnownHeaders.MessageType, _router.ResolveMessageType(messageType)) + .Set(KnownHeaders.MessageType, _typeRegistry.GetName(messageType)) .Set(KnownHeaders.Priority, options.Priority.ToString()); if (!String.IsNullOrEmpty(options.CorrelationId)) @@ -961,11 +985,6 @@ public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancella : throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support lock renewal."); } - public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) - { - throw new NotSupportedException("Message progress reporting requires tracked job execution and is not available for untracked queue or pub/sub messages."); - } - // Terminal settlement. Prefer the transport's native dead-letter sink (preserves native DLQ tooling). When the // transport has none, fall back to a configured core-managed dead-letter destination: copy the raw entry there // (recording the reason) and complete the original. With neither, the message can't be parked, so it is completed diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index 65e69078e..c6cabd449 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -80,8 +80,16 @@ public sealed record QueueOptions public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; + public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); public IJobRuntimeStore? RuntimeStore { get; init; } public RetryPolicy RetryPolicy { get; init; } = new(); + + /// + /// Whether disposing this queue also disposes the transport. True (default) for a transport this queue created or + /// solely uses; set false when the transport is a shared/externally-owned instance (e.g. a DI singleton also used + /// by a pub/sub client) so it is disposed exactly once by its owner. + /// + public bool OwnsTransport { get; init; } = true; public TimeProvider TimeProvider { get; init; } = TimeProvider.System; public ILoggerFactory? LoggerFactory { get; init; } } @@ -157,7 +165,6 @@ public interface IReceivedMessage Task CompleteAsync(CancellationToken cancellationToken = default); Task RejectAsync(RejectOptions? options = null, CancellationToken cancellationToken = default); Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); - Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default); } public interface IReceivedMessage : IReceivedMessage where T : class @@ -179,7 +186,7 @@ public MessageQueue(IMessageTransport transport, QueueOptions? options = null) options ??= new QueueOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy); + static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes); } public Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/src/Foundatio/Messaging/MessageRouting.cs b/src/Foundatio/Messaging/MessageRouting.cs index fe04a1b38..1b1245cea 100644 --- a/src/Foundatio/Messaging/MessageRouting.cs +++ b/src/Foundatio/Messaging/MessageRouting.cs @@ -29,7 +29,6 @@ public interface IMessageRouter { string ResolveRoute(MessageRouteContext context); string ResolveSubscription(MessageSubscriptionContext context); - string ResolveMessageType(Type messageType); } public sealed record MessageRouteMap @@ -49,7 +48,6 @@ public sealed class MessageRoutingOptions public string? SubscriptionIdentity { get; set; } public string? ServiceIdentity { get; set; } public Func? Convention { get; set; } - public Func? MessageTypeResolver { get; set; } public IReadOnlyList GetTopologyDeclarations() { @@ -158,11 +156,6 @@ public MessageRoutingOptionsBuilder UseConvention(Func resolver) - { - _options.MessageTypeResolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); - return this; - } public MessageRoutingOptions Build() { @@ -311,11 +304,6 @@ public string ResolveSubscription(MessageSubscriptionContext context) return GetDefaultServiceIdentity(); } - public string ResolveMessageType(Type messageType) - { - ArgumentNullException.ThrowIfNull(messageType); - return _options.MessageTypeResolver?.Invoke(messageType) ?? messageType.FullName ?? messageType.Name; - } private static string GetDefaultServiceIdentity() { diff --git a/src/Foundatio/Messaging/MessageTypeRegistry.cs b/src/Foundatio/Messaging/MessageTypeRegistry.cs new file mode 100644 index 000000000..6a1d80492 --- /dev/null +++ b/src/Foundatio/Messaging/MessageTypeRegistry.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; + +namespace Foundatio.Messaging; + +/// +/// Resolves the stable wire discriminator written to the message.type header in both directions: a CLR type to +/// its name (for sending) and a name back to its CLR type (so a grouped/interface consumer can deserialize the actual +/// payload type). Register stable names for types that may move between assemblies or namespaces; unregistered types +/// fall back to (never AssemblyQualifiedName). +/// +public interface IMessageTypeRegistry +{ + string GetName(Type messageType); + Type? Resolve(string name); +} + +public sealed record MessageTypeRegistration(string Name, Type MessageType); + +public sealed class MessageTypeRegistry : IMessageTypeRegistry +{ + private readonly Dictionary _nameToType = new(StringComparer.Ordinal); + private readonly Dictionary _typeToName = []; + + public MessageTypeRegistry(IEnumerable? registrations = null) + { + foreach (var registration in registrations ?? []) + Add(registration); + } + + public string GetName(Type messageType) + { + ArgumentNullException.ThrowIfNull(messageType); + return _typeToName.TryGetValue(messageType, out string? name) + ? name + : messageType.FullName ?? messageType.Name; + } + + public Type? Resolve(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + + if (_nameToType.TryGetValue(name, out var registered)) + return registered; + + var type = Type.GetType(name, throwOnError: false); + if (type is not null) + return type; + + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + type = assembly.GetType(name, throwOnError: false); + if (type is not null) + return type; + } + + return null; + } + + private void Add(MessageTypeRegistration registration) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentException.ThrowIfNullOrEmpty(registration.Name); + ArgumentNullException.ThrowIfNull(registration.MessageType); + + if (_nameToType.TryGetValue(registration.Name, out var existing) && existing != registration.MessageType) + throw new InvalidOperationException($"Message type name \"{registration.Name}\" is already registered for \"{existing.FullName}\"."); + + _nameToType[registration.Name] = registration.MessageType; + _typeToName[registration.MessageType] = registration.Name; + } +} diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 9b76c4c07..cc78c1479 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -41,8 +41,15 @@ public sealed record PubSubOptions public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; + public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); public IJobRuntimeStore? RuntimeStore { get; init; } public RetryPolicy RetryPolicy { get; init; } = new(); + + /// + /// Whether disposing this pub/sub client also disposes the transport. True (default) for a transport it solely + /// uses; set false when the transport is shared/externally owned (e.g. a DI singleton also used by a queue client). + /// + public bool OwnsTransport { get; init; } = true; public TimeProvider TimeProvider { get; init; } = TimeProvider.System; public ILoggerFactory? LoggerFactory { get; init; } } @@ -63,6 +70,13 @@ public interface IMessageSubscription : IAsyncDisposable string Topic { get; } string Subscription { get; } string Key { get; } + + /// + /// The transport destination this subscription receives from. It encodes both the topic and the subscription + /// identity (so the same subscription name on two different topics maps to two distinct sources), rather than the + /// bare subscription name. + /// + string Source { get; } } /// @@ -79,7 +93,7 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) options ??= new PubSubOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy); + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes); } public Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class @@ -146,7 +160,9 @@ private ListenerConfig BuildConfig(Type routeType, PubSubSubscriptionOptions opt { Topic = topic, Subscription = subscription, - Source = subscription, // a pub/sub consumer receives from its subscription destination + // The transport source is the topic-qualified subscription destination, not the bare subscription name, so + // the same subscription identity used on two topics resolves to two distinct sources (and isolates). + Source = SubscriptionDestination(topic, subscription), Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{routeType.FullName ?? routeType.Name}", MessageType = routeType, AckMode = options.AckMode, @@ -165,10 +181,17 @@ private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken ca { return _core.EnsureAsync([ new DestinationDeclaration { Name = config.Topic, Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = config.Subscription, Role = DestinationRole.Subscription, Source = config.Topic } + new DestinationDeclaration { Name = config.Source, Role = DestinationRole.Subscription, Source = config.Topic } ], cancellationToken); } + // The topic-qualified subscription destination. The topic is part of the identity so the same subscription name on + // two topics does not collide on one transport source. (A provider can map this to its native subscription address.) + private static string SubscriptionDestination(string topic, string subscription) + { + return $"{topic}/{subscription}"; + } + private string GetTopic(Type messageType, string? topic) { return _core.Router.ResolveRoute(new MessageRouteContext diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index 1cab823cf..1f3c6a7d6 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -10,6 +10,87 @@ namespace Foundatio.Tests.Jobs; public class JobRuntimeTests { + [Fact] + public async Task RunAsync_WithExecutionContext_ReportsProgressAndIdentityAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + await using var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var worker = new JobWorker(store, serviceProvider, nodeId: "ctx-node"); + + await store.CreateIfAbsentAsync(new JobState + { + JobId = "ctx-job", + Name = "ctx", + JobType = typeof(ProgressJob).FullName, + Status = JobStatus.Queued + }, cancellationToken); + + Assert.True(await worker.RunAsync("ctx-job", cancellationToken)); + + var state = await store.GetAsync("ctx-job", cancellationToken); + Assert.Equal(JobStatus.Completed, state!.Status); + Assert.Equal(100, state.Progress); // a completed job is 100%; the worker sets this on success + // The job wrote its context identity + attempt into the progress message (preserved through completion), + // proving the store-backed context is wired through to job code. + Assert.Equal("ctx-job:1", state.ProgressMessage); + } + + [Fact] + public async Task RecoverStaleAsync_ReclaimsExpiredProcessingJobsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + await using var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var worker = new JobWorker(store, serviceProvider, nodeId: "recovery-node"); + + var expired = DateTimeOffset.UtcNow.AddMinutes(-5); + + // A crashed job with attempts remaining -> re-queued. + await store.CreateIfAbsentAsync(new JobState { JobId = "retry-me", Name = "j", Status = JobStatus.Processing, NodeId = "dead-node", LeaseExpiresUtc = expired, Attempt = 1 }, cancellationToken); + // A crashed job that exhausted its attempts -> dead-lettered. + await store.CreateIfAbsentAsync(new JobState { JobId = "give-up", Name = "j", Status = JobStatus.Processing, NodeId = "dead-node", LeaseExpiresUtc = expired, Attempt = 3 }, cancellationToken); + // A healthy job whose lease is still valid -> untouched. + await store.CreateIfAbsentAsync(new JobState { JobId = "alive", Name = "j", Status = JobStatus.Processing, NodeId = "live-node", LeaseExpiresUtc = DateTimeOffset.UtcNow.AddMinutes(5), Attempt = 1 }, cancellationToken); + // A CRON occurrence (ScheduledForUtc set) with an expired lease -> NOT reclaimed here; the scheduler owns it. + await store.CreateIfAbsentAsync(new JobState { JobId = "occurrence", Name = "j", Status = JobStatus.Processing, NodeId = "dead-node", LeaseExpiresUtc = expired, Attempt = 1, ScheduledForUtc = expired }, cancellationToken); + + int recovered = await worker.RecoverStaleAsync(maxAttempts: 3, cancellationToken: cancellationToken); + + Assert.Equal(2, recovered); + + var retried = await store.GetAsync("retry-me", cancellationToken); + Assert.Equal(JobStatus.Queued, retried!.Status); + Assert.Null(retried.NodeId); + Assert.Null(retried.LeaseExpiresUtc); + + Assert.Equal(JobStatus.DeadLettered, (await store.GetAsync("give-up", cancellationToken))!.Status); + Assert.Equal(JobStatus.Processing, (await store.GetAsync("alive", cancellationToken))!.Status); + // The CRON occurrence is left for the scheduler's own recovery, not reclaimed as a plain job. + Assert.Equal(JobStatus.Processing, (await store.GetAsync("occurrence", cancellationToken))!.Status); + } + + [Fact] + public async Task TryReclaimExpiredAsync_GuardsAgainstOwnerRenewAndForeignNodeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var now = DateTimeOffset.UtcNow; + + await store.CreateIfAbsentAsync(new JobState { JobId = "expired", Name = "j", Status = JobStatus.Processing, NodeId = "owner", LeaseExpiresUtc = now.AddMinutes(-1), Attempt = 1 }, cancellationToken); + await store.CreateIfAbsentAsync(new JobState { JobId = "renewed", Name = "j", Status = JobStatus.Processing, NodeId = "owner", LeaseExpiresUtc = now.AddMinutes(5), Attempt = 1 }, cancellationToken); + + // Wrong owner -> rejected (another node already reclaimed/re-ran it). + Assert.False(await store.TryReclaimExpiredAsync("expired", now, "different-node", JobStatus.Queued, cancellationToken: cancellationToken)); + // Owner renewed its lease (no longer expired) -> rejected, so a live worker is never yanked out from under itself. + Assert.False(await store.TryReclaimExpiredAsync("renewed", now, "owner", JobStatus.Queued, cancellationToken: cancellationToken)); + // Still owned by the presumed-dead node and still expired -> reclaimed. + Assert.True(await store.TryReclaimExpiredAsync("expired", now, "owner", JobStatus.Queued, cancellationToken: cancellationToken)); + + Assert.Equal(JobStatus.Queued, (await store.GetAsync("expired", cancellationToken))!.Status); + Assert.Equal(JobStatus.Processing, (await store.GetAsync("renewed", cancellationToken))!.Status); + } + [Fact] public async Task CreateIfAbsentAsync_WithExistingJob_DoesNotOverwriteStateAsync() { @@ -278,4 +359,16 @@ public async Task RunAsync(CancellationToken cancellationToken = defa } } } + + private sealed class ProgressJob : IJobWithExecutionContext + { + public JobExecutionContext? ExecutionContext { get; set; } + + public async Task RunAsync(CancellationToken cancellationToken = default) + { + var context = ExecutionContext!; + await context.ReportProgressAsync(75, $"{context.JobId}:{context.Attempt}", cancellationToken); + return JobResult.Success; + } + } } diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 997614ec2..4203b85aa 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -44,8 +44,8 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() await firstReceived.WaitAsync(TimeSpan.FromSeconds(2)); await secondReceived.WaitAsync(TimeSpan.FromSeconds(2)); - var firstStats = await transport.GetStatsAsync("subscriber-a", cancellationToken); - var secondStats = await transport.GetStatsAsync("subscriber-b", cancellationToken); + var firstStats = await transport.GetStatsAsync(first.Source, cancellationToken); + var secondStats = await transport.GetStatsAsync(second.Source, cancellationToken); Assert.Equal(1, firstStats.Completed); Assert.Equal(1, secondStats.Completed); } @@ -85,15 +85,65 @@ await pubSub.PublishBatchAsync([ ], cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - await WaitForCompletedAsync(transport, "billing-service", 2, cancellationToken); + await WaitForCompletedAsync(transport, first.Source, 2, cancellationToken); Assert.Equal(first.Topic, second.Topic); Assert.Equal(first.Subscription, second.Subscription); + Assert.Equal(first.Source, second.Source); // same topic + subscription -> one shared transport source Assert.NotEqual(first.Key, second.Key); Assert.Equal(2, deliveriesByMessageId.Count); Assert.All(deliveriesByMessageId.Values, count => Assert.Equal(1, count)); } + [Fact] + public async Task SubscribeAsync_SameSubscriptionOnTwoTopics_IsolatesPerTopicAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var pubSub = new PubSub(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var ordersReceived = new List(); + var paymentsReceived = new List(); + var ordersSignal = new AsyncCountdownEvent(1); + var paymentsSignal = new AsyncCountdownEvent(1); + + // The same subscription identity ("shared") on two different topics. + await using var orders = await pubSub.SubscribeAsync((message, _) => + { + lock (ordersReceived) + ordersReceived.Add(message.Message.Data); + ordersSignal.Signal(); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { Topic = "orders", Subscription = "shared" }, cts.Token); + + await using var payments = await pubSub.SubscribeAsync((message, _) => + { + lock (paymentsReceived) + paymentsReceived.Add(message.Message.Data); + paymentsSignal.Signal(); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { Topic = "payments", Subscription = "shared" }, cts.Token); + + Assert.Equal(orders.Subscription, payments.Subscription); // same logical subscription identity + Assert.NotEqual(orders.Source, payments.Source); // but distinct topic-qualified transport sources + + // Publish one message to each topic. Each subscriber must receive only its own topic's message — proving both + // subscribers are live (not an always-broken one passing a negative-only assertion) and that they are isolated. + await pubSub.PublishAsync(new PreviewEvent { Data = "to-orders" }, new PubSubMessageOptions { Topic = "orders" }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "to-payments" }, new PubSubMessageOptions { Topic = "payments" }, cancellationToken); + + await ordersSignal.WaitAsync(TimeSpan.FromSeconds(2)); + await paymentsSignal.WaitAsync(TimeSpan.FromSeconds(2)); + + // Let any (incorrect) cross-topic delivery arrive before asserting each side received only its own message. + await Task.Delay(TimeSpan.FromMilliseconds(150), cancellationToken); + + Assert.Equal(new[] { "to-orders" }, ordersReceived); + Assert.Equal(new[] { "to-payments" }, paymentsReceived); + } + [Fact] public async Task PublishBatchAsync_DeliversAllMessagesAsync() { @@ -117,7 +167,7 @@ await pubSub.PublishBatchAsync([ ], cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - var stats = await transport.GetStatsAsync("batch-subscription", cancellationToken); + var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); Assert.Equal(2, stats.Completed); } @@ -209,7 +259,7 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() await pubSub.PublishAsync(new PreviewEvent { Data = "retry" }, cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - var stats = await transport.GetStatsAsync("retry-subscription", cancellationToken); + var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); Assert.Equal(1, stats.Completed); Assert.Equal(1, stats.Abandoned); } @@ -274,10 +324,11 @@ await pubSub.PublishBatchAsync(new object[] Assert.Equal("order-events", subscription.Topic); Assert.Equal("billing-service", subscription.Subscription); + Assert.Equal("order-events/billing-service", subscription.Source); // topic-qualified transport source Assert.Contains(typeof(PreviewEvent).FullName!, messageTypes); Assert.Contains(typeof(OtherEvent).FullName!, messageTypes); - var stats = await transport.GetStatsAsync("billing-service", cancellationToken); + var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); Assert.Equal(2, stats.Completed); } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index dc0909ed1..41c5879f7 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -109,19 +109,6 @@ public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() await Assert.ThrowsAsync(async () => await message.RenewLockAsync(cancellationToken: cancellationToken)); } - [Fact] - public async Task ReportProgressAsync_WhenUntracked_ThrowsAsync() - { - var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); - - await queue.EnqueueAsync(new PreviewWorkItem { Data = "progress" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - Assert.NotNull(message); - - await Assert.ThrowsAsync(async () => await message.ReportProgressAsync(50, "half", cancellationToken)); - } - [Fact] public async Task RejectAsync_Terminal_DeadLettersAsync() { @@ -515,6 +502,47 @@ await queue.EnqueueBatchAsync(new object[] await second.CompleteAsync(cancellationToken); } + [Fact] + public async Task StartConsumerAsync_WithGroupedInterfaceRoute_DeserializesConcreteTypeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var routing = new MessageRoutingOptionsBuilder() + .MapQueue("grouped-work", typeof(IGroupedWorkItem)) + .Build(); + await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var received = new ConcurrentDictionary(); + var signal = new AsyncCountdownEvent(2); + + // An interface-typed consumer receives the concrete payload (assignable to the interface), not raw bytes — + // the core resolves the concrete type from the message-type header and deserializes that. + await using var consumer = await queue.StartConsumerAsync((message, _) => + { + string? data = message.Message switch + { + PreviewWorkItem p => p.Data, + OtherWorkItem o => o.Data, + _ => null + }; + received[message.Message.GetType()] = data; + signal.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + await queue.EnqueueBatchAsync(new object[] + { + new PreviewWorkItem { Data = "one" }, + new OtherWorkItem { Data = "two" } + }, cancellationToken: cts.Token); + + await signal.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal("one", received[typeof(PreviewWorkItem)]); + Assert.Equal("two", received[typeof(OtherWorkItem)]); + } + [Fact] public async Task ReceiveAsync_WithDefaultQueueRoute_ReturnsRawMessageAsync() { @@ -674,6 +702,44 @@ public async Task StartConsumerAsync_UsesDefaultRetryPolicyMaxAttempts_WhenConsu Assert.Equal(2, attempts); } + [Fact] + public async Task DisposeAsync_RespectsTransportOwnershipAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // Non-owning client (shared transport): disposing the client leaves the transport usable. + var shared = new InMemoryMessageTransport(); + var nonOwning = new MessageQueue(shared, new QueueOptions { OwnsTransport = false }); + await nonOwning.DisposeAsync(); + await shared.SendAsync("still-alive", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken); + await shared.DisposeAsync(); + + // Owning client (default): disposing the client disposes the transport. + var owned = new InMemoryMessageTransport(); + var owning = new MessageQueue(owned); + await owning.DisposeAsync(); + await Assert.ThrowsAsync(async () => + await owned.SendAsync("dead", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken)); + } + + [Fact] + public async Task DiBuiltClients_ShareTransport_DisposedExactlyOnceAsync() + { + var transport = new DisposeCountingTransport(); + var services = new ServiceCollection(); + services.AddFoundatio().Messaging.UseTransport(transport); + await using var provider = services.BuildServiceProvider(); + + // Both clients resolve the same singleton transport. + _ = provider.GetRequiredService(); + _ = provider.GetRequiredService(); + + await provider.DisposeAsync(); + + // The container owns the shared transport singleton; neither client disposes it, so it is disposed once. + Assert.Equal(1, transport.DisposeCount); + } + private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore store, IMessageTransport transport) { var serviceProvider = new ServiceCollection().BuildServiceProvider(); @@ -818,4 +884,22 @@ public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) public ValueTask DisposeAsync() => ValueTask.CompletedTask; } + + // Counts dispose calls without an idempotency guard, so a double-dispose (the bug item 6 fixes) would show as > 1. + private sealed class DisposeCountingTransport : IMessageTransport + { + public int DisposeCount { get; private set; } + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + => Task.FromResult(new SendResult { Items = Array.Empty() }); + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + + public ValueTask DisposeAsync() + { + DisposeCount++; + return ValueTask.CompletedTask; + } + } } \ No newline at end of file From 84298b734389a6e04298dbdf9cade7851c98707b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 29 Jun 2026 16:44:03 -0500 Subject: [PATCH 16/57] feat: add AWS SQS/SNS transport and make the conformance harness cross-transport Adds a temporary in-repo Foundatio.Aws provider (AwsMessageTransport over SQS/SNS) to validate the redesigned IMessageTransport contract against a real broker, plus the contract refinements that validation surfaced. Verified against LocalStack: 8 conformance tests pass, 5 skip for capabilities SQS lacks (priority, per-message expiration, push, transport-native dead-letter); in-memory conformance and the full messaging/queue/jobs suite remain green. Transport contract refinements driven by the AWS implementation: - TransportSendOptions.DestinationRole: the caller states queue vs topic so a transport routes without inferring (SNS publish vs SQS send). MessageClientCore sets it from the dispatch kind. - TransportMessage.ContentType: lets a text-native broker (SQS/SNS) store a text body (e.g. JSON) directly instead of base64; binary still base64s. - MessageDestinationStats: lifetime counters (Enqueued/Dequeued/Completed/Abandoned/Errors/Timeouts) are now nullable (null = not reported, e.g. SQS exposes no lifetime completed count); Queued/Working/Deadletter remain best-effort gauges. - ReceiptExpiredException documented as a best-effort, transport-specific signal (SQS delete is idempotent). - InMemoryMessageTransport now wakes a blocked receive when a visibility window lapses (reclaim timer), matching real brokers so the harness can long-poll uniformly. AWS provider: SQS queues + SNS topics/subscriptions (raw delivery + queue policy), capability max-bounds (15-min delay, 12h visibility/redelivery), well-known headers surfaced as native attributes for SNS filter policies, ResourcePrefix for run isolation, LocalStack docker-compose + README. Harness: whole-second timing windows, eventual-consistency-tolerant stats (gauges only), and capability/opt-in gating so the suite runs across in-memory and real brokers. Co-Authored-By: Claude Opus 4.8 --- Foundatio.slnx | 4 + src/Foundatio.Aws/AwsMessageTransport.cs | 508 ++++++++++++++++++ .../AwsMessageTransportOptions.cs | 80 +++ src/Foundatio.Aws/Foundatio.Aws.csproj | 12 + .../MessageTransportConformanceTests.cs | 70 ++- .../Messaging/InMemoryMessageTransport.cs | 43 ++ src/Foundatio/Messaging/MessageClientCore.cs | 18 +- src/Foundatio/Messaging/MessageQueue.cs | 2 +- src/Foundatio/Messaging/MessageTransport.cs | 36 +- src/Foundatio/Messaging/PubSub.cs | 2 +- .../AwsMessageTransportConformanceTests.cs | 76 +++ .../AwsMessageTransportTests.cs | 71 +++ .../Foundatio.Aws.Tests.csproj | 6 + tests/Foundatio.Aws.Tests/README.md | 33 ++ tests/Foundatio.Aws.Tests/docker-compose.yml | 10 + 15 files changed, 936 insertions(+), 35 deletions(-) create mode 100644 src/Foundatio.Aws/AwsMessageTransport.cs create mode 100644 src/Foundatio.Aws/AwsMessageTransportOptions.cs create mode 100644 src/Foundatio.Aws/Foundatio.Aws.csproj create mode 100644 tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs create mode 100644 tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs create mode 100644 tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj create mode 100644 tests/Foundatio.Aws.Tests/README.md create mode 100644 tests/Foundatio.Aws.Tests/docker-compose.yml diff --git a/Foundatio.slnx b/Foundatio.slnx index ebe457b12..f674e0d53 100644 --- a/Foundatio.slnx +++ b/Foundatio.slnx @@ -13,8 +13,12 @@ + + + + diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs new file mode 100644 index 000000000..653a76030 --- /dev/null +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -0,0 +1,508 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS; +using Amazon.SQS.Model; +using Amazon.SimpleNotificationService; +using Amazon.SimpleNotificationService.Model; +using SnsMessageAttributeValue = Amazon.SimpleNotificationService.Model.MessageAttributeValue; +using SqsMessageAttributeValue = Amazon.SQS.Model.MessageAttributeValue; +using SqsMessage = Amazon.SQS.Model.Message; + +namespace Foundatio.Messaging; + +/// +/// An over AWS SQS (queues + competing-consumer subscriptions) and SNS (topics). This +/// is a temporary in-repo provider used to validate the redesigned transport contract against a real broker. Queue and +/// subscription destinations are SQS queues; topic destinations are SNS topics fanned out to SQS subscription queues. +/// +/// +/// Capability mapping: pull receive (SQS long poll), visibility timeout, redelivery delay (ChangeMessageVisibility, +/// 12h cap), delayed delivery (SQS DelaySeconds, 15-minute cap), provisioning, and stats. SQS has no per-message +/// priority, per-message TTL, or push delivery, and no transport-native dead-letter that the core controls the timing +/// of, so those capabilities are intentionally not implemented (the core owns retry/dead-lettering). +/// +public sealed class AwsMessageTransport : IMessageTransport, ISupportsPull, ISupportsVisibilityTimeout, + ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsDelayedDelivery, ISupportsProvisioning, ISupportsStats, ITransportInfo +{ + private const string HeadersAttributeName = "fnd.headers"; + private const string EncodingAttributeName = "fnd.encoding"; + + // Well-known headers surfaced as native message attributes (in addition to the authoritative JSON blob) so brokers + // can filter/route on them — e.g. SNS subscription filter policies match on native attributes. + private static readonly string[] WellKnownNativeHeaders = [KnownHeaders.MessageType, KnownHeaders.Priority, KnownHeaders.CorrelationId]; + + private static readonly IReadOnlySet _supportedRoles = + new HashSet { DestinationRole.Queue, DestinationRole.Topic, DestinationRole.Subscription, DestinationRole.Binding }; + + private readonly AwsMessageTransportOptions _options; + private readonly Lazy _sqs; + private readonly Lazy _sns; + private readonly ConcurrentDictionary _queueUrls = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _topicArns = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _roles = new(StringComparer.Ordinal); + private int _isDisposed; + + public AwsMessageTransport(AwsMessageTransportOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _sqs = new Lazy(CreateSqsClient); + _sns = new Lazy(CreateSnsClient); + } + + public AwsMessageTransport(string connectionString) : this(AwsMessageTransportOptions.FromConnectionString(connectionString)) { } + + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public OrderingGuarantee Ordering => OrderingGuarantee.None; + public IReadOnlySet SupportedRoles => _supportedRoles; + public int? MaxBatchSize => null; // sends are issued per message + public long? MaxMessageBytes => 262144; // 256 KB SQS/SNS limit + + public TimeSpan? MaxDeliveryDelay => TimeSpan.FromMinutes(15); // SQS DelaySeconds maximum + public TimeSpan? MaxRedeliveryDelay => TimeSpan.FromHours(12); // SQS ChangeMessageVisibility maximum + public TimeSpan? MaxVisibilityTimeout => TimeSpan.FromHours(12); // SQS visibility maximum + + public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(messages); + + var items = new List(messages.Count); + + // The caller states the destination role, so route without inferring: a topic publishes to SNS, anything else + // sends to an SQS queue. + if (options.DestinationRole == DestinationRole.Topic) + { + string topicArn = await ResolveTopicArnAsync(destination, ct).ConfigureAwait(false); + foreach (var message in messages) + { + var (body, encoding) = EncodeBody(message); + var response = await _sns.Value.PublishAsync(new PublishRequest + { + TopicArn = topicArn, + Message = body, + MessageAttributes = BuildAttributes(message.Headers, encoding, static value => new SnsMessageAttributeValue { DataType = "String", StringValue = value }) + }, ct).ConfigureAwait(false); + + items.Add(new SendItemResult { MessageId = response.MessageId, Success = true }); + } + + return new SendResult { Items = items }; + } + + string queueUrl = await ResolveQueueUrlAsync(destination, ct).ConfigureAwait(false); + int? delaySeconds = ToDelaySeconds(options.DeliverAt); + foreach (var message in messages) + { + var (body, encoding) = EncodeBody(message); + var request = new SendMessageRequest + { + QueueUrl = queueUrl, + MessageBody = body, + MessageAttributes = BuildAttributes(message.Headers, encoding, static value => new SqsMessageAttributeValue { DataType = "String", StringValue = value }) + }; + if (delaySeconds is { } delay) + request.DelaySeconds = delay; + + var response = await _sqs.Value.SendMessageAsync(request, ct).ConfigureAwait(false); + items.Add(new SendItemResult { MessageId = response.MessageId, Success = true }); + } + + return new SendResult { Items = items }; + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + { + return ReceiveAsync(source, request, _options.DefaultVisibilityTimeout, ct); + } + + public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(request); + + string queueUrl = await ResolveQueueUrlAsync(source, ct).ConfigureAwait(false); + + var sqsRequest = new ReceiveMessageRequest + { + QueueUrl = queueUrl, + MaxNumberOfMessages = Math.Clamp(request.MaxMessages <= 0 ? 1 : request.MaxMessages, 1, 10), + VisibilityTimeout = (int)Math.Clamp(visibility.TotalSeconds, 0, 43200), + MessageAttributeNames = ["All"], + MessageSystemAttributeNames = ["All"] + }; + if (request.MaxWaitTime is { } wait) + sqsRequest.WaitTimeSeconds = (int)Math.Clamp(wait.TotalSeconds, 0, 20); + + var response = await _sqs.Value.ReceiveMessageAsync(sqsRequest, ct).ConfigureAwait(false); + if (response.Messages is not { Count: > 0 }) + return []; + + var entries = new List(response.Messages.Count); + foreach (var message in response.Messages) + { + entries.Add(new TransportEntry + { + Id = message.MessageId, + Destination = source, + Body = DecodeBody(message.Body, GetAttribute(message.MessageAttributes, EncodingAttributeName)), + Headers = FromSqsAttributes(message.MessageAttributes), + DeliveryCount = GetReceiveCount(message), + Receipt = new Receipt { TransportState = message.ReceiptHandle } + }); + } + + return entries; + } + + public async Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + ThrowIfDisposed(); + string queueUrl = await ResolveQueueUrlAsync(entry.Destination, ct).ConfigureAwait(false); + await _sqs.Value.DeleteMessageAsync(queueUrl, GetReceiptHandle(entry), ct).ConfigureAwait(false); + } + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + return AbandonAsync(entry, TimeSpan.Zero, ct); + } + + public async Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct) + { + ThrowIfDisposed(); + string queueUrl = await ResolveQueueUrlAsync(entry.Destination, ct).ConfigureAwait(false); + // Returning a message to the queue is a visibility change to the requested delay (0 = immediately visible). + await _sqs.Value.ChangeMessageVisibilityAsync(queueUrl, GetReceiptHandle(entry), (int)Math.Clamp(redeliveryDelay.TotalSeconds, 0, 43200), ct).ConfigureAwait(false); + } + + public async Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct) + { + ThrowIfDisposed(); + string queueUrl = await ResolveQueueUrlAsync(entry.Destination, ct).ConfigureAwait(false); + int seconds = (int)Math.Clamp((duration ?? _options.DefaultVisibilityTimeout).TotalSeconds, 0, 43200); + await _sqs.Value.ChangeMessageVisibilityAsync(queueUrl, GetReceiptHandle(entry), seconds, ct).ConfigureAwait(false); + } + + public async Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(declarations); + + foreach (var declaration in declarations) + { + switch (declaration.Role) + { + case DestinationRole.Topic: + await ResolveTopicArnAsync(declaration.Name, ct).ConfigureAwait(false); + break; + case DestinationRole.Subscription: + case DestinationRole.Binding: + await EnsureSubscriptionAsync(declaration.Name, declaration.Source, ct).ConfigureAwait(false); + break; + default: + await ResolveQueueUrlAsync(declaration.Name, ct).ConfigureAwait(false); + break; + } + } + } + + public async Task DeleteAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(name); + + if (_roles.TryGetValue(name, out var role) && role == DestinationRole.Topic) + { + if (_topicArns.TryRemove(name, out string? arn)) + await _sns.Value.DeleteTopicAsync(arn, ct).ConfigureAwait(false); + } + else if (_queueUrls.TryRemove(name, out string? url)) + { + await _sqs.Value.DeleteQueueAsync(url, ct).ConfigureAwait(false); + } + + _roles.TryRemove(name, out _); + } + + public async Task ExistsAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(name); + + if (_roles.TryGetValue(name, out var role) && role == DestinationRole.Topic) + return _topicArns.ContainsKey(name); + + try + { + await _sqs.Value.GetQueueUrlAsync(ResourceName(name), ct).ConfigureAwait(false); + return true; + } + catch (QueueDoesNotExistException) + { + return false; + } + } + + public async Task GetStatsAsync(string destination, CancellationToken ct) + { + ThrowIfDisposed(); + string queueUrl = await ResolveQueueUrlAsync(destination, ct).ConfigureAwait(false); + var response = await _sqs.Value.GetQueueAttributesAsync(new GetQueueAttributesRequest + { + QueueUrl = queueUrl, + AttributeNames = ["All"] + }, ct).ConfigureAwait(false); + + return new MessageDestinationStats + { + Queued = response.ApproximateNumberOfMessages, + Working = response.ApproximateNumberOfMessagesNotVisible + }; + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _isDisposed, 1) == 1) + return; + + if (_sqs.IsValueCreated) + _sqs.Value.Dispose(); + if (_sns.IsValueCreated) + _sns.Value.Dispose(); + + await ValueTask.CompletedTask.ConfigureAwait(false); + } + + private async Task EnsureSubscriptionAsync(string subscriptionName, string? topicName, CancellationToken ct) + { + string queueUrl = await ResolveQueueUrlAsync(subscriptionName, ct).ConfigureAwait(false); + _roles[subscriptionName] = DestinationRole.Subscription; + + if (String.IsNullOrEmpty(topicName)) + return; + + string topicArn = await ResolveTopicArnAsync(topicName, ct).ConfigureAwait(false); + string queueArn = await GetQueueArnAsync(queueUrl, ct).ConfigureAwait(false); + + // Allow the topic to deliver to the queue, then subscribe with raw delivery so the SQS body/attributes match a + // direct SQS send (no SNS envelope). + await _sqs.Value.SetQueueAttributesAsync(new SetQueueAttributesRequest + { + QueueUrl = queueUrl, + Attributes = new Dictionary { ["Policy"] = BuildQueuePolicy(queueArn, topicArn) } + }, ct).ConfigureAwait(false); + + await _sns.Value.SubscribeAsync(new SubscribeRequest + { + TopicArn = topicArn, + Protocol = "sqs", + Endpoint = queueArn, + Attributes = new Dictionary { ["RawMessageDelivery"] = "true" }, + ReturnSubscriptionArn = true + }, ct).ConfigureAwait(false); + } + + private async Task ResolveQueueUrlAsync(string name, CancellationToken ct) + { + if (_queueUrls.TryGetValue(name, out string? cached)) + return cached; + + string resourceName = ResourceName(name); + try + { + var response = await _sqs.Value.GetQueueUrlAsync(resourceName, ct).ConfigureAwait(false); + _queueUrls[name] = response.QueueUrl; + _roles.TryAdd(name, DestinationRole.Queue); + return response.QueueUrl; + } + catch (QueueDoesNotExistException) when (_options.AutoCreateDestinations) + { + var response = await _sqs.Value.CreateQueueAsync(new CreateQueueRequest { QueueName = resourceName }, ct).ConfigureAwait(false); + _queueUrls[name] = response.QueueUrl; + _roles.TryAdd(name, DestinationRole.Queue); + return response.QueueUrl; + } + } + + private async Task ResolveTopicArnAsync(string name, CancellationToken ct) + { + if (_topicArns.TryGetValue(name, out string? cached)) + return cached; + + // CreateTopic is idempotent and returns the ARN of an existing topic with the same name. + var response = await _sns.Value.CreateTopicAsync(new CreateTopicRequest { Name = ResourceName(name) }, ct).ConfigureAwait(false); + _topicArns[name] = response.TopicArn; + _roles[name] = DestinationRole.Topic; + return response.TopicArn; + } + + // SQS queue names and SNS topic names allow alphanumerics, hyphens and underscores; the logical destination name + // already conforms, so we only prepend the configured prefix. + private string ResourceName(string logicalName) => _options.ResourcePrefix + logicalName; + + private async Task GetQueueArnAsync(string queueUrl, CancellationToken ct) + { + var response = await _sqs.Value.GetQueueAttributesAsync(new GetQueueAttributesRequest + { + QueueUrl = queueUrl, + AttributeNames = ["QueueArn"] + }, ct).ConfigureAwait(false); + return response.QueueARN; + } + + private static string BuildQueuePolicy(string queueArn, string topicArn) + { + return JsonSerializer.Serialize(new + { + Version = "2012-10-17", + Statement = new[] + { + new + { + Effect = "Allow", + Principal = new { Service = "sns.amazonaws.com" }, + Action = "sqs:SendMessage", + Resource = queueArn, + Condition = new { ArnEquals = new Dictionary { ["aws:SourceArn"] = topicArn } } + } + } + }); + } + + private int? ToDelaySeconds(DateTimeOffset? deliverAt) + { + if (deliverAt is not { } at) + return null; + + double seconds = (at - DateTimeOffset.UtcNow).TotalSeconds; + if (seconds <= 0) + return null; + + return (int)Math.Clamp(seconds, 1, 900); // SQS DelaySeconds maximum is 900 (15 minutes) + } + + private static int GetReceiveCount(SqsMessage message) + { + if (message.Attributes is not null && message.Attributes.TryGetValue("ApproximateReceiveCount", out string? value) && Int32.TryParse(value, out int count) && count > 0) + return count; + return 1; + } + + private static string GetReceiptHandle(TransportEntry entry) + { + return entry.Receipt.TransportState as string + ?? throw new ReceiptExpiredException("The transport entry does not carry an SQS receipt handle."); + } + + // A text body (e.g. JSON, the default) is stored as-is so it is human-readable in the console and avoids base64 + // overhead; anything else is base64-encoded so arbitrary bytes round-trip through SQS/SNS string bodies. The chosen + // encoding is recorded in a native attribute for the receive side. + private static (string Body, string Encoding) EncodeBody(TransportMessage message) + { + return IsTextContent(message.ContentType) + ? (Encoding.UTF8.GetString(message.Body.Span), "text") + : (Convert.ToBase64String(message.Body.Span), "base64"); + } + + private static ReadOnlyMemory DecodeBody(string body, string? encoding) + { + if (String.IsNullOrEmpty(body)) + return ReadOnlyMemory.Empty; + + return String.Equals(encoding, "text", StringComparison.Ordinal) + ? Encoding.UTF8.GetBytes(body) + : Convert.FromBase64String(body); + } + + private static bool IsTextContent(string? contentType) + { + return !String.IsNullOrEmpty(contentType) + && (contentType.Contains("json", StringComparison.OrdinalIgnoreCase) + || contentType.Contains("xml", StringComparison.OrdinalIgnoreCase) + || contentType.StartsWith("text/", StringComparison.OrdinalIgnoreCase)); + } + + private static Dictionary BuildAttributes(MessageHeaders headers, string encoding, Func stringAttribute) + { + var attributes = new Dictionary(StringComparer.Ordinal) + { + [HeadersAttributeName] = stringAttribute(EncodeHeaders(headers)), + [EncodingAttributeName] = stringAttribute(encoding) + }; + + foreach (string name in WellKnownNativeHeaders) + { + string? value = headers.GetValueOrDefault(name); + if (!String.IsNullOrEmpty(value)) + attributes[name] = stringAttribute(value); + } + + return attributes; + } + + private static string? GetAttribute(Dictionary? attributes, string name) + { + return attributes is not null && attributes.TryGetValue(name, out var value) ? value.StringValue : null; + } + + private static MessageHeaders FromSqsAttributes(Dictionary? attributes) + { + if (attributes is null || !attributes.TryGetValue(HeadersAttributeName, out var value) || String.IsNullOrEmpty(value.StringValue)) + return MessageHeaders.Empty; + + return DecodeHeaders(value.StringValue); + } + + private static string EncodeHeaders(MessageHeaders headers) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var header in headers) + map[header.Key] = header.Value; + return JsonSerializer.Serialize(map); + } + + private static MessageHeaders DecodeHeaders(string json) + { + var map = JsonSerializer.Deserialize>(json); + return map is null ? MessageHeaders.Empty : MessageHeaders.Create(map); + } + + private IAmazonSQS CreateSqsClient() + { + var config = new AmazonSQSConfig(); + ApplyEndpoint(config); + return _options.Credentials is { } credentials ? new AmazonSQSClient(credentials, config) : new AmazonSQSClient(config); + } + + private IAmazonSimpleNotificationService CreateSnsClient() + { + var config = new AmazonSimpleNotificationServiceConfig(); + ApplyEndpoint(config); + return _options.Credentials is { } credentials ? new AmazonSimpleNotificationServiceClient(credentials, config) : new AmazonSimpleNotificationServiceClient(config); + } + + private void ApplyEndpoint(Amazon.Runtime.ClientConfig config) + { + if (!String.IsNullOrEmpty(_options.ServiceUrl)) + { + config.ServiceURL = _options.ServiceUrl; + config.AuthenticationRegion = (_options.Region ?? Amazon.RegionEndpoint.USEast1).SystemName; + } + else if (_options.Region is { } region) + { + config.RegionEndpoint = region; + } + } + + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + } +} diff --git a/src/Foundatio.Aws/AwsMessageTransportOptions.cs b/src/Foundatio.Aws/AwsMessageTransportOptions.cs new file mode 100644 index 000000000..8b02fd267 --- /dev/null +++ b/src/Foundatio.Aws/AwsMessageTransportOptions.cs @@ -0,0 +1,80 @@ +using System; +using Amazon; +using Amazon.Runtime; + +namespace Foundatio.Messaging; + +public class AwsMessageTransportOptions +{ + /// AWS credentials. When null, the SDK's default credential chain is used. + public AWSCredentials? Credentials { get; set; } + + /// AWS region. When null, the SDK's default region resolution is used (ignored when is set). + public RegionEndpoint? Region { get; set; } + + /// Custom service endpoint, e.g. http://localhost:4566 for LocalStack. + public string? ServiceUrl { get; set; } + + /// Create queues/topics/subscriptions on demand when sending or receiving (in addition to explicit provisioning). + public bool AutoCreateDestinations { get; set; } = true; + + /// + /// Optional prefix applied to the underlying SQS queue and SNS topic names (not the logical destination names used + /// by callers). Useful to isolate runs/environments on a shared broker — e.g. a unique prefix per conformance run + /// so leftover messages from a prior run can't leak in. + /// + public string ResourcePrefix { get; set; } = ""; + + /// Default receive visibility timeout when none is supplied. Maps to the SQS visibility window. + public TimeSpan DefaultVisibilityTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Parses a connection string of the form + /// serviceurl=http://localhost:4566;accesskey=...;secretkey=...;region=us-east-1 into options. Any subset of + /// keys may be provided; unknown keys are ignored. + /// + public static AwsMessageTransportOptions FromConnectionString(string connectionString) + { + ArgumentException.ThrowIfNullOrEmpty(connectionString); + + string? accessKey = null, secretKey = null, region = null, serviceUrl = null; + foreach (string pair in connectionString.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + int separator = pair.IndexOf('='); + if (separator < 0) + continue; + + string key = pair[..separator].Trim().ToLowerInvariant().Replace(" ", ""); + string value = pair[(separator + 1)..].Trim(); + + switch (key) + { + case "accesskey": + case "accesskeyid": + case "id": + accessKey = value; + break; + case "secretkey": + case "secret": + secretKey = value; + break; + case "region": + case "endpoint": + region = value; + break; + case "serviceurl": + case "service": + serviceUrl = value; + break; + } + } + + var options = new AwsMessageTransportOptions { ServiceUrl = serviceUrl }; + if (!String.IsNullOrEmpty(accessKey) && !String.IsNullOrEmpty(secretKey)) + options.Credentials = new BasicAWSCredentials(accessKey, secretKey); + if (!String.IsNullOrEmpty(region)) + options.Region = RegionEndpoint.GetBySystemName(region); + + return options; + } +} diff --git a/src/Foundatio.Aws/Foundatio.Aws.csproj b/src/Foundatio.Aws/Foundatio.Aws.csproj new file mode 100644 index 000000000..2c8635702 --- /dev/null +++ b/src/Foundatio.Aws/Foundatio.Aws.csproj @@ -0,0 +1,12 @@ + + + AWS (SQS/SNS) IMessageTransport for Foundatio messaging. Temporary in-repo provider for validating the redesigned transport contract against a real broker. + + + + + + + + + diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index f52c3d535..9257a56e6 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -73,10 +73,10 @@ public virtual async Task CanSendAndReceiveBatchAsync() if (transport is ISupportsStats stats) { - MessageDestinationStats queueStats = await stats.GetStatsAsync("orders", TestCancellationToken); - Assert.Equal(0, queueStats.Queued); - Assert.Equal(0, queueStats.Working); - Assert.Equal(2, queueStats.Completed); + // Assert only the point-in-time gauges every broker can report, and tolerate eventual consistency + // (e.g. SQS ApproximateNumberOf* lag). Lifetime counters such as Completed are not universally + // available across transports, so they are not part of the shared contract. + await AssertQueueDrainedAsync(stats, "orders", TestCancellationToken); } } finally @@ -192,9 +192,10 @@ await EnsureAsync(transport, new DestinationDeclaration { Name = "orders-subscription-a", Role = DestinationRole.Subscription, Source = "orders-topic" }, new DestinationDeclaration { Name = "orders-subscription-b", Role = DestinationRole.Subscription, Source = "orders-topic" }); - await transport.SendAsync("orders-topic", [CreateMessage("fanout")], new TransportSendOptions(), TestCancellationToken); + // The caller states the destination role; publishing to a topic must set DestinationRole.Topic. + await transport.SendAsync("orders-topic", [CreateMessage("fanout")], new TransportSendOptions { DestinationRole = DestinationRole.Topic }, TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("orders-subscription-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync("orders-subscription-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); var second = Assert.Single(await pull.ReceiveAsync("orders-subscription-b", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); Assert.Equal("fanout", ReadBody(first)); @@ -350,16 +351,19 @@ public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() await EnsureAsync(transport, new DestinationDeclaration { Name = "visibility", Role = DestinationRole.Queue }); await transport.SendAsync("visibility", [CreateMessage("lease")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(250), TestCancellationToken)); + // Whole-second visibility window: real brokers (e.g. SQS) only support second-resolution visibility timeouts. + var visibilityWindow = TimeSpan.FromSeconds(2); + var first = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibilityWindow, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); // Still within the visibility window: a competing receive must not see the in-flight message. - var hidden = await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TimeSpan.FromMilliseconds(250), TestCancellationToken); + var hidden = await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibilityWindow, TestCancellationToken); Assert.Empty(hidden); - // After the visibility window lapses without settlement, the message must be redelivered (at-least-once). - await Task.Delay(TimeSpan.FromMilliseconds(400), TestCancellationToken); - var second = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(250), TestCancellationToken)); + // After the visibility window lapses without settlement the message must be redelivered (at-least-once). A + // long poll observes the lapse — a transport wakes a blocked receive when a visibility window expires — so + // this is robust to coarse/variable redelivery latency without a fixed sleep. + var second = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = visibilityWindow + TimeSpan.FromSeconds(5) }, visibilityWindow, TestCancellationToken)); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.DeliveryCount); @@ -385,17 +389,19 @@ public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayA await EnsureAsync(transport, new DestinationDeclaration { Name = "redelivery-delay", Role = DestinationRole.Queue }); await transport.SendAsync("redelivery-delay", [CreateMessage("delay-me")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); - await redelivery.AbandonAsync(first, TimeSpan.FromMilliseconds(300), TestCancellationToken); + // Whole-second redelivery delay: SQS serves this via ChangeMessageVisibility, which is second-resolution. + var redeliveryDelay = TimeSpan.FromSeconds(2); + await redelivery.AbandonAsync(first, redeliveryDelay, TestCancellationToken); // Within the delay window the message must not be visible again. - var early = await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + var early = await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); Assert.Empty(early); - // After the delay lapses it is redelivered with an incremented delivery count. - var second = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + // After the delay lapses it is redelivered with an incremented delivery count. Long poll for robustness. + var second = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = redeliveryDelay + TimeSpan.FromSeconds(5) }, TestCancellationToken)); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.DeliveryCount); Assert.Equal("delay-me", ReadBody(second)); @@ -422,17 +428,20 @@ public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() await EnsureAsync(transport, new DestinationDeclaration { Name = "lock-renewal", Role = DestinationRole.Queue }); await transport.SendAsync("lock-renewal", [CreateMessage("hold")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TimeSpan.FromMilliseconds(300), TestCancellationToken)); + // Whole-second windows so the test maps onto second-resolution brokers (e.g. SQS). + var originalWindow = TimeSpan.FromSeconds(2); + var renewedWindow = TimeSpan.FromSeconds(8); + var first = Assert.Single(await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, originalWindow, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); // Renew before the original window lapses, extending it well past the original expiry. - await Task.Delay(TimeSpan.FromMilliseconds(150), TestCancellationToken); - await lockRenewal.RenewLockAsync(first, TimeSpan.FromSeconds(2), TestCancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), TestCancellationToken); + await lockRenewal.RenewLockAsync(first, renewedWindow, TestCancellationToken); - // Past the original 300ms window but inside the renewed window: the message must still be held, so a - // competing receive sees nothing rather than a premature redelivery. - await Task.Delay(TimeSpan.FromMilliseconds(300), TestCancellationToken); - var held = await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TimeSpan.FromMilliseconds(300), TestCancellationToken); + // Past the original window but inside the renewed window: the message must still be held, so a competing + // receive sees nothing rather than a premature redelivery. + await Task.Delay(originalWindow, TestCancellationToken); + var held = await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, originalWindow, TestCancellationToken); Assert.Empty(held); await transport.CompleteAsync(first, TestCancellationToken); @@ -506,6 +515,21 @@ private async ValueTask CleanupTransportIfNotNullAsync(IMessageTransport? transp await CleanupTransportAsync(transport); } + // Polls until the destination reports no queued or in-flight messages (the point-in-time gauges every broker can + // report), tolerating transports whose stats are only eventually consistent (e.g. SQS ApproximateNumberOf*). + private async Task AssertQueueDrainedAsync(ISupportsStats stats, string destination, CancellationToken cancellationToken) + { + var current = await stats.GetStatsAsync(destination, cancellationToken); + for (int attempt = 0; attempt < 50 && (current.Queued != 0 || current.Working != 0); attempt++) + { + await Task.Delay(100, cancellationToken); + current = await stats.GetStatsAsync(destination, cancellationToken); + } + + Assert.Equal(0, current.Queued); + Assert.Equal(0, current.Working); + } + private static async Task EnsureAsync(IMessageTransport transport, params DestinationDeclaration[] declarations) { if (transport is ISupportsProvisioning provisioning) diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index f481e1a40..0a755d323 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -52,6 +52,11 @@ public Task SendAsync(string destination, IReadOnlyList _timeProvider.GetUtcNow()) throw new NotSupportedException($"Transport \"{GetType().Name}\" does not support native delayed delivery. Use the runtime-store scheduled dispatch fallback."); @@ -210,6 +215,8 @@ public Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, Cancellatio if (!state.InFlight.TryUpdate(receipt.LockToken, renewed, inFlight)) throw new ReceiptExpiredException(); + // Re-arm the reclaim wake for the extended window. + ScheduleReclaim(state, duration ?? _defaultLockRenewal); return Task.CompletedTask; } @@ -486,6 +493,37 @@ private void ScheduleRedelivery(string destination, StoredMessage message, TimeS timer.Dispose(); } + // Fires shortly after a visibility window lapses and reclaims any expired in-flight messages, which re-enqueues + // them and releases the destination's availability semaphore — waking a consumer blocked in a long receive. + // ReclaimExpired re-checks each message's current expiry, so a renewed or already-settled message is left alone. + private void ScheduleReclaim(DestinationState state, TimeSpan delay) + { + // Small buffer so the timer fires just after expiry rather than racing it (clock granularity). + var fireAfter = delay + TimeSpan.FromMilliseconds(50); + + ITimer? timer = null; + timer = _timeProvider.CreateTimer(timerState => + { + if (timer is not null && _redeliveryTimers.TryRemove(timer, out _)) + timer.Dispose(); + + if (Volatile.Read(ref _isDisposed) == 1) + return; + + try + { + state.ReclaimExpired(_timeProvider.GetUtcNow()); + } + catch (ObjectDisposedException) { } + catch (InvalidOperationException) { } // destination was completed/deleted between scheduling and firing + }, null, fireAfter, Timeout.InfiniteTimeSpan); + + _redeliveryTimers[timer] = 0; + + if (Volatile.Read(ref _isDisposed) == 1 && _redeliveryTimers.TryRemove(timer, out _)) + timer.Dispose(); + } + private bool TryReceive(string source, DestinationState state, TimeSpan? visibility, out TransportEntry entry) { while (state.TryDequeue(out var message)) @@ -501,6 +539,11 @@ private bool TryReceive(string source, DestinationState state, TimeSpan? visibil state.InFlight[receipt.LockToken] = new InFlightMessage(message, receipt, visibilityExpiresUtc); Interlocked.Increment(ref state.Dequeued); + // Schedule a reclaim at the visibility expiry so a consumer blocked in a long receive wakes when the lease + // lapses (matching real brokers like SQS), rather than only being reclaimed at the next receive call. + if (visibility is { } visibilityWindow) + ScheduleReclaim(state, visibilityWindow); + entry = new TransportEntry { Id = message.Id, diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index f8642495e..774625448 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -77,12 +77,13 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly Func _exceptionFactory; private readonly RetryPolicy _retryPolicy; private readonly IMessageTypeRegistry _typeRegistry; + private readonly string? _contentType; private readonly bool _ownsTransport; private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, - IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null) + IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null, string? contentType = null) { _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _serializer = serializer; @@ -93,6 +94,7 @@ public MessageClientCore(IMessageTransport transport, ISerializer serializer, IM _exceptionFactory = exceptionFactory; _retryPolicy = retryPolicy ?? new RetryPolicy(); _typeRegistry = typeRegistry ?? new MessageTypeRegistry(); + _contentType = contentType; _ownsTransport = ownsTransport; } @@ -110,7 +112,7 @@ public async Task SendAsync(ScheduledDispatchKind kind, Type messageType ThrowIfDisposed(); ValidateCapabilities(options.Priority, options.TimeToLive); - var sendOptions = BuildSendOptions(options); + var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); var transportMessage = CreateTransportMessage(message, messageType, options, messageId); @@ -133,7 +135,7 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable ThrowIfDisposed(); ValidateCapabilities(options.Priority, options.TimeToLive); - var sendOptions = BuildSendOptions(options); + var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; var grouped = new Dictionary>(StringComparer.Ordinal); int index = 0; @@ -538,10 +540,18 @@ private TransportMessage CreateTransportMessage(object message, Type messageType { Body = _serializer.SerializeToBytes(message), Headers = headers.Build(), - MessageId = messageId + MessageId = messageId, + ContentType = _contentType }; } + // A pub/sub publish targets a topic; everything else targets a queue. Stating the role lets the transport route + // without inferring (e.g. SNS publish vs. SQS send). + private static DestinationRole RoleFor(ScheduledDispatchKind kind) + { + return kind == ScheduledDispatchKind.PubSubMessage ? DestinationRole.Topic : DestinationRole.Queue; + } + private TransportSendOptions BuildSendOptions(MessageEnvelopeOptions options) { return new TransportSendOptions diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index c6cabd449..d710a0429 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -186,7 +186,7 @@ public MessageQueue(IMessageTransport transport, QueueOptions? options = null) options ??= new QueueOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes); + static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } public Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index adc665b95..49e1c900f 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -39,6 +39,13 @@ public sealed record TransportMessage public required ReadOnlyMemory Body { get; init; } public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; public string? MessageId { get; init; } + + /// + /// Content type of (e.g. application/json). A transport whose native wire format is text + /// (such as SQS/SNS) can store a text body directly when this indicates text, avoiding base64 overhead; null means + /// unknown, so a byte-safe encoding should be used. + /// + public string? ContentType { get; init; } } public sealed record TransportSendOptions @@ -47,6 +54,13 @@ public sealed record TransportSendOptions public DateTimeOffset? DeliverAt { get; init; } public string? DeduplicationId { get; init; } public string? PartitionKey { get; init; } + + /// + /// The role of the destination being sent to. Lets a transport route the send without inferring (for example, a + /// queue send to SQS vs. a topic publish to SNS) — the caller always knows whether it is sending to a queue or a + /// topic, so it states it rather than relying on prior provisioning. + /// + public DestinationRole DestinationRole { get; init; } = DestinationRole.Queue; } public sealed record TransportEntry @@ -73,15 +87,20 @@ public sealed record ReceiveRequest public sealed record MessageDestinationStats { + // Point-in-time gauges every transport can report (may be approximate / eventually consistent on real brokers, + // e.g. SQS ApproximateNumberOf*). public long Queued { get; init; } public long Working { get; init; } public long Deadletter { get; init; } - public long Enqueued { get; init; } - public long Dequeued { get; init; } - public long Completed { get; init; } - public long Abandoned { get; init; } - public long Errors { get; init; } - public long Timeouts { get; init; } + + // Lifetime counters. Not universally available — a transport that does not track a counter leaves it null (e.g. + // SQS exposes no lifetime "completed" count). Null means "not reported", distinct from a reported zero. + public long? Enqueued { get; init; } + public long? Dequeued { get; init; } + public long? Completed { get; init; } + public long? Abandoned { get; init; } + public long? Errors { get; init; } + public long? Timeouts { get; init; } } public sealed record SendItemResult @@ -98,6 +117,11 @@ public sealed record SendResult public bool AllSucceeded => Items.All(i => i.Success); } +/// +/// Thrown when a transport settle operation is given a receipt that has expired or was already settled. Strict receipt +/// validation is transport-specific: some brokers (e.g. SQS) treat settling with a stale receipt as idempotent and do +/// not raise, so callers must not depend on this exception for correctness — it is a best-effort safety signal. +/// public sealed class ReceiptExpiredException : Exception { public ReceiptExpiredException() : base("The transport receipt has expired or has already been settled.") { } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index cc78c1479..1b730b98d 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -93,7 +93,7 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) options ??= new PubSubOptions(); var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes); + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } public Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs b/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs new file mode 100644 index 000000000..25af9746f --- /dev/null +++ b/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Xunit; + +namespace Foundatio.Aws.Tests; + +/// +/// Runs the shared transport conformance suite against AWS SQS/SNS. Set the environment variable +/// FOUNDATIO_AWS_CONNECTION_STRING (e.g. serviceurl=http://localhost:4566;accesskey=test;secretkey=test;region=us-east-1 +/// for LocalStack, or real AWS credentials) to run; when it is not set every test is skipped. Capabilities SQS/SNS do +/// not support (priority, per-message expiration, push delivery, transport-native dead-letter) are skipped by the base +/// suite via their ISupports* capability checks. +/// +public class AwsMessageTransportConformanceTests : MessageTransportConformanceTests +{ + // One prefix per test run isolates these queues/topics from prior runs and other environments on the same broker. + private static readonly string RunPrefix = "fnd-conf-" + Guid.NewGuid().ToString("N")[..8] + "-"; + + public AwsMessageTransportConformanceTests(ITestOutputHelper output) : base(output) { } + + protected override IMessageTransport? CreateTransport() + { + string? connectionString = Environment.GetEnvironmentVariable("FOUNDATIO_AWS_CONNECTION_STRING"); + if (String.IsNullOrEmpty(connectionString)) + return null; // not configured -> the base suite skips every test + + var options = AwsMessageTransportOptions.FromConnectionString(connectionString); + options.ResourcePrefix = RunPrefix; + return new AwsMessageTransport(options); + } + + [Fact] + public override Task CanSendAndReceiveBatchAsync() => base.CanSendAndReceiveBatchAsync(); + + [Fact] + public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() => base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); + + // CompleteAsync_WithExpiredReceipt is intentionally not run for SQS: DeleteMessage with a stale/used receipt + // handle is idempotent and does not raise — strict receipt validation is a transport-specific behavior, not part + // of the shared contract, so only transports that guarantee it (e.g. the in-memory reference) opt in. + + [Fact] + public override Task SubscribeAsync_DeliversPushMessagesAsync() => base.SubscribeAsync_DeliversPushMessagesAsync(); + + [Fact] + public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() => base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); + + [Fact] + public override Task ReceiveAsync_RespectsPriorityAsync() => base.ReceiveAsync_RespectsPriorityAsync(); + + [Fact] + public override Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() => base.SendAsync_WithDeliverAt_DelaysVisibilityAsync(); + + [Fact] + public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() => base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); + + [Fact] + public override Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() => base.ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync(); + + [Fact] + public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() => base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); + + [Fact] + public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() => base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); + + [Fact] + public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() => base.RenewLockAsync_ExtendsVisibilityWindowAsync(); + + [Fact] + public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() => base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); + + [Fact] + public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() => base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); +} diff --git a/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs b/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs new file mode 100644 index 000000000..68cc82f21 --- /dev/null +++ b/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs @@ -0,0 +1,71 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Aws.Tests; + +public class AwsMessageTransportTests +{ + private static AwsMessageTransport? CreateTransport(string testName) + { + string? connectionString = Environment.GetEnvironmentVariable("FOUNDATIO_AWS_CONNECTION_STRING"); + if (String.IsNullOrEmpty(connectionString)) + return null; + + var options = AwsMessageTransportOptions.FromConnectionString(connectionString); + options.ResourcePrefix = $"fnd-{testName}-{Guid.NewGuid():N}"[..24] + "-"; + return new AwsMessageTransport(options); + } + + [Fact] + public async Task TextContentBody_RoundTripsThroughSqsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = CreateTransport("text"); + if (transport is null) + { + Assert.Skip("FOUNDATIO_AWS_CONNECTION_STRING not set."); + return; + } + + // Non-ASCII JSON exercises UTF-8 round-trip through the SQS string body (the text-content path that avoids base64). + string json = "{\"greeting\":\"héllo wörld\",\"n\":42}"; + await transport.EnsureAsync([new DestinationDeclaration { Name = "text-body", Role = DestinationRole.Queue }], cancellationToken); + + await transport.SendAsync("text-body", + [new TransportMessage { Body = Encoding.UTF8.GetBytes(json), ContentType = "application/json" }], + new TransportSendOptions(), cancellationToken); + + var entries = await transport.ReceiveAsync("text-body", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var entry = Assert.Single(entries); + Assert.Equal(json, Encoding.UTF8.GetString(entry.Body.Span)); + await transport.CompleteAsync(entry, cancellationToken); + } + + [Fact] + public async Task BinaryContentBody_RoundTripsThroughSqsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = CreateTransport("binary"); + if (transport is null) + { + Assert.Skip("FOUNDATIO_AWS_CONNECTION_STRING not set."); + return; + } + + // Non-UTF-8 bytes must still round-trip (via base64) when no text content type is declared. + byte[] payload = [0x00, 0x01, 0xFF, 0xFE, 0x10, 0x80]; + await transport.EnsureAsync([new DestinationDeclaration { Name = "binary-body", Role = DestinationRole.Queue }], cancellationToken); + + await transport.SendAsync("binary-body", + [new TransportMessage { Body = payload }], + new TransportSendOptions(), cancellationToken); + + var entries = await transport.ReceiveAsync("binary-body", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var entry = Assert.Single(entries); + Assert.Equal(payload, entry.Body.ToArray()); + await transport.CompleteAsync(entry, cancellationToken); + } +} diff --git a/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj b/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj new file mode 100644 index 000000000..8fe4f59ce --- /dev/null +++ b/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/Foundatio.Aws.Tests/README.md b/tests/Foundatio.Aws.Tests/README.md new file mode 100644 index 000000000..8828961cb --- /dev/null +++ b/tests/Foundatio.Aws.Tests/README.md @@ -0,0 +1,33 @@ +# Foundatio.Aws.Tests + +Runs the shared transport conformance suite (`MessageTransportConformanceTests`) against the AWS SQS/SNS +`IMessageTransport` (`Foundatio.Aws`). This is a temporary in-repo provider used to validate the redesigned transport +contract against a real broker before it is extracted to its own package. + +## Run against LocalStack + +```sh +# 1. Start LocalStack (SQS + SNS) +docker compose -f tests/Foundatio.Aws.Tests/docker-compose.yml up -d + +# 2. Point the tests at it +export FOUNDATIO_AWS_CONNECTION_STRING="serviceurl=http://localhost:4566;accesskey=test;secretkey=test;region=us-east-1" + +# 3. Run the conformance suite +dotnet test tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj +``` + +When `FOUNDATIO_AWS_CONNECTION_STRING` is **not** set, every test is skipped (so the project is safe in CI without a broker). + +To run against real AWS, set the connection string to real credentials/region (omit `serviceurl`), e.g. +`accesskey=...;secretkey=...;region=us-east-1`. + +## Capability coverage + +SQS/SNS supports pull receive, visibility timeout, lock renewal, redelivery delay (12h cap), delayed delivery (15-min +cap), provisioning, and stats. It does **not** support per-message priority, per-message TTL/expiration, push delivery, +or transport-native dead-lettering (the core owns retry/dead-lettering). Conformance tests for those capabilities skip +automatically via their `ISupports*` checks. + +Each run uses a unique `ResourcePrefix` so leftover messages from a prior run cannot leak in. LocalStack state is +ephemeral; restart the container to reset. diff --git a/tests/Foundatio.Aws.Tests/docker-compose.yml b/tests/Foundatio.Aws.Tests/docker-compose.yml new file mode 100644 index 000000000..053c8bc0d --- /dev/null +++ b/tests/Foundatio.Aws.Tests/docker-compose.yml @@ -0,0 +1,10 @@ +services: + localstack: + # Pinned to the 3.x community edition: SQS/SNS run free, with no auth token (the rolling `latest`/Pro tag now + # requires a LocalStack license). + image: localstack/localstack:3 + ports: + - "4566:4566" + environment: + - SERVICES=sqs,sns + - DEBUG=0 From fa7748307bb415cd767eee7545bc0cc62bf5a16f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 29 Jun 2026 17:05:23 -0500 Subject: [PATCH 17/57] Add Redis IJobRuntimeStore provider + cross-store conformance harness Validates the durable job runtime substrate against a real distributed store. RedisJobRuntimeStore implements all of IJobRuntimeStore using StackExchange.Redis transactions with hash-field conditions for optimistic concurrency (CAS), and a single Lua script for batch due-dispatch claiming. The reclaim path predicates on the exact observed lease value so a concurrent renew defeats a stale reclaim. Extracts a shared JobRuntimeStoreConformanceTests suite (in TestHarness) that both the in-memory reference and Redis run against the same invariants: state round-trips, optimistic transitions (status + node guards + patch application), leases/claims/steal-after-expiry, stale recovery excluding live leases and CRON occurrences (with the renew-during-reclaim race), scheduled-dispatch claim/complete/reschedule, and a contention test asserting exactly-one-winner for concurrent claims, transitions, and dispatch claiming. A FakeTimeProvider drives lease/expiry timing so the suite is fast and deterministic with no real sleeps. The Redis suite is gated on FOUNDATIO_REDIS_CONNECTION_STRING (skips when unset); each test isolates under a unique key prefix. Both suites: 6/6 green (Redis vs redis:7). Co-Authored-By: Claude Opus 4.8 --- Foundatio.slnx | 2 + src/Foundatio.Redis/Foundatio.Redis.csproj | 11 + src/Foundatio.Redis/RedisJobRuntimeStore.cs | 486 ++++++++++++++++++ .../RedisJobRuntimeStoreOptions.cs | 16 + .../Jobs/JobRuntimeStoreConformanceTests.cs | 372 ++++++++++++++ .../Foundatio.Redis.Tests.csproj | 6 + tests/Foundatio.Redis.Tests/README.md | 18 + .../RedisJobRuntimeStoreConformanceTests.cs | 55 ++ .../Foundatio.Redis.Tests/docker-compose.yml | 6 + .../Jobs/InMemoryJobRuntimeStoreTests.cs | 31 ++ 10 files changed, 1003 insertions(+) create mode 100644 src/Foundatio.Redis/Foundatio.Redis.csproj create mode 100644 src/Foundatio.Redis/RedisJobRuntimeStore.cs create mode 100644 src/Foundatio.Redis/RedisJobRuntimeStoreOptions.cs create mode 100644 src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs create mode 100644 tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj create mode 100644 tests/Foundatio.Redis.Tests/README.md create mode 100644 tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs create mode 100644 tests/Foundatio.Redis.Tests/docker-compose.yml create mode 100644 tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs diff --git a/Foundatio.slnx b/Foundatio.slnx index f674e0d53..7423951fd 100644 --- a/Foundatio.slnx +++ b/Foundatio.slnx @@ -15,10 +15,12 @@ + + diff --git a/src/Foundatio.Redis/Foundatio.Redis.csproj b/src/Foundatio.Redis/Foundatio.Redis.csproj new file mode 100644 index 000000000..f70dd8073 --- /dev/null +++ b/src/Foundatio.Redis/Foundatio.Redis.csproj @@ -0,0 +1,11 @@ + + + Redis-backed durable job runtime store (IJobRuntimeStore) for Foundatio. Temporary in-repo provider for validating the redesigned job runtime against a real distributed store. + + + + + + + + diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.cs new file mode 100644 index 000000000..8db55b072 --- /dev/null +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.cs @@ -0,0 +1,486 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using StackExchange.Redis; + +namespace Foundatio.Jobs; + +/// +/// A Redis-backed . Temporary in-repo provider used to validate the durable job runtime +/// (state transitions, leases/claims, scheduled dispatches) against a real distributed store. +/// +/// +/// Job state is a hash at {prefix}job:{id}; status and name indexes are sets; due dispatches are a sorted set +/// scored by due time. Conditional transitions use Redis transactions with hash-field conditions (optimistic +/// concurrency), so a state change only commits if the fields it was predicated on are unchanged — including a +/// lease-value condition that makes reclaim safe against a concurrent renew. Times are stored as UTC ticks for +/// unambiguous numeric comparison. +/// +public sealed class RedisJobRuntimeStore : IJobRuntimeStore +{ + private const string ClaimDueScript = """ + local ids = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, tonumber(ARGV[2])) + local claimed = {} + for _, id in ipairs(ids) do + local dkey = ARGV[5] .. id + if redis.call('EXISTS', dkey) == 1 then + local owner = redis.call('HGET', dkey, 'claimOwner') + local expires = redis.call('HGET', dkey, 'claimExpiresUtc') + if (not owner or owner == '') or (expires and expires ~= '' and tonumber(expires) <= tonumber(ARGV[1])) then + redis.call('HSET', dkey, 'claimOwner', ARGV[3], 'claimExpiresUtc', ARGV[4]) + redis.call('HINCRBY', dkey, 'attempts', 1) + table.insert(claimed, id) + end + end + end + return claimed + """; + + private readonly IDatabase _db; + private readonly string _prefix; + private readonly TimeProvider _timeProvider; + + public RedisJobRuntimeStore(RedisJobRuntimeStoreOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(options.ConnectionMultiplexer); + _db = options.ConnectionMultiplexer.GetDatabase(); + _prefix = options.KeyPrefix ?? ""; + _timeProvider = options.TimeProvider ?? TimeProvider.System; + } + + public RedisJobRuntimeStore(IConnectionMultiplexer connectionMultiplexer, string keyPrefix = "fnd:jobs:", TimeProvider? timeProvider = null) + : this(new RedisJobRuntimeStoreOptions { ConnectionMultiplexer = connectionMultiplexer, KeyPrefix = keyPrefix, TimeProvider = timeProvider }) { } + + public Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(initial); + cancellationToken.ThrowIfCancellationRequested(); + + var now = _timeProvider.GetUtcNow(); + var state = initial with + { + CreatedUtc = initial.CreatedUtc == default ? now : initial.CreatedUtc, + LastUpdatedUtc = initial.LastUpdatedUtc == default ? now : initial.LastUpdatedUtc + }; + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.KeyNotExists(JobKey(state.JobId))); + _ = tx.HashSetAsync(JobKey(state.JobId), ToHash(state)); + _ = tx.SetAddAsync(StatusKey(state.Status), state.JobId); + _ = tx.SetAddAsync(NameKey(state.Name), state.JobId); + _ = tx.SetAddAsync(AllKey, state.JobId); + return tx.ExecuteAsync(); // result ignored: false => already present; create-if-absent is a no-op + } + + public async Task GetAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var entries = await _db.HashGetAllAsync(JobKey(jobId)).ConfigureAwait(false); + return entries.Length == 0 ? null : FromHash(entries); + } + + public async Task> QueryAsync(JobQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + cancellationToken.ThrowIfCancellationRequested(); + + RedisValue[] ids; + if (query.Status is { } status && !String.IsNullOrEmpty(query.Name)) + ids = await _db.SetCombineAsync(SetOperation.Intersect, StatusKey(status), NameKey(query.Name)).ConfigureAwait(false); + else if (query.Status is { } onlyStatus) + ids = await _db.SetMembersAsync(StatusKey(onlyStatus)).ConfigureAwait(false); + else if (!String.IsNullOrEmpty(query.Name)) + ids = await _db.SetMembersAsync(NameKey(query.Name)).ConfigureAwait(false); + else + ids = await _db.SetMembersAsync(AllKey).ConfigureAwait(false); + + var states = await LoadAsync(ids).ConfigureAwait(false); + return states + .OrderByDescending(s => s.LastUpdatedUtc) + .Take(Math.Max(1, query.Limit)) + .ToArray(); + } + + public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, string? expectedNodeId = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "status", expectedStatus.ToString())); + if (expectedNodeId is not null) + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "nodeId", expectedNodeId)); + + ApplyTransition(tx, jobId, expectedStatus, newStatus, patch); + return tx.ExecuteAsync(); + } + + public async Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(nodeId); + + var now = _timeProvider.GetUtcNow(); + var current = await _db.HashGetAsync(JobKey(jobId), ["nodeId", "leaseExpiresUtc"]).ConfigureAwait(false); + if (!await _db.KeyExistsAsync(JobKey(jobId)).ConfigureAwait(false)) + return false; + + string? owner = ToStringOrNull(current[0]); + var leaseExpires = ParseTime(current[1]); + bool heldByOther = !String.IsNullOrEmpty(owner) && owner != nodeId && leaseExpires is { } e && e > now; + if (heldByOther) + return false; + + var tx = _db.CreateTransaction(); + // Predicate on the owner we observed so a competing claim that lands first invalidates this one. + tx.AddCondition(String.IsNullOrEmpty(owner) ? Condition.HashNotExists(JobKey(jobId), "nodeId") : Condition.HashEqual(JobKey(jobId), "nodeId", owner)); + _ = tx.HashSetAsync(JobKey(jobId), + [ + new HashEntry("nodeId", nodeId), + new HashEntry("leaseExpiresUtc", Ticks(now.Add(lease))), + new HashEntry("lastUpdatedUtc", Ticks(now)) + ]); + return await tx.ExecuteAsync().ConfigureAwait(false); + } + + public Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var now = _timeProvider.GetUtcNow(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "nodeId", nodeId)); + _ = tx.HashSetAsync(JobKey(jobId), + [ + new HashEntry("leaseExpiresUtc", Ticks(now.Add(lease))), + new HashEntry("lastUpdatedUtc", Ticks(now)) + ]); + return tx.ExecuteAsync(); + } + + public Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "nodeId", nodeId)); + _ = tx.HashDeleteAsync(JobKey(jobId), ["nodeId", "leaseExpiresUtc"]); + _ = tx.HashSetAsync(JobKey(jobId), "lastUpdatedUtc", Ticks(_timeProvider.GetUtcNow())); + return tx.ExecuteAsync(); + } + + public async Task> GetExpiredProcessingAsync(DateTimeOffset now, int limit, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var ids = await _db.SetMembersAsync(StatusKey(JobStatus.Processing)).ConfigureAwait(false); + var states = await LoadAsync(ids).ConfigureAwait(false); + return states + // Exclude CRON occurrences (ScheduledForUtc set): the scheduler owns their recovery. + .Where(s => s.ScheduledForUtc is null && s.LeaseExpiresUtc is { } lease && lease <= now) + .OrderBy(s => s.LeaseExpiresUtc) + .Take(Math.Max(1, limit)) + .ToArray(); + } + + public async Task TryReclaimExpiredAsync(string jobId, DateTimeOffset now, string expectedNodeId, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(expectedNodeId); + + var state = await GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (state is null || state.Status != JobStatus.Processing || !String.Equals(state.NodeId, expectedNodeId, StringComparison.Ordinal)) + return false; + if (state.LeaseExpiresUtc is not { } lease || lease > now) + return false; + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "status", JobStatus.Processing.ToString())); + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "nodeId", expectedNodeId)); + // Predicate on the exact lease we read; a concurrent renew changes it and invalidates the reclaim. + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "leaseExpiresUtc", Ticks(lease))); + + ApplyTransition(tx, jobId, JobStatus.Processing, newStatus, patch); + return await tx.ExecuteAsync().ConfigureAwait(false); + } + + public Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.KeyExists(JobKey(jobId))); + if (percent is { } p) + _ = tx.HashSetAsync(JobKey(jobId), "progress", p); + if (message is not null) + _ = tx.HashSetAsync(JobKey(jobId), "progressMessage", message); + _ = tx.HashSetAsync(JobKey(jobId), "lastUpdatedUtc", Ticks(_timeProvider.GetUtcNow())); + return tx.ExecuteAsync(); + } + + public Task IncrementAttemptAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.KeyExists(JobKey(jobId))); + _ = tx.HashIncrementAsync(JobKey(jobId), "attempt", 1); + _ = tx.HashSetAsync(JobKey(jobId), "lastUpdatedUtc", Ticks(_timeProvider.GetUtcNow())); + return tx.ExecuteAsync(); + } + + public Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.KeyExists(JobKey(jobId))); + _ = tx.HashSetAsync(JobKey(jobId), + [ + new HashEntry("cancellationRequested", "1"), + new HashEntry("lastUpdatedUtc", Ticks(_timeProvider.GetUtcNow())) + ]); + return tx.ExecuteAsync(); + } + + public async Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + var value = await _db.HashGetAsync(JobKey(jobId), "cancellationRequested").ConfigureAwait(false); + return value == "1"; + } + + public Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dispatch); + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.KeyNotExists(DispatchKey(dispatch.DispatchId))); + _ = tx.HashSetAsync(DispatchKey(dispatch.DispatchId), ToHash(dispatch)); + _ = tx.SortedSetAddAsync(DueKey, dispatch.DispatchId, dispatch.DueUtc.UtcTicks); + return tx.ExecuteAsync(); // result ignored: false => already scheduled; no-op + } + + public async Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrEmpty(nodeId); + + var result = await _db.ScriptEvaluateAsync(ClaimDueScript, + [DueKey], + [now.UtcTicks, Math.Max(1, limit), nodeId, Ticks(now.Add(lease)), $"{_prefix}dispatch:"]).ConfigureAwait(false); + + var ids = (RedisValue[]?)result ?? []; + var dispatches = new List(ids.Length); + foreach (var id in ids) + { + var entries = await _db.HashGetAllAsync(DispatchKey(id!)).ConfigureAwait(false); + if (entries.Length > 0) + dispatches.Add(DispatchFromHash(entries)); + } + + return dispatches; + } + + public Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(DispatchKey(dispatchId), "claimOwner", nodeId)); + _ = tx.KeyDeleteAsync(DispatchKey(dispatchId)); + _ = tx.SortedSetRemoveAsync(DueKey, dispatchId); + return tx.ExecuteAsync(); + } + + public Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var tx = _db.CreateTransaction(); + tx.AddCondition(Condition.HashEqual(DispatchKey(dispatchId), "claimOwner", nodeId)); + _ = tx.HashDeleteAsync(DispatchKey(dispatchId), ["claimOwner", "claimExpiresUtc"]); + _ = tx.HashSetAsync(DispatchKey(dispatchId), "dueUtc", Ticks(nextDueUtc)); + _ = tx.SortedSetAddAsync(DueKey, dispatchId, nextDueUtc.UtcTicks); + return tx.ExecuteAsync(); + } + + private void ApplyTransition(ITransaction tx, string jobId, JobStatus fromStatus, JobStatus toStatus, JobStatePatch? patch) + { + var sets = new List + { + new("status", toStatus.ToString()), + new("lastUpdatedUtc", Ticks(patch?.LastUpdatedUtc ?? _timeProvider.GetUtcNow())) + }; + var deletes = new List(); + + if (patch is not null) + { + if (patch.JobType is not null) sets.Add(new("jobType", patch.JobType)); + if (patch.Progress is { } progress) sets.Add(new("progress", progress)); + if (patch.ProgressMessage is not null) sets.Add(new("progressMessage", patch.ProgressMessage)); + if (patch.Error is not null) sets.Add(new("error", patch.Error)); + if (patch.StartedUtc is { } started) sets.Add(new("startedUtc", Ticks(started))); + if (patch.CompletedUtc is { } completed) sets.Add(new("completedUtc", Ticks(completed))); + if (patch.CancellationRequested is { } cancel) sets.Add(new("cancellationRequested", cancel ? "1" : "0")); + + if (patch.ClearNodeId) deletes.Add("nodeId"); + else if (patch.NodeId is not null) sets.Add(new("nodeId", patch.NodeId)); + + if (patch.ClearLeaseExpiresUtc) deletes.Add("leaseExpiresUtc"); + else if (patch.LeaseExpiresUtc is { } leaseExpires) sets.Add(new("leaseExpiresUtc", Ticks(leaseExpires))); + + if (patch.AttemptDelta != 0) + _ = tx.HashIncrementAsync(JobKey(jobId), "attempt", patch.AttemptDelta); + } + + _ = tx.HashSetAsync(JobKey(jobId), sets.ToArray()); + if (deletes.Count > 0) + _ = tx.HashDeleteAsync(JobKey(jobId), deletes.ToArray()); + + if (fromStatus != toStatus) + { + _ = tx.SetRemoveAsync(StatusKey(fromStatus), jobId); + _ = tx.SetAddAsync(StatusKey(toStatus), jobId); + } + } + + private async Task> LoadAsync(RedisValue[] ids) + { + var states = new List(ids.Length); + foreach (var id in ids) + { + var entries = await _db.HashGetAllAsync(JobKey(id!)).ConfigureAwait(false); + if (entries.Length > 0) + states.Add(FromHash(entries)); + } + + return states; + } + + private RedisKey JobKey(string id) => $"{_prefix}job:{id}"; + private RedisKey StatusKey(JobStatus status) => $"{_prefix}status:{status}"; + private RedisKey NameKey(string name) => $"{_prefix}name:{name}"; + private RedisKey DispatchKey(string id) => $"{_prefix}dispatch:{id}"; + private RedisKey AllKey => $"{_prefix}all"; + private RedisKey DueKey => $"{_prefix}dispatches:due"; + + private static string Ticks(DateTimeOffset value) => value.UtcTicks.ToString(CultureInfo.InvariantCulture); + + private static DateTimeOffset? ParseTime(RedisValue value) + { + return value.IsNullOrEmpty || !Int64.TryParse((string?)value, NumberStyles.Integer, CultureInfo.InvariantCulture, out long ticks) + ? null + : new DateTimeOffset(ticks, TimeSpan.Zero); + } + + private static string? ToStringOrNull(RedisValue value) => value.IsNullOrEmpty ? null : (string)value!; + + private static HashEntry[] ToHash(JobState state) + { + var entries = new List + { + new("jobId", state.JobId), + new("name", state.Name), + new("status", state.Status.ToString()), + new("attempt", state.Attempt), + new("cancellationRequested", state.CancellationRequested ? "1" : "0"), + new("createdUtc", Ticks(state.CreatedUtc)), + new("lastUpdatedUtc", Ticks(state.LastUpdatedUtc)) + }; + + if (state.JobType is not null) entries.Add(new("jobType", state.JobType)); + if (state.Progress is { } progress) entries.Add(new("progress", progress)); + if (state.ProgressMessage is not null) entries.Add(new("progressMessage", state.ProgressMessage)); + if (state.NodeId is not null) entries.Add(new("nodeId", state.NodeId)); + if (state.StartedUtc is { } started) entries.Add(new("startedUtc", Ticks(started))); + if (state.CompletedUtc is { } completed) entries.Add(new("completedUtc", Ticks(completed))); + if (state.LeaseExpiresUtc is { } leaseExpires) entries.Add(new("leaseExpiresUtc", Ticks(leaseExpires))); + if (state.Error is not null) entries.Add(new("error", state.Error)); + if (state.ScheduledForUtc is { } scheduledFor) entries.Add(new("scheduledForUtc", Ticks(scheduledFor))); + + return entries.ToArray(); + } + + private static JobState FromHash(HashEntry[] entries) + { + var map = entries.ToDictionary(e => (string)e.Name!, e => e.Value); + RedisValue Get(string field) => map.TryGetValue(field, out var value) ? value : RedisValue.Null; + + return new JobState + { + JobId = (string)Get("jobId")!, + Name = (string)Get("name")!, + JobType = ToStringOrNull(Get("jobType")), + Status = Enum.Parse((string)Get("status")!), + Progress = Get("progress").IsNullOrEmpty ? null : (int)Get("progress"), + ProgressMessage = ToStringOrNull(Get("progressMessage")), + Attempt = Get("attempt").IsNullOrEmpty ? 0 : (int)Get("attempt"), + NodeId = ToStringOrNull(Get("nodeId")), + CreatedUtc = ParseTime(Get("createdUtc")) ?? default, + LastUpdatedUtc = ParseTime(Get("lastUpdatedUtc")) ?? default, + StartedUtc = ParseTime(Get("startedUtc")), + CompletedUtc = ParseTime(Get("completedUtc")), + LeaseExpiresUtc = ParseTime(Get("leaseExpiresUtc")), + Error = ToStringOrNull(Get("error")), + CancellationRequested = Get("cancellationRequested") == "1", + ScheduledForUtc = ParseTime(Get("scheduledForUtc")) + }; + } + + private static HashEntry[] ToHash(ScheduledDispatchState dispatch) + { + var headers = new Dictionary(StringComparer.Ordinal); + foreach (var header in dispatch.Headers) + headers[header.Key] = header.Value; + + var entries = new List + { + new("dispatchId", dispatch.DispatchId), + new("kind", dispatch.Kind.ToString()), + new("destination", dispatch.Destination), + new("body", Convert.ToBase64String(dispatch.Body.Span)), + new("headers", JsonSerializer.Serialize(headers)), + new("options", JsonSerializer.Serialize(dispatch.Options)), + new("dueUtc", Ticks(dispatch.DueUtc)), + new("attempts", dispatch.Attempts) + }; + + if (dispatch.ClaimOwner is not null) entries.Add(new("claimOwner", dispatch.ClaimOwner)); + if (dispatch.ClaimExpiresUtc is { } claimExpires) entries.Add(new("claimExpiresUtc", Ticks(claimExpires))); + if (dispatch.JobId is not null) entries.Add(new("jobId", dispatch.JobId)); + + return entries.ToArray(); + } + + private static ScheduledDispatchState DispatchFromHash(HashEntry[] entries) + { + var map = entries.ToDictionary(e => (string)e.Name!, e => e.Value); + RedisValue Get(string field) => map.TryGetValue(field, out var value) ? value : RedisValue.Null; + + string headersJson = (string?)Get("headers") ?? "{}"; + var headerMap = JsonSerializer.Deserialize>(headersJson) ?? []; + var options = JsonSerializer.Deserialize((string?)Get("options") ?? "{}") ?? new TransportSendOptions(); + + return new ScheduledDispatchState + { + DispatchId = (string)Get("dispatchId")!, + Kind = Enum.Parse((string)Get("kind")!), + Destination = (string)Get("destination")!, + Body = Get("body").IsNullOrEmpty ? ReadOnlyMemory.Empty : Convert.FromBase64String((string)Get("body")!), + Headers = MessageHeaders.Create(headerMap), + Options = options, + DueUtc = ParseTime(Get("dueUtc")) ?? default, + ClaimOwner = ToStringOrNull(Get("claimOwner")), + ClaimExpiresUtc = ParseTime(Get("claimExpiresUtc")), + Attempts = Get("attempts").IsNullOrEmpty ? 0 : (int)Get("attempts"), + JobId = ToStringOrNull(Get("jobId")) + }; + } +} diff --git a/src/Foundatio.Redis/RedisJobRuntimeStoreOptions.cs b/src/Foundatio.Redis/RedisJobRuntimeStoreOptions.cs new file mode 100644 index 000000000..0675083a2 --- /dev/null +++ b/src/Foundatio.Redis/RedisJobRuntimeStoreOptions.cs @@ -0,0 +1,16 @@ +using System; +using StackExchange.Redis; + +namespace Foundatio.Jobs; + +public class RedisJobRuntimeStoreOptions +{ + /// The Redis connection to use. Required. + public IConnectionMultiplexer ConnectionMultiplexer { get; set; } = null!; + + /// Prefix applied to every key this store creates. Useful to isolate environments/runs on a shared Redis. + public string KeyPrefix { get; set; } = "fnd:jobs:"; + + /// Time source (defaults to ). + public TimeProvider? TimeProvider { get; set; } +} diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs new file mode 100644 index 000000000..22622028b --- /dev/null +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -0,0 +1,372 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Foundatio.Tests.Jobs; + +/// +/// Shared conformance suite every implementation must satisfy: state round-trips, +/// optimistic-concurrency transitions, leases/claims, stale recovery (including the renew-during-reclaim race), and +/// scheduled-dispatch claiming. The in-memory reference and any real store (Redis, etc.) run the same assertions so a +/// new backend is validated against the exact behavior the runtime depends on. +/// +/// +/// A drives time so lease-expiry and claim-steal paths are deterministic without real +/// sleeps. returns null when the backing store is unavailable (e.g. Redis not +/// configured), in which case every test skips. +/// +public abstract class JobRuntimeStoreConformanceTests : TestWithLoggingBase +{ + protected JobRuntimeStoreConformanceTests(ITestOutputHelper output) : base(output) { } + + /// Creates a fresh, isolated store bound to , or null when unavailable. + protected abstract IJobRuntimeStore? CreateStore(TimeProvider timeProvider); + + protected static JobState NewJob(TimeProvider time, string id, string name = "conformance-job", JobStatus status = JobStatus.Queued) + { + var now = time.GetUtcNow(); + return new JobState { JobId = id, Name = name, Status = status, CreatedUtc = now, LastUpdatedUtc = now }; + } + + public virtual async Task JobLifecycle_RoundTripsAndTransitionsAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + var created = time.GetUtcNow(); + + // Create with a rich, fully-populated state and assert every field survives the round-trip. + var job = NewJob(time, "job-1", "emailer") with + { + JobType = "Acme.EmailJob", + Progress = 10, + ProgressMessage = "starting", + Attempt = 1, + ScheduledForUtc = created.AddMinutes(1) + }; + await store.CreateIfAbsentAsync(job, ct); + + var got = await store.GetAsync("job-1", ct); + Assert.NotNull(got); + Assert.Equal("emailer", got.Name); + Assert.Equal("Acme.EmailJob", got.JobType); + Assert.Equal(JobStatus.Queued, got.Status); + Assert.Equal(10, got.Progress); + Assert.Equal("starting", got.ProgressMessage); + Assert.Equal(1, got.Attempt); + Assert.Equal(created, got.CreatedUtc); + Assert.Equal(created.AddMinutes(1), got.ScheduledForUtc); + + // Create-if-absent is a no-op once the row exists: a second create must not overwrite. + await store.CreateIfAbsentAsync(job with { Name = "overwritten" }, ct); + Assert.Equal("emailer", (await store.GetAsync("job-1", ct))!.Name); + + // A transition from the wrong current status must fail and leave state untouched. + Assert.False(await store.TryTransitionAsync("job-1", JobStatus.Processing, JobStatus.Completed, cancellationToken: ct)); + Assert.Equal(JobStatus.Queued, (await store.GetAsync("job-1", ct))!.Status); + + // Happy-path transition applies the patch atomically (status + node + lease + started + attempt delta). + var lease = time.GetUtcNow().AddMinutes(5); + Assert.True(await store.TryTransitionAsync("job-1", JobStatus.Queued, JobStatus.Processing, + new JobStatePatch { NodeId = "node-a", LeaseExpiresUtc = lease, StartedUtc = created, AttemptDelta = 1 }, cancellationToken: ct)); + got = await store.GetAsync("job-1", ct); + Assert.Equal(JobStatus.Processing, got!.Status); + Assert.Equal("node-a", got.NodeId); + Assert.Equal(lease, got.LeaseExpiresUtc); + Assert.Equal(2, got.Attempt); + Assert.Equal(created, got.StartedUtc); + + // expectedNodeId guards the transition: a stale worker (wrong node) cannot overwrite the owner's state. + Assert.False(await store.TryTransitionAsync("job-1", JobStatus.Processing, JobStatus.Completed, expectedNodeId: "node-b", cancellationToken: ct)); + Assert.Equal(JobStatus.Processing, (await store.GetAsync("job-1", ct))!.Status); + + // Correct owner completes and clears the lease/node. + var completedAt = time.GetUtcNow(); + Assert.True(await store.TryTransitionAsync("job-1", JobStatus.Processing, JobStatus.Completed, + new JobStatePatch { ClearNodeId = true, ClearLeaseExpiresUtc = true, CompletedUtc = completedAt }, expectedNodeId: "node-a", cancellationToken: ct)); + got = await store.GetAsync("job-1", ct); + Assert.Equal(JobStatus.Completed, got!.Status); + Assert.Null(got.NodeId); + Assert.Null(got.LeaseExpiresUtc); + Assert.Equal(completedAt, got.CompletedUtc); + + // Progress, attempt, and cancellation are independent of transitions. + await store.CreateIfAbsentAsync(NewJob(time, "job-2", "worker"), ct); + await store.SetProgressAsync("job-2", 55, "halfway", ct); + await store.IncrementAttemptAsync("job-2", ct); + got = await store.GetAsync("job-2", ct); + Assert.Equal(55, got!.Progress); + Assert.Equal("halfway", got.ProgressMessage); + Assert.Equal(1, got.Attempt); + + Assert.False(await store.IsCancellationRequestedAsync("job-2", ct)); + Assert.True(await store.RequestCancellationAsync("job-2", ct)); + Assert.True(await store.IsCancellationRequestedAsync("job-2", ct)); + + // Operating on a missing job is a benign no-op (returns false / does not throw). + Assert.False(await store.RequestCancellationAsync("missing", ct)); + Assert.Null(await store.GetAsync("missing", ct)); + } + + public virtual async Task Query_FiltersByNameStatusAndLimitAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + var t = time.GetUtcNow(); + + // Distinct LastUpdatedUtc values make the default newest-first ordering (and limit) deterministic. + await store.CreateIfAbsentAsync(NewJob(time, "a", "alpha", JobStatus.Queued) with { LastUpdatedUtc = t }, ct); + await store.CreateIfAbsentAsync(NewJob(time, "b", "alpha", JobStatus.Processing) with { LastUpdatedUtc = t.AddSeconds(1) }, ct); + await store.CreateIfAbsentAsync(NewJob(time, "c", "beta", JobStatus.Queued) with { LastUpdatedUtc = t.AddSeconds(2) }, ct); + + var byName = await store.QueryAsync(new JobQuery { Name = "alpha" }, ct); + Assert.Equal(["b", "a"], byName.Select(j => j.JobId)); + + var byStatus = await store.QueryAsync(new JobQuery { Status = JobStatus.Queued }, ct); + Assert.Equal(new HashSet { "a", "c" }, byStatus.Select(j => j.JobId).ToHashSet()); + + var byBoth = await store.QueryAsync(new JobQuery { Name = "alpha", Status = JobStatus.Queued }, ct); + Assert.Equal("a", Assert.Single(byBoth).JobId); + + var all = await store.QueryAsync(new JobQuery(), ct); + Assert.Equal(new HashSet { "a", "b", "c" }, all.Select(j => j.JobId).ToHashSet()); + + // Limit is honored against the newest-first ordering, so the most recently updated row wins. + var limited = await store.QueryAsync(new JobQuery { Limit = 1 }, ct); + Assert.Equal("c", Assert.Single(limited).JobId); + } + + public virtual async Task Leasing_ClaimRenewReleaseAndStealAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + await store.CreateIfAbsentAsync(NewJob(time, "job-1"), ct); + + var claimedAt = time.GetUtcNow(); + Assert.True(await store.TryClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(5), ct)); + var got = await store.GetAsync("job-1", ct); + Assert.Equal("node-a", got!.NodeId); + Assert.Equal(claimedAt.AddMinutes(5), got.LeaseExpiresUtc); + + // The current owner can re-claim/renew; a different node cannot while the lease is live. + Assert.True(await store.TryClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(5), ct)); + Assert.False(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(5), ct)); + + // RenewClaim is owner-scoped. + Assert.False(await store.RenewClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(10), ct)); + Assert.True(await store.RenewClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(10), ct)); + Assert.Equal(time.GetUtcNow().AddMinutes(10), (await store.GetAsync("job-1", ct))!.LeaseExpiresUtc); + + // Once the lease lapses, another node may steal the claim. + time.Advance(TimeSpan.FromMinutes(11)); + Assert.True(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(5), ct)); + Assert.Equal("node-b", (await store.GetAsync("job-1", ct))!.NodeId); + + // Release is owner-scoped and clears the lease. + Assert.False(await store.ReleaseClaimAsync("job-1", "node-a", ct)); + Assert.True(await store.ReleaseClaimAsync("job-1", "node-b", ct)); + got = await store.GetAsync("job-1", ct); + Assert.Null(got!.NodeId); + Assert.Null(got.LeaseExpiresUtc); + } + + public virtual async Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + var now = time.GetUtcNow(); + + JobState Processing(string id, DateTimeOffset lease, string node = "node-a", DateTimeOffset? scheduledFor = null) => + NewJob(time, id, "worker", JobStatus.Processing) with { NodeId = node, LeaseExpiresUtc = lease, ScheduledForUtc = scheduledFor }; + + await store.CreateIfAbsentAsync(Processing("plain", now.AddMinutes(-1)), ct); + await store.CreateIfAbsentAsync(Processing("cron", now.AddMinutes(-1), scheduledFor: now), ct); + await store.CreateIfAbsentAsync(Processing("live", now.AddMinutes(10)), ct); + + // Only the plain expired job is recoverable: the live lease and the CRON occurrence are excluded. + var expired = await store.GetExpiredProcessingAsync(now, 100, ct); + Assert.Equal("plain", Assert.Single(expired).JobId); + + // Reclaim re-queues it (still owned by node-a, lease still expired). + Assert.True(await store.TryReclaimExpiredAsync("plain", now, "node-a", JobStatus.Queued, + new JobStatePatch { ClearNodeId = true, ClearLeaseExpiresUtc = true, AttemptDelta = 1 }, ct)); + var got = await store.GetAsync("plain", ct); + Assert.Equal(JobStatus.Queued, got!.Status); + Assert.Null(got.NodeId); + Assert.Equal(1, got.Attempt); + + // Renew-during-reclaim race: a job whose owner renewed since the scan must NOT be reclaimed (lease no longer expired). + await store.CreateIfAbsentAsync(Processing("renewed", now.AddMinutes(-1)), ct); + Assert.True(await store.RenewClaimAsync("renewed", "node-a", TimeSpan.FromMinutes(10), ct)); + Assert.False(await store.TryReclaimExpiredAsync("renewed", now, "node-a", JobStatus.Queued, cancellationToken: ct)); + Assert.Equal(JobStatus.Processing, (await store.GetAsync("renewed", ct))!.Status); + + // Owner mismatch since the scan also blocks the reclaim. + await store.CreateIfAbsentAsync(Processing("reowned", now.AddMinutes(-1), node: "node-b"), ct); + Assert.False(await store.TryReclaimExpiredAsync("reowned", now, "node-a", JobStatus.Queued, cancellationToken: ct)); + Assert.Equal("node-b", (await store.GetAsync("reowned", ct))!.NodeId); + } + + public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + var t = time.GetUtcNow(); + + var headers = MessageHeaders.Create(new Dictionary { ["message.type"] = "order.created", ["tenant"] = "acme" }); + var options = new TransportSendOptions { DestinationRole = DestinationRole.Topic, Priority = MessagePriority.High }; + byte[] body = [0x01, 0x02, 0xFF, 0x00, 0x10]; + + var due = new ScheduledDispatchState + { + DispatchId = "d1", + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = "jobs", + Body = body, + Headers = headers, + Options = options, + DueUtc = t.AddMinutes(-1), + JobId = "job-x" + }; + var future = new ScheduledDispatchState + { + DispatchId = "d2", + Kind = ScheduledDispatchKind.QueueMessage, + Destination = "later", + Body = body, + DueUtc = t.AddHours(1) + }; + await store.ScheduleDispatchAsync(due, ct); + await store.ScheduleDispatchAsync(future, ct); + // Re-scheduling the same id is a no-op (must not overwrite the destination). + await store.ScheduleDispatchAsync(due with { Destination = "overwritten" }, ct); + + // Only the due dispatch is claimed; the full payload round-trips and the attempt counter increments. + var claimed = await store.ClaimDueDispatchesAsync(t, 100, "node-a", TimeSpan.FromMinutes(5), ct); + var d = Assert.Single(claimed); + Assert.Equal("d1", d.DispatchId); + Assert.Equal(ScheduledDispatchKind.JobOccurrence, d.Kind); + Assert.Equal("jobs", d.Destination); + Assert.Equal(body, d.Body.ToArray()); + Assert.Equal("acme", d.Headers["tenant"]); + Assert.Equal("order.created", d.Headers["message.type"]); + Assert.Equal(DestinationRole.Topic, d.Options.DestinationRole); + Assert.Equal(MessagePriority.High, d.Options.Priority); + Assert.Equal("node-a", d.ClaimOwner); + Assert.Equal(1, d.Attempts); + Assert.Equal("job-x", d.JobId); + + // A competing claim sees nothing while the lease is live (and d2 is not yet due). + Assert.Empty(await store.ClaimDueDispatchesAsync(t, 100, "node-b", TimeSpan.FromMinutes(5), ct)); + + // A complete from the wrong owner is ignored: after the lease lapses the dispatch is re-claimable, attempt 2. + await store.CompleteDispatchAsync("d1", "node-b", ct); + var reclaimed = await store.ClaimDueDispatchesAsync(t.AddMinutes(6), 100, "node-a", TimeSpan.FromMinutes(5), ct); + Assert.Equal(2, Assert.Single(reclaimed).Attempts); + + // The owning node completes it for good. + await store.CompleteDispatchAsync("d1", "node-a", ct); + Assert.Empty(await store.ClaimDueDispatchesAsync(t.AddMinutes(12), 100, "node-a", TimeSpan.FromMinutes(5), ct)); + + // Release reschedules a claimed dispatch to its next due time and clears ownership (recurring-occurrence path). + var recurring = new ScheduledDispatchState + { + DispatchId = "d3", + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = "cron", + Body = body, + DueUtc = t.AddMinutes(20) + }; + await store.ScheduleDispatchAsync(recurring, ct); + Assert.Equal("d3", Assert.Single(await store.ClaimDueDispatchesAsync(t.AddMinutes(21), 100, "node-a", TimeSpan.FromMinutes(5), ct)).DispatchId); + + await store.ReleaseDispatchAsync("d3", "node-b", t.AddMinutes(50), ct); // wrong owner: ignored + await store.ReleaseDispatchAsync("d3", "node-a", t.AddMinutes(50), ct); + Assert.Empty(await store.ClaimDueDispatchesAsync(t.AddMinutes(40), 100, "node-c", TimeSpan.FromMinutes(5), ct)); + var rescheduled = Assert.Single(await store.ClaimDueDispatchesAsync(t.AddMinutes(51), 100, "node-c", TimeSpan.FromMinutes(5), ct)); + Assert.Equal("d3", rescheduled.DispatchId); + Assert.Equal("node-c", rescheduled.ClaimOwner); + } + + public virtual async Task Concurrency_OptimisticControlElectsSingleWinnerAsync() + { + var time = new FakeTimeProvider(); + var store = CreateStore(time); + if (store is null) + { + Assert.Skip("Job runtime store not configured."); + return; + } + + var ct = TestCancellationToken; + const int contenders = 25; + + // Many nodes race to claim the same unclaimed job: exactly one may win, and the store must agree on the owner. + await store.CreateIfAbsentAsync(NewJob(time, "claim-race"), ct); + var claims = await Task.WhenAll(Enumerable.Range(0, contenders) + .Select(i => Task.Run(() => store.TryClaimAsync("claim-race", $"node-{i}", TimeSpan.FromMinutes(5), ct), ct))); + Assert.Equal(1, claims.Count(won => won)); + var ownedBy = (await store.GetAsync("claim-race", ct))!.NodeId; + Assert.StartsWith("node-", ownedBy); + + // Many nodes race the same Queued -> Processing transition: optimistic concurrency must admit exactly one. + await store.CreateIfAbsentAsync(NewJob(time, "transition-race"), ct); + var transitions = await Task.WhenAll(Enumerable.Range(0, contenders) + .Select(i => Task.Run(() => store.TryTransitionAsync("transition-race", JobStatus.Queued, JobStatus.Processing, + new JobStatePatch { NodeId = $"node-{i}" }, cancellationToken: ct), ct))); + Assert.Equal(1, transitions.Count(won => won)); + Assert.Equal(JobStatus.Processing, (await store.GetAsync("transition-race", ct))!.Status); + + // A single due dispatch contested by many claimers must be handed to exactly one. + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = "dispatch-race", + Destination = "q", + Body = new byte[] { 1 }, + DueUtc = time.GetUtcNow().AddMinutes(-1) + }, ct); + var dispatchClaims = await Task.WhenAll(Enumerable.Range(0, contenders) + .Select(i => Task.Run(() => store.ClaimDueDispatchesAsync(time.GetUtcNow(), 100, $"node-{i}", TimeSpan.FromMinutes(5), ct), ct))); + Assert.Equal(1, dispatchClaims.Sum(claimed => claimed.Count(d => d.DispatchId == "dispatch-race"))); + } +} diff --git a/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj b/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj new file mode 100644 index 000000000..33f37fc01 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/Foundatio.Redis.Tests/README.md b/tests/Foundatio.Redis.Tests/README.md new file mode 100644 index 000000000..df67fa68f --- /dev/null +++ b/tests/Foundatio.Redis.Tests/README.md @@ -0,0 +1,18 @@ +# Foundatio.Redis.Tests + +Validates the temporary in-repo `RedisJobRuntimeStore` against a real Redis by running the shared +`JobRuntimeStoreConformanceTests` suite (the same assertions the in-memory reference store passes). + +## Running + +Start Redis and point the tests at it. Without the connection string every test is skipped. + +```sh +docker compose -f tests/Foundatio.Redis.Tests/docker-compose.yml up -d + +export FOUNDATIO_REDIS_CONNECTION_STRING=localhost:6399 +dotnet run --project tests/Foundatio.Redis.Tests +``` + +Each test runs under a unique key prefix (`fnd-conf:{guid}:`), so concurrent runs and leftover keys never collide. +A `FakeTimeProvider` drives lease/expiry timing, so the suite is fast and deterministic — no real sleeps. diff --git a/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs new file mode 100644 index 000000000..caf570196 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Tests.Jobs; +using StackExchange.Redis; +using Xunit; + +namespace Foundatio.Redis.Tests; + +/// +/// Runs the shared conformance suite against a real Redis. Set +/// FOUNDATIO_REDIS_CONNECTION_STRING (e.g. localhost:6399 for the bundled docker-compose Redis) to run; +/// when it is not set every test is skipped. Each test gets a unique key prefix so runs never collide. +/// +public class RedisJobRuntimeStoreConformanceTests : JobRuntimeStoreConformanceTests +{ + private static readonly Lazy SharedConnection = new(() => + { + string? connectionString = Environment.GetEnvironmentVariable("FOUNDATIO_REDIS_CONNECTION_STRING"); + return String.IsNullOrEmpty(connectionString) ? null : ConnectionMultiplexer.Connect(connectionString); + }); + + public RedisJobRuntimeStoreConformanceTests(ITestOutputHelper output) : base(output) { } + + protected override IJobRuntimeStore? CreateStore(TimeProvider timeProvider) + { + if (SharedConnection.Value is not { } connection) + return null; // not configured -> the base suite skips every test + + return new RedisJobRuntimeStore(new RedisJobRuntimeStoreOptions + { + ConnectionMultiplexer = connection, + KeyPrefix = $"fnd-conf:{Guid.NewGuid():N}:", + TimeProvider = timeProvider + }); + } + + [Fact] + public override Task JobLifecycle_RoundTripsAndTransitionsAsync() => base.JobLifecycle_RoundTripsAndTransitionsAsync(); + + [Fact] + public override Task Query_FiltersByNameStatusAndLimitAsync() => base.Query_FiltersByNameStatusAndLimitAsync(); + + [Fact] + public override Task Leasing_ClaimRenewReleaseAndStealAsync() => base.Leasing_ClaimRenewReleaseAndStealAsync(); + + [Fact] + public override Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() => base.StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync(); + + [Fact] + public override Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() => base.ScheduledDispatches_ClaimCompleteAndRescheduleAsync(); + + [Fact] + public override Task Concurrency_OptimisticControlElectsSingleWinnerAsync() => base.Concurrency_OptimisticControlElectsSingleWinnerAsync(); +} diff --git a/tests/Foundatio.Redis.Tests/docker-compose.yml b/tests/Foundatio.Redis.Tests/docker-compose.yml new file mode 100644 index 000000000..d41de3cdd --- /dev/null +++ b/tests/Foundatio.Redis.Tests/docker-compose.yml @@ -0,0 +1,6 @@ +services: + redis: + image: redis:7-alpine + # Mapped to 6399 to avoid clashing with a local Redis on the default 6379. + ports: + - "6399:6379" diff --git a/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs b/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs new file mode 100644 index 000000000..7a596c9bf --- /dev/null +++ b/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Xunit; + +namespace Foundatio.Tests.Jobs; + +public class InMemoryJobRuntimeStoreTests : JobRuntimeStoreConformanceTests +{ + public InMemoryJobRuntimeStoreTests(ITestOutputHelper output) : base(output) { } + + protected override IJobRuntimeStore CreateStore(TimeProvider timeProvider) => new InMemoryJobRuntimeStore(timeProvider); + + [Fact] + public override Task JobLifecycle_RoundTripsAndTransitionsAsync() => base.JobLifecycle_RoundTripsAndTransitionsAsync(); + + [Fact] + public override Task Query_FiltersByNameStatusAndLimitAsync() => base.Query_FiltersByNameStatusAndLimitAsync(); + + [Fact] + public override Task Leasing_ClaimRenewReleaseAndStealAsync() => base.Leasing_ClaimRenewReleaseAndStealAsync(); + + [Fact] + public override Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() => base.StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync(); + + [Fact] + public override Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() => base.ScheduledDispatches_ClaimCompleteAndRescheduleAsync(); + + [Fact] + public override Task Concurrency_OptimisticControlElectsSingleWinnerAsync() => base.Concurrency_OptimisticControlElectsSingleWinnerAsync(); +} From 4911b28ef689266a69ab999358d266fc1b848625 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 29 Jun 2026 17:21:40 -0500 Subject: [PATCH 18/57] Add Redis end-to-end integration tests: delayed-send fallback + CRON The conformance suite covers ScheduleDispatch/ClaimDueDispatches as primitives; these tests wire the real messaging core and CRON scheduler on top of the Redis IJobRuntimeStore and exercise the two paths the store exists to support: 1. A queue send whose delay exceeds the transport's MaxDeliveryDelay is routed into Redis (not truncated to the broker ceiling), stays time-gated (a drain before the due time claims nothing), and is pulled from Redis and handed to the transport once due. A within-cap delay still goes native and never touches the store. 2. CRON occurrences are materialized into Redis (Scheduled JobState + JobOccurrence dispatch), deduped by deterministic occurrence id, claimed and run to completion, retried-then-dead-lettered when they keep failing, and stale-reclaimed (via the Redis CAS reclaim) when an occurrence is stuck Processing under a dead node with an expired lease. Extracts a shared RedisTestConnection helper (gated on FOUNDATIO_REDIS_CONNECTION_STRING, unique key prefix per store) used by both the conformance and integration suites. Redis suite: 9/9 green (6 conformance + 3 integration) against redis:7; in-memory unaffected. Co-Authored-By: Claude Opus 4.8 --- .../RedisJobRuntimeStoreConformanceTests.cs | 23 +- .../RedisJobStoreIntegrationTests.cs | 271 ++++++++++++++++++ .../RedisTestConnection.cs | 30 ++ 3 files changed, 305 insertions(+), 19 deletions(-) create mode 100644 tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs create mode 100644 tests/Foundatio.Redis.Tests/RedisTestConnection.cs diff --git a/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs index caf570196..956c0c50f 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs @@ -2,7 +2,6 @@ using System.Threading.Tasks; using Foundatio.Jobs; using Foundatio.Tests.Jobs; -using StackExchange.Redis; using Xunit; namespace Foundatio.Redis.Tests; @@ -14,26 +13,12 @@ namespace Foundatio.Redis.Tests; /// public class RedisJobRuntimeStoreConformanceTests : JobRuntimeStoreConformanceTests { - private static readonly Lazy SharedConnection = new(() => - { - string? connectionString = Environment.GetEnvironmentVariable("FOUNDATIO_REDIS_CONNECTION_STRING"); - return String.IsNullOrEmpty(connectionString) ? null : ConnectionMultiplexer.Connect(connectionString); - }); - public RedisJobRuntimeStoreConformanceTests(ITestOutputHelper output) : base(output) { } - protected override IJobRuntimeStore? CreateStore(TimeProvider timeProvider) - { - if (SharedConnection.Value is not { } connection) - return null; // not configured -> the base suite skips every test - - return new RedisJobRuntimeStore(new RedisJobRuntimeStoreOptions - { - ConnectionMultiplexer = connection, - KeyPrefix = $"fnd-conf:{Guid.NewGuid():N}:", - TimeProvider = timeProvider - }); - } + protected override IJobRuntimeStore? CreateStore(TimeProvider timeProvider) => + RedisTestConnection.Multiplexer is { } connection + ? RedisTestConnection.CreateStore(connection, timeProvider) + : null; // not configured -> the base suite skips every test [Fact] public override Task JobLifecycle_RoundTripsAndTransitionsAsync() => base.JobLifecycle_RoundTripsAndTransitionsAsync(); diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs new file mode 100644 index 000000000..e02396bf3 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Foundatio.Redis.Tests; + +/// +/// End-to-end tests that wire the real messaging core / CRON scheduler on top of the Redis +/// and exercise the two paths the store exists to support but that the primitive-level conformance suite does not cover: +/// (1) a delayed send whose delay exceeds the transport's being +/// durably stored in Redis and drained by the dispatch pump when due, and (2) CRON occurrences being materialized, run, +/// retried/dead-lettered, and stale-reclaimed through Redis. +/// +/// Gated on FOUNDATIO_REDIS_CONNECTION_STRING; skips when unset. Each test isolates under a unique key prefix. +/// +public class RedisJobStoreIntegrationTests +{ + [Fact] + public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWhenDueAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var cancellationToken = TestContext.Current.CancellationToken; + var now = DateTimeOffset.UtcNow; + + // Within the transport's advertised maximum: delivered natively, nothing is parked in Redis. + var nativeStore = RedisTestConnection.CreateStore(connection); + await using var nativeTransport = new CappedDelayTransport(TimeSpan.FromMinutes(15)); + await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); + var nativeProcessor = CreateProcessor(nativeStore, nativeTransport).Processor; + + await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + Assert.Equal(1, nativeTransport.SendCount); + Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); + Assert.Equal(0, await nativeProcessor.RunDueOccurrencesAsync(now.AddYears(1), cancellationToken: cancellationToken)); + + // Beyond the transport's maximum: routed into the Redis store rather than truncated to the broker ceiling. + var fallbackStore = RedisTestConnection.CreateStore(connection); + await using var fallbackTransport = new CappedDelayTransport(TimeSpan.FromMinutes(15)); + await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); + var fallbackProcessor = CreateProcessor(fallbackStore, fallbackTransport).Processor; + + await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + Assert.Equal(0, fallbackTransport.SendCount); + + // Durably parked in Redis and time-gated: a drain before the due time claims nothing; only when due does the + // pump pull it from Redis and hand it to the transport. + Assert.Equal(0, await fallbackProcessor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + Assert.Equal(0, fallbackTransport.SendCount); + + Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(now.AddHours(2), cancellationToken: cancellationToken)); + Assert.Equal(1, fallbackTransport.SendCount); + + var delivered = await fallbackQueue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + Assert.NotNull(delivered); + Assert.Equal("later", delivered.Message.Data); + await delivered.CompleteAsync(cancellationToken); + } + + [Fact] + public async Task CronOccurrence_MaterializesRunsAndDedupesThroughRedisAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var cancellationToken = TestContext.Current.CancellationToken; + var store = RedisTestConnection.CreateStore(connection); + var scheduler = new InMemoryJobScheduler(); + var (processor, probe) = CreateProcessor(store, scheduler); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(ProbeJob) + }, cancellationToken); + + // Materialize: one occurrence is written to Redis as a Scheduled JobState + a JobOccurrence dispatch. + var first = await processor.EnqueueDueOccurrencesAsync(now, cancellationToken); + var dispatch = Assert.Single(first); + Assert.Equal("nightly:20260101000000:global", dispatch.DispatchId); + Assert.Equal(ScheduledDispatchKind.JobOccurrence, dispatch.Kind); + Assert.Equal("nightly", dispatch.Headers["job.name"]); + + var scheduled = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(scheduled); + Assert.Equal(JobStatus.Scheduled, scheduled.Status); + Assert.Equal(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero), scheduled.ScheduledForUtc); + + // Deterministic occurrence id dedupes against the Redis row: a second materialize pass at the same time is a no-op. + Assert.Empty(await processor.EnqueueDueOccurrencesAsync(now, cancellationToken)); + + // Claim from Redis and run: the occurrence completes and the run is recorded once. + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + Assert.Equal(1, probe.RunCount); + + var completed = await store.GetAsync(dispatch.JobId!, cancellationToken); + Assert.NotNull(completed); + Assert.Equal(JobStatus.Completed, completed.Status); + Assert.Equal(1, completed.Attempt); + Assert.Equal(100, completed.Progress); + + // The dispatch was completed (removed) in Redis, so a later drain finds nothing. + Assert.Equal(0, await processor.RunDueOccurrencesAsync(now.AddMinutes(1), cancellationToken: cancellationToken)); + } + + [Fact] + public async Task CronOccurrence_RetryDeadLetterAndStaleReclaimThroughRedisAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var cancellationToken = TestContext.Current.CancellationToken; + var store = RedisTestConnection.CreateStore(connection); + var scheduler = new InMemoryJobScheduler(); + var (processor, probe) = CreateProcessor(store, scheduler); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + // (a) Retry-then-dead-letter: a failing occurrence is rescheduled in Redis until its retry budget is spent. + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "flaky", + Cron = "* * * * *", + JobType = typeof(FailingJob), + MaxRetries = 1 + }, cancellationToken); + var flaky = Assert.Single(await processor.EnqueueDueOccurrencesAsync(now, cancellationToken)); + + Assert.Equal(0, await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + var retried = await store.GetAsync(flaky.JobId!, cancellationToken); + Assert.NotNull(retried); + Assert.Equal(JobStatus.Scheduled, retried.Status); + Assert.Equal(1, retried.Attempt); + + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now.AddMinutes(2), cancellationToken: cancellationToken)); + var deadlettered = await store.GetAsync(flaky.JobId!, cancellationToken); + Assert.NotNull(deadlettered); + Assert.Equal(JobStatus.DeadLettered, deadlettered.Status); + Assert.Equal(2, deadlettered.Attempt); + + // (b) Stale reclaim: an occurrence stuck in Processing under a dead node with an expired lease is reclaimed + // (via the Redis CAS reclaim) and run to completion by the live node. + const string jobId = "nightly:20260101000000:global"; + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "nightly", + Cron = "* * * * *", + JobType = typeof(ProbeJob), + MaxRetries = 1 + }, cancellationToken); + await store.CreateIfAbsentAsync(new JobState + { + JobId = jobId, + Name = "nightly", + Status = JobStatus.Processing, + Attempt = 1, + NodeId = "node-b", + LeaseExpiresUtc = now.AddMinutes(-1), + ScheduledForUtc = now.AddSeconds(-30) + }, cancellationToken); + await store.ScheduleDispatchAsync(new ScheduledDispatchState + { + DispatchId = jobId, + Kind = ScheduledDispatchKind.JobOccurrence, + Destination = "nightly", + Body = Array.Empty(), + DueUtc = now, + JobId = jobId + }, cancellationToken); + + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken)); + var reclaimed = await store.GetAsync(jobId, cancellationToken); + Assert.NotNull(reclaimed); + Assert.Equal(JobStatus.Completed, reclaimed.Status); + Assert.Equal(2, reclaimed.Attempt); + Assert.Equal(1, probe.RunCount); + } + + private static (JobScheduleProcessor Processor, Probe Probe) CreateProcessor(IJobRuntimeStore store, IMessageTransport? transport = null) + => CreateProcessor(store, new InMemoryJobScheduler(), transport); + + private static (JobScheduleProcessor Processor, Probe Probe) CreateProcessor(IJobRuntimeStore store, IJobScheduler scheduler, IMessageTransport? transport = null) + { + var probe = new Probe(); + var serviceProvider = new ServiceCollection().AddSingleton(probe).BuildServiceProvider(); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + return (new JobScheduleProcessor(scheduler, store, worker, nodeId: "node-a", transport: transport), probe); + } + + private sealed class Probe + { + private int _runCount; + public int RunCount => Volatile.Read(ref _runCount); + public void Record() => Interlocked.Increment(ref _runCount); + } + + private sealed class ProbeJob(Probe probe) : IJob + { + public Task RunAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + probe.Record(); + return Task.FromResult(JobResult.Success); + } + } + + private sealed class FailingJob : IJob + { + public Task RunAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(JobResult.FromException(new InvalidOperationException("boom"))); + } + } + + private sealed class PreviewWorkItem + { + public string? Data { get; set; } + } + + // Minimal pull transport with a configurable native delayed-delivery ceiling, so a delay beyond the cap is forced + // through the runtime store (mirrors the fixture used by the in-memory MessageQueue tests). + private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ISupportsDelayedDelivery + { + private readonly Queue _entries = new(); + + public CappedDelayTransport(TimeSpan? maxDeliveryDelay) => MaxDeliveryDelay = maxDeliveryDelay; + + public TimeSpan? MaxDeliveryDelay { get; } + public int SendCount { get; private set; } + public TransportSendOptions? LastSendOptions { get; private set; } + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + SendCount += messages.Count; + LastSendOptions = options; + var items = new SendItemResult[messages.Count]; + for (int i = 0; i < messages.Count; i++) + { + string id = messages[i].MessageId ?? Guid.NewGuid().ToString("N"); + _entries.Enqueue(new TransportEntry { Id = id, Destination = destination, Body = messages[i].Body, Headers = messages[i].Headers, Receipt = new Receipt() }); + items[i] = new SendItemResult { MessageId = id, Success = true }; + } + + return Task.FromResult(new SendResult { Items = items }); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + => Task.FromResult>(_entries.Count > 0 ? [_entries.Dequeue()] : []); + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/tests/Foundatio.Redis.Tests/RedisTestConnection.cs b/tests/Foundatio.Redis.Tests/RedisTestConnection.cs new file mode 100644 index 000000000..41a06dfc5 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisTestConnection.cs @@ -0,0 +1,30 @@ +using System; +using Foundatio.Jobs; +using StackExchange.Redis; + +namespace Foundatio.Redis.Tests; + +/// +/// Shared, lazily-opened Redis connection for the Redis test suites. Gated on +/// FOUNDATIO_REDIS_CONNECTION_STRING (e.g. localhost:6399 for the bundled docker-compose Redis); when it +/// is unset is null and the suites skip every test. +/// +internal static class RedisTestConnection +{ + private static readonly Lazy Shared = new(() => + { + string? connectionString = Environment.GetEnvironmentVariable("FOUNDATIO_REDIS_CONNECTION_STRING"); + return String.IsNullOrEmpty(connectionString) ? null : ConnectionMultiplexer.Connect(connectionString); + }); + + public static IConnectionMultiplexer? Multiplexer => Shared.Value; + + /// Creates a store under a unique key prefix so concurrent tests and leftover keys never collide. + public static RedisJobRuntimeStore CreateStore(IConnectionMultiplexer connection, TimeProvider? timeProvider = null) => + new(new RedisJobRuntimeStoreOptions + { + ConnectionMultiplexer = connection, + KeyPrefix = $"fnd-it:{Guid.NewGuid():N}:", + TimeProvider = timeProvider + }); +} From 5e8eba961b623315f8a9504d78aea99d1116c9ff Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 29 Jun 2026 19:50:43 -0500 Subject: [PATCH 19/57] Add Redis Streams message transport (at-least-once, ack/retry/dead-letter) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pivots the Redis pub/sub work to Streams + consumer groups, the Redis primitive that actually supports ack/reject/retry (native pub/sub is fire-and-forget and can't redeliver). A stream is a queue/topic; a consumer group is a subscription — the default group for a plain queue gives competing consumers, one group per named subscription gives topic fan-out. RedisStreamsMessageTransport implements ISupportsPull, VisibilityTimeout, LockRenewal, RedeliveryDelay, DeadLetter, Provisioning, Stats and ITransportInfo (AtLeastOnce/Fifo) — the same capability set as the AWS SQS transport plus native dead-letter. XADD produces, XREADGROUP consumes, XACK+XDEL completes, and reclaim (abandon, redelivery delay, lock expiry, crashed consumer) is driven by a per-group lease: a sorted set scored by visible-until (unix-ms) plus a hash of owner-token|delivery -count. Because the lease lives in Redis, a message held by a crashed instance is recovered by any other instance; a stale receipt is detected by the owner token and surfaced as ReceiptExpiredException. Streams has no native per-message delay/priority, so those route through the runtime store / are unsupported (the contract's core owns that). Validation against redis:7: the cross-transport conformance suite runs 10/14 (push/priority/expiration/delayed-delivery skip via capability gates) and integration tests cover cross-instance crash recovery, the core's retry-then-dead-letter machinery driving the transport unchanged, and PubSub fan-out. Full Redis suite 22/22 green and stable; in-memory unaffected. Co-Authored-By: Claude Opus 4.8 --- .../Messaging/RedisStreamsMessageTransport.cs | 470 ++++++++++++++++++ .../RedisStreamsMessageTransportOptions.cs | 28 ++ tests/Foundatio.Redis.Tests/README.md | 15 +- .../RedisStreamsTransportConformanceTests.cs | 60 +++ .../RedisStreamsTransportIntegrationTests.cs | 177 +++++++ 5 files changed, 746 insertions(+), 4 deletions(-) create mode 100644 src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs create mode 100644 src/Foundatio.Redis/Messaging/RedisStreamsMessageTransportOptions.cs create mode 100644 tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs create mode 100644 tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs new file mode 100644 index 000000000..2c1d2bc25 --- /dev/null +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -0,0 +1,470 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace Foundatio.Messaging; + +/// +/// An over Redis Streams + consumer groups (at-least-once). Temporary in-repo provider +/// used to validate the redesigned transport contract — and the core's retry/dead-letter machinery — against a real +/// broker. A stream is a queue/topic; a consumer group is a subscription (the default group for a plain queue gives +/// competing consumers; one group per named subscription gives topic fan-out). +/// +/// +/// Streams has no per-message visible-until or per-message delay, so this transport keeps the lease explicitly: a +/// per-group sorted set (member = stream entry id, score = visible-until unix-ms) is the authoritative +/// in-flight lease and a per-group hash holds token|delivery-count per entry. Reclaim (abandon, redelivery +/// delay, lock expiry, crashed consumer) is driven by that sorted set — entries whose lease has lapsed are +/// XCLAIMed and redelivered (same stream id, delivery count incremented). Because the lease lives in Redis, a +/// message held by a crashed instance is recovered by any other instance. A stale receipt (already settled, or the +/// entry was redelivered to someone else) is detected by an owner token and surfaced as . +/// +public sealed class RedisStreamsMessageTransport : IMessageTransport, ISupportsPull, ISupportsVisibilityTimeout, + ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsDeadLetter, ISupportsProvisioning, ISupportsStats, ITransportInfo +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(250); + + private static readonly IReadOnlySet _supportedRoles = + new HashSet { DestinationRole.Queue, DestinationRole.Topic, DestinationRole.Subscription, DestinationRole.Binding }; + + private readonly RedisStreamsMessageTransportOptions _options; + private readonly IDatabase _db; + private readonly TimeProvider _timeProvider; + private readonly string _prefix; + private readonly string _consumer; + // Logical destination name -> resolved (stream key, consumer group, group-create position). Populated by EnsureAsync. + private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _ensuredGroups = new(StringComparer.Ordinal); + private int _isDisposed; + + public RedisStreamsMessageTransport(RedisStreamsMessageTransportOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + ArgumentNullException.ThrowIfNull(options.ConnectionMultiplexer); + _db = options.ConnectionMultiplexer.GetDatabase(); + _timeProvider = options.TimeProvider ?? TimeProvider.System; + _prefix = options.KeyPrefix ?? ""; + _consumer = !String.IsNullOrEmpty(options.ConsumerName) ? options.ConsumerName : $"c-{Guid.NewGuid():N}"[..16]; + } + + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; + public IReadOnlySet SupportedRoles => _supportedRoles; + public int? MaxBatchSize => null; + public long? MaxMessageBytes => null; + public TimeSpan? MaxRedeliveryDelay => null; // lease is tracked in Redis, so any delay is honored + public TimeSpan? MaxVisibilityTimeout => null; + + public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(messages); + + // The stream IS the queue/topic; subscriptions read it through their own group, so a send is always an XADD to + // the destination's stream regardless of role. + RedisKey streamKey = StreamKey(destination); + var items = new List(messages.Count); + foreach (var message in messages) + { + RedisValue id = await _db.StreamAddAsync(streamKey, BuildFields(message), messageId: null, + maxLength: _options.MaxStreamLength, useApproximateMaxLength: true).ConfigureAwait(false); + items.Add(new SendItemResult { MessageId = id.ToString(), Success = true }); + } + + return new SendResult { Items = items }; + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + => ReceiveAsync(source, request, _options.DefaultVisibilityTimeout, ct); + + public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(request); + + var resolved = Resolve(source); + await EnsureGroupAsync(resolved).ConfigureAwait(false); + + int max = Math.Max(1, request.MaxMessages); + long visibilityMs = (long)Math.Max(0, visibility.TotalMilliseconds); + var deadline = _timeProvider.GetUtcNow() + (request.MaxWaitTime ?? TimeSpan.Zero); + + while (true) + { + ct.ThrowIfCancellationRequested(); + var entries = await PollOnceAsync(source, resolved, max, visibilityMs, ct).ConfigureAwait(false); + if (entries.Count > 0) + return entries; + + var remaining = deadline - _timeProvider.GetUtcNow(); + if (remaining <= TimeSpan.Zero) + return []; + + await Task.Delay(remaining < PollInterval ? remaining : PollInterval, ct).ConfigureAwait(false); + } + } + + private async Task> PollOnceAsync(string source, ResolvedSource resolved, int max, long visibilityMs, CancellationToken ct) + { + var result = new List(max); + long nowMs = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); + RedisKey lockKey = LockKey(resolved); + RedisKey metaKey = MetaKey(resolved); + + // 1. Reclaim entries whose lease has lapsed (abandoned, redelivery-delay due, lock expired, crashed consumer). + // The lease score is updated only after the claim, never removed first, so a crash mid-reclaim can't orphan an + // entry (it stays reclaimable); the cost is that two instances racing the same lapsed entry may both deliver it + // — acceptable under at-least-once. + var dueIds = await _db.SortedSetRangeByScoreAsync(lockKey, Double.NegativeInfinity, nowMs, take: max).ConfigureAwait(false); + if (dueIds.Length > 0) + { + var claimed = await _db.StreamClaimAsync(resolved.StreamKey, resolved.Group, _consumer, 0, dueIds).ConfigureAwait(false); + foreach (var entry in claimed) + { + ct.ThrowIfCancellationRequested(); + if (entry.IsNull || entry.Values is not { Length: > 0 }) + { + // The entry was settled/trimmed since we read the lease; drop our bookkeeping for it. + await _db.SortedSetRemoveAsync(lockKey, entry.Id).ConfigureAwait(false); + await _db.HashDeleteAsync(metaKey, entry.Id).ConfigureAwait(false); + continue; + } + + int deliveries = ParseDeliveries(await _db.HashGetAsync(metaKey, entry.Id).ConfigureAwait(false)) + 1; + result.Add(await TrackAsync(source, resolved, entry, deliveries, nowMs, visibilityMs).ConfigureAwait(false)); + if (result.Count >= max) + return result; + } + } + + // 2. New, never-delivered entries. + var fresh = await _db.StreamReadGroupAsync(resolved.StreamKey, resolved.Group, _consumer, StreamPosition.NewMessages, max - result.Count).ConfigureAwait(false); + foreach (var entry in fresh) + { + ct.ThrowIfCancellationRequested(); + result.Add(await TrackAsync(source, resolved, entry, 1, nowMs, visibilityMs).ConfigureAwait(false)); + } + + return result; + } + + // Records the lease (sorted set) + owner token & delivery count (hash) for a just-delivered entry and projects it + // into a TransportEntry whose Receipt carries everything needed to settle it. + private async Task TrackAsync(string source, ResolvedSource resolved, StreamEntry entry, int deliveries, long nowMs, long visibilityMs) + { + string token = Guid.NewGuid().ToString("N"); + await _db.HashSetAsync(MetaKey(resolved), entry.Id, $"{token}|{deliveries}").ConfigureAwait(false); + await _db.SortedSetAddAsync(LockKey(resolved), entry.Id, nowMs + visibilityMs).ConfigureAwait(false); + return ToEntry(source, resolved, entry, deliveries, token); + } + + public async Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + ThrowIfDisposed(); + var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); + + long acked = await _db.StreamAcknowledgeAsync(r.StreamKey, r.Group, r.EntryId).ConfigureAwait(false); + await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); + await ClearTrackingAsync(r).ConfigureAwait(false); + + if (acked == 0) + throw new ReceiptExpiredException(); + } + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => AbandonAsync(entry, TimeSpan.Zero, ct); + + public async Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct) + { + ThrowIfDisposed(); + var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); + + // Make the (still-pending) entry reclaimable when the delay lapses; the reclaim pass redelivers the same stream + // id with an incremented delivery count. delay <= 0 => immediately due. + long dueMs = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds() + (long)Math.Max(0, redeliveryDelay.TotalMilliseconds); + await _db.SortedSetAddAsync(LockKey(r), r.EntryId, dueMs).ConfigureAwait(false); + } + + public async Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct) + { + ThrowIfDisposed(); + var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); + long until = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds() + (long)(duration ?? _options.DefaultVisibilityTimeout).TotalMilliseconds; + await _db.SortedSetAddAsync(LockKey(r), r.EntryId, until).ConfigureAwait(false); + } + + public async Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct) + { + ThrowIfDisposed(); + var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); + + var headers = entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason ?? "").Build(); + await _db.StreamAddAsync(DeadKey(r.StreamKey), BuildFields(entry.Id, entry.Body, headers), messageId: null, + maxLength: _options.MaxStreamLength, useApproximateMaxLength: true).ConfigureAwait(false); + + await _db.StreamAcknowledgeAsync(r.StreamKey, r.Group, r.EntryId).ConfigureAwait(false); + await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); + await ClearTrackingAsync(r).ConfigureAwait(false); + } + + public async Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(request); + + RedisKey deadKey = DeadKey(StreamKey(destination)); + var entries = await _db.StreamRangeAsync(deadKey, count: Math.Max(1, request.MaxMessages)).ConfigureAwait(false); + if (entries.Length == 0) + return []; + + var result = new List(entries.Length); + var ids = new RedisValue[entries.Length]; + for (int i = 0; i < entries.Length; i++) + { + ids[i] = entries[i].Id; + result.Add(ToEntry(destination, resolved: null, entries[i], deliveries: 1, token: "")); + } + + // Inspecting the dead-letter backlog consumes it. + await _db.StreamDeleteAsync(deadKey, ids).ConfigureAwait(false); + return result; + } + + public async Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(declarations); + + foreach (var declaration in declarations) + { + switch (declaration.Role) + { + case DestinationRole.Topic: + // Topics are read through subscription groups; nothing to create until a subscription appears. + break; + case DestinationRole.Subscription: + case DestinationRole.Binding: + string topic = declaration.Source ?? declaration.Name; + var sub = new ResolvedSource(StreamKey(topic), declaration.Name, "$"); + _sources[declaration.Name] = sub; + await EnsureGroupAsync(sub).ConfigureAwait(false); + break; + default: + var queue = new ResolvedSource(StreamKey(declaration.Name), _options.DefaultConsumerGroup, "0"); + _sources[declaration.Name] = queue; + await EnsureGroupAsync(queue).ConfigureAwait(false); + break; + } + } + } + + public async Task DeleteAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(name); + + var resolved = Resolve(name); + await _db.KeyDeleteAsync([resolved.StreamKey, DeadKey(resolved.StreamKey), LockKey(resolved), MetaKey(resolved)]).ConfigureAwait(false); + _sources.TryRemove(name, out _); + _ensuredGroups.TryRemove(GroupKey(resolved), out _); + } + + public Task ExistsAsync(string name, CancellationToken ct) + { + ThrowIfDisposed(); + ArgumentException.ThrowIfNullOrEmpty(name); + return _db.KeyExistsAsync(Resolve(name).StreamKey); + } + + public async Task GetStatsAsync(string destination, CancellationToken ct) + { + ThrowIfDisposed(); + var resolved = Resolve(destination); + await EnsureGroupAsync(resolved).ConfigureAwait(false); + + long length = await _db.StreamLengthAsync(resolved.StreamKey).ConfigureAwait(false); + long working = (await _db.StreamPendingAsync(resolved.StreamKey, resolved.Group).ConfigureAwait(false)).PendingMessageCount; + RedisKey deadKey = DeadKey(resolved.StreamKey); + long dead = await _db.KeyExistsAsync(deadKey).ConfigureAwait(false) ? await _db.StreamLengthAsync(deadKey).ConfigureAwait(false) : 0; + + return new MessageDestinationStats + { + Queued = Math.Max(0, length - working), + Working = working, + Deadletter = dead + }; + } + + public ValueTask DisposeAsync() + { + Interlocked.Exchange(ref _isDisposed, 1); + return ValueTask.CompletedTask; // the connection multiplexer is owned by the caller + } + + private async Task ValidateReceiptAsync(TransportEntry entry) + { + if (entry.Receipt.TransportState is not StreamReceipt r) + throw new ReceiptExpiredException("The transport entry does not carry a Redis Streams receipt."); + + // The owner token guards stale receipts: once the entry is redelivered (reclaimed) or settled, the token in the + // meta hash no longer matches, so a late Complete/Abandon from the previous holder is rejected. + var current = await _db.HashGetAsync(MetaKey(r), r.EntryId).ConfigureAwait(false); + if (current.IsNull || ParseToken(current) != r.Token) + throw new ReceiptExpiredException(); + + return r; + } + + private async Task ClearTrackingAsync(StreamReceipt r) + { + await _db.SortedSetRemoveAsync(LockKey(r), r.EntryId).ConfigureAwait(false); + await _db.HashDeleteAsync(MetaKey(r), r.EntryId).ConfigureAwait(false); + } + + private async Task EnsureGroupAsync(ResolvedSource resolved) + { + if (!_ensuredGroups.TryAdd(GroupKey(resolved), 0)) + return; + + try + { + await _db.StreamCreateConsumerGroupAsync(resolved.StreamKey, resolved.Group, resolved.Position, createStream: true).ConfigureAwait(false); + } + catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.Ordinal)) + { + // Group already exists — creation is idempotent. + } + } + + private ResolvedSource Resolve(string source) + { + if (_sources.TryGetValue(source, out var registered)) + return registered; + + // PubSub facade sources are "topic/subscription"; a bare name is a queue on the default group. + int slash = source.IndexOf('/'); + return slash > 0 + ? new ResolvedSource(StreamKey(source[..slash]), source[(slash + 1)..], "$") + : new ResolvedSource(StreamKey(source), _options.DefaultConsumerGroup, "0"); + } + + private TransportEntry ToEntry(string destination, ResolvedSource? resolved, StreamEntry entry, int deliveries, string token) + { + string? messageId = GetField(entry, "id"); + var headers = DecodeHeaders(GetField(entry, "h")); + Receipt receipt = resolved is null + ? default + : new Receipt { TransportState = new StreamReceipt(resolved.StreamKey.ToString(), resolved.Group, entry.Id.ToString(), token) }; + + return new TransportEntry + { + Id = String.IsNullOrEmpty(messageId) ? entry.Id.ToString() : messageId, + Destination = destination, + Body = GetBody(entry), + Headers = headers, + DeliveryCount = deliveries, + EnqueuedUtc = ParseStreamIdTime(entry.Id), + Receipt = receipt + }; + } + + private static NameValueEntry[] BuildFields(TransportMessage message) + => BuildFields(message.MessageId, message.Body, message.Headers, message.ContentType); + + private static NameValueEntry[] BuildFields(string? messageId, ReadOnlyMemory body, MessageHeaders headers, string? contentType = null) + { + return + [ + new NameValueEntry("id", messageId ?? ""), + new NameValueEntry("ct", contentType ?? ""), + new NameValueEntry("h", EncodeHeaders(headers)), + new NameValueEntry("b", body.ToArray()) + ]; + } + + private static string? GetField(StreamEntry entry, string name) + { + foreach (var value in entry.Values) + { + if (value.Name == name) + return value.Value.IsNull ? null : value.Value.ToString(); + } + + return null; + } + + private static ReadOnlyMemory GetBody(StreamEntry entry) + { + foreach (var value in entry.Values) + { + if (value.Name == "b") + return value.Value.IsNullOrEmpty ? ReadOnlyMemory.Empty : (byte[])value.Value!; + } + + return ReadOnlyMemory.Empty; + } + + private static string EncodeHeaders(MessageHeaders headers) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var header in headers) + map[header.Key] = header.Value; + return JsonSerializer.Serialize(map); + } + + private static MessageHeaders DecodeHeaders(string? json) + { + if (String.IsNullOrEmpty(json)) + return MessageHeaders.Empty; + var map = JsonSerializer.Deserialize>(json); + return map is null ? MessageHeaders.Empty : MessageHeaders.Create(map); + } + + // Stream ids are "-"; the timestamp half is the broker enqueue time. + private static DateTimeOffset? ParseStreamIdTime(RedisValue id) + { + string s = id.ToString(); + int dash = s.IndexOf('-'); + string ms = dash > 0 ? s[..dash] : s; + return Int64.TryParse(ms, NumberStyles.Integer, CultureInfo.InvariantCulture, out long unixMs) + ? DateTimeOffset.FromUnixTimeMilliseconds(unixMs) + : null; + } + + private static int ParseDeliveries(RedisValue meta) + { + if (meta.IsNullOrEmpty) + return 0; + string s = meta.ToString(); + int bar = s.IndexOf('|'); + return bar >= 0 && Int32.TryParse(s.AsSpan(bar + 1), NumberStyles.Integer, CultureInfo.InvariantCulture, out int n) ? n : 0; + } + + private static string ParseToken(RedisValue meta) + { + string s = meta.ToString(); + int bar = s.IndexOf('|'); + return bar >= 0 ? s[..bar] : s; + } + + private RedisKey StreamKey(string name) => $"{_prefix}{name}"; + private static RedisKey DeadKey(RedisKey streamKey) => streamKey.ToString() + ":dead"; + private static RedisKey LockKey(ResolvedSource r) => $"{r.StreamKey}:lock:{r.Group}"; + private static RedisKey MetaKey(ResolvedSource r) => $"{r.StreamKey}:meta:{r.Group}"; + private static RedisKey LockKey(StreamReceipt r) => $"{r.StreamKey}:lock:{r.Group}"; + private static RedisKey MetaKey(StreamReceipt r) => $"{r.StreamKey}:meta:{r.Group}"; + private static string GroupKey(ResolvedSource r) => $"{r.StreamKey}|{r.Group}"; + + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(Volatile.Read(ref _isDisposed) == 1, this); + + private sealed record ResolvedSource(RedisKey StreamKey, string Group, RedisValue Position); + + private sealed record StreamReceipt(string StreamKey, string Group, string EntryId, string Token); +} diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransportOptions.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransportOptions.cs new file mode 100644 index 000000000..2d053e356 --- /dev/null +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransportOptions.cs @@ -0,0 +1,28 @@ +using System; +using StackExchange.Redis; + +namespace Foundatio.Messaging; + +public class RedisStreamsMessageTransportOptions +{ + /// The Redis connection to use. Required. + public IConnectionMultiplexer ConnectionMultiplexer { get; set; } = null!; + + /// Prefix applied to every stream/key this transport creates. Isolates environments/runs on a shared Redis. + public string KeyPrefix { get; set; } = "fnd:msg:"; + + /// Consumer-group name used for plain queue destinations (its members are competing consumers). + public string DefaultConsumerGroup { get; set; } = "foundatio"; + + /// How long a received message stays invisible to other consumers before it can be reclaimed (the lease). + public TimeSpan DefaultVisibilityTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// Approximate MAXLEN cap applied on XADD (null = no trimming). Trimming can drop un-acked entries; keep ample headroom. + public int? MaxStreamLength { get; set; } + + /// This node's consumer name within every group (defaults to a stable per-instance id). Distinct instances are competing consumers. + public string? ConsumerName { get; set; } + + /// Time source (defaults to ). + public TimeProvider? TimeProvider { get; set; } +} diff --git a/tests/Foundatio.Redis.Tests/README.md b/tests/Foundatio.Redis.Tests/README.md index df67fa68f..cda720aab 100644 --- a/tests/Foundatio.Redis.Tests/README.md +++ b/tests/Foundatio.Redis.Tests/README.md @@ -1,7 +1,13 @@ # Foundatio.Redis.Tests -Validates the temporary in-repo `RedisJobRuntimeStore` against a real Redis by running the shared -`JobRuntimeStoreConformanceTests` suite (the same assertions the in-memory reference store passes). +Validates the temporary in-repo Redis providers against a real Redis: + +- **`RedisJobRuntimeStore`** — runs the shared `JobRuntimeStoreConformanceTests` suite (the same assertions the + in-memory reference store passes) plus `RedisJobStoreIntegrationTests` (delayed-send fallback + CRON end-to-end). +- **`RedisStreamsMessageTransport`** — runs the shared `MessageTransportConformanceTests` suite (pull, settlement, + visibility timeout, lock renewal, redelivery delay, dead-letter, provisioning, stats, topic fan-out; push/priority/ + expiration/delayed-delivery skip via capability gates) plus `RedisStreamsTransportIntegrationTests` (cross-instance + crash recovery, the core's retry/dead-letter machinery over Streams, and `PubSub` fan-out). ## Running @@ -14,5 +20,6 @@ export FOUNDATIO_REDIS_CONNECTION_STRING=localhost:6399 dotnet run --project tests/Foundatio.Redis.Tests ``` -Each test runs under a unique key prefix (`fnd-conf:{guid}:`), so concurrent runs and leftover keys never collide. -A `FakeTimeProvider` drives lease/expiry timing, so the suite is fast and deterministic — no real sleeps. +Each test runs under a unique key prefix, so concurrent runs and leftover keys never collide. The job-store suite +drives lease/expiry timing with a `FakeTimeProvider` (no real sleeps); the transport suite uses real, whole-second +timing windows (the same cross-transport windows the AWS suite uses). diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs new file mode 100644 index 000000000..2fa849d88 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Xunit; + +namespace Foundatio.Redis.Tests; + +/// +/// Runs the shared transport conformance suite against the Redis Streams transport. Set +/// FOUNDATIO_REDIS_CONNECTION_STRING (e.g. localhost:6399) to run; skips when unset. A unique key prefix +/// per transport isolates each test's streams. Capabilities Streams does not provide (push delivery, per-message +/// priority, per-message expiration, native scheduled delivery) are skipped by the base suite's capability gates. +/// +public class RedisStreamsTransportConformanceTests : MessageTransportConformanceTests +{ + public RedisStreamsTransportConformanceTests(ITestOutputHelper output) : base(output) { } + + protected override IMessageTransport? CreateTransport() + { + if (RedisTestConnection.Multiplexer is not { } connection) + return null; // not configured -> the base suite skips every test + + return new RedisStreamsMessageTransport(new RedisStreamsMessageTransportOptions + { + ConnectionMultiplexer = connection, + KeyPrefix = $"fnd-conf:{Guid.NewGuid():N}:" + }); + } + + [Fact] + public override Task CanSendAndReceiveBatchAsync() => base.CanSendAndReceiveBatchAsync(); + + [Fact] + public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() => base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); + + [Fact] + public override Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() => base.CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync(); + + [Fact] + public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() => base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); + + [Fact] + public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() => base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); + + [Fact] + public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() => base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); + + [Fact] + public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() => base.RenewLockAsync_ExtendsVisibilityWindowAsync(); + + [Fact] + public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() => base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); + + [Fact] + public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() => base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); + + [Fact] + public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() => base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); +} diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs new file mode 100644 index 000000000..e4809ec52 --- /dev/null +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs @@ -0,0 +1,177 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Redis.Tests; + +/// +/// End-to-end tests for the Redis Streams transport that the cross-transport conformance suite can't express: at-least-once +/// recovery across two consumer instances, the core's retry/dead-letter machinery driving the transport, and topic +/// fan-out through the facade. Gated on FOUNDATIO_REDIS_CONNECTION_STRING; unique key prefix +/// per test. +/// +public class RedisStreamsTransportIntegrationTests +{ + private static RedisStreamsMessageTransport CreateTransport(StackExchange.Redis.IConnectionMultiplexer connection, string prefix, string? consumer = null) => + new(new RedisStreamsMessageTransportOptions + { + ConnectionMultiplexer = connection, + KeyPrefix = prefix, + ConsumerName = consumer + }); + + private static string NewPrefix() => $"fnd-it:{Guid.NewGuid():N}:"; + + [Fact] + public async Task CrashedConsumer_LeaseLapses_AnotherInstanceReclaimsAndCompletesAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var ct = TestContext.Current.CancellationToken; + string prefix = NewPrefix(); + var visibility = TimeSpan.FromSeconds(2); + + // Both instances share the same key prefix so they operate on the same streams (the lease lives in Redis). + await using var nodeA = CreateTransport(connection, prefix, "node-a"); + await using var nodeB = CreateTransport(connection, prefix, "node-b"); + + await nodeA.EnsureAsync([new DestinationDeclaration { Name = "work", Role = DestinationRole.Queue }], ct); + await nodeA.SendAsync("work", [Message("survive-me")], new TransportSendOptions(), ct); + + // node-a receives and then "crashes" — it never settles the message. + var heldByA = Assert.Single(await nodeA.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibility, ct)); + Assert.Equal(1, heldByA.DeliveryCount); + + // While node-a's lease is live, node-b must not see it. + Assert.Empty(await nodeB.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibility, ct)); + + // After the lease lapses, node-b reclaims the in-flight message (lease state lives in Redis) and completes it. + var reclaimedByB = Assert.Single(await nodeB.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = visibility + TimeSpan.FromSeconds(5) }, visibility, ct)); + Assert.Equal(heldByA.Id, reclaimedByB.Id); + Assert.Equal(2, reclaimedByB.DeliveryCount); + Assert.Equal("survive-me", System.Text.Encoding.UTF8.GetString(reclaimedByB.Body.Span)); + await nodeB.CompleteAsync(reclaimedByB, ct); + + var stats = await nodeB.GetStatsAsync("work", ct); + Assert.Equal(0, stats.Queued); + Assert.Equal(0, stats.Working); + } + + [Fact] + public async Task Core_RetriesFailedHandler_ThenDeadLettersAfterMaxAttemptsAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var ct = TestContext.Current.CancellationToken; + var transport = CreateTransport(connection, NewPrefix()); + await using var queue = new MessageQueue(transport, new QueueOptions()); + + // (a) A handler that throws once is redelivered (via the transport) and succeeds on the second attempt — the + // core's retry machinery works unchanged over Streams. + int retryAttempts = 0; + var succeeded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var retryConsumer = await queue.StartConsumerAsync((message, _) => + { + int attempt = Interlocked.Increment(ref retryAttempts); + if (attempt == 1) + throw new InvalidOperationException("first attempt fails"); + + Assert.Equal(2, message.Attempts); + succeeded.TrySetResult(); + return Task.CompletedTask; + }, new QueueConsumerOptions { MaxAttempts = 3, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(200) }, ct); + + // (b) A handler that always throws is dead-lettered once its attempt budget is spent. + await using var poisonConsumer = await queue.StartConsumerAsync((_, _) => + throw new InvalidOperationException("always fails"), + new QueueConsumerOptions { MaxAttempts = 2, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(100) }, ct); + + await queue.EnqueueAsync(new RetryItem { Data = "retry" }, cancellationToken: ct); + await queue.EnqueueAsync(new PoisonItem { Data = "poison" }, cancellationToken: ct); + + await succeeded.Task.WaitAsync(TimeSpan.FromSeconds(15), ct); + Assert.Equal(2, Volatile.Read(ref retryAttempts)); + + // The poison message lands in the dead-letter stream after exhausting its 2 attempts. + MessageDestinationStats stats = await transport.GetStatsAsync("streams-poison", ct); + for (int i = 0; i < 100 && stats.Deadletter == 0; i++) + { + await Task.Delay(100, ct); + stats = await transport.GetStatsAsync("streams-poison", ct); + } + + Assert.Equal(1, stats.Deadletter); + Assert.Equal(0, stats.Working); + + // The poison payload is inspectable in the dead-letter stream with a reason recorded by the core. + var deadLettered = Assert.Single(await transport.ReceiveDeadLetteredAsync("streams-poison", new ReceiveRequest { MaxMessages = 10 }, ct)); + Assert.NotEmpty(deadLettered.Headers[KnownHeaders.DeadLetterReason]); + } + + [Fact] + public async Task PubSub_PublishToTopic_FansOutToEverySubscriptionAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var ct = TestContext.Current.CancellationToken; + var transport = CreateTransport(connection, NewPrefix()); + await using var pubsub = new PubSub(transport, new PubSubOptions()); + + var receivedByA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receivedByB = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var subA = await pubsub.SubscribeAsync((message, _) => + { + receivedByA.TrySetResult(message.Message.Data ?? ""); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { Subscription = "sub-a" }, ct); + + await using var subB = await pubsub.SubscribeAsync((message, _) => + { + receivedByB.TrySetResult(message.Message.Data ?? ""); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { Subscription = "sub-b" }, ct); + + await pubsub.PublishAsync(new FanItem { Data = "broadcast" }, cancellationToken: ct); + + // Each named subscription is its own consumer group, so both receive an independent copy. + await Task.WhenAll(receivedByA.Task, receivedByB.Task).WaitAsync(TimeSpan.FromSeconds(15), ct); + Assert.Equal("broadcast", await receivedByA.Task); + Assert.Equal("broadcast", await receivedByB.Task); + } + + private static TransportMessage Message(string body) => + new() { Body = System.Text.Encoding.UTF8.GetBytes(body) }; + + [MessageRoute("streams-retry")] + private sealed class RetryItem + { + public string? Data { get; set; } + } + + [MessageRoute("streams-poison")] + private sealed class PoisonItem + { + public string? Data { get; set; } + } + + [MessageRoute("streams-topic")] + private sealed class FanItem + { + public string? Data { get; set; } + } +} From 6bf54f886002b41bec424f5178215498bf661947 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 00:02:23 -0500 Subject: [PATCH 20/57] Fix review P0 #1 (redelivery loop) + jobs runtime cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial contract review findings, batch 1: - P0 #1: runtime-store redelivery computed nextAttempt from the raw transport DeliveryCount, which resets to 1 on re-send for every real broker (SQS, Redis Streams) — pinning the carried attempts header at 2 and redelivering forever (never reaching MaxAttempts). Now advances from the reconciled Attempts. Masked by the in-memory transport, so added a regression test on a DeliveryCount-resetting transport that asserts attempts advance 1->2->3 (fails 3!=2 without the fix). - #3: CRON occurrences are now excluded from the generic worker (JobQuery.ExcludeOccurrences, applied in both stores; RunQueuedAsync sets it) so the scheduler is the sole executor; and a terminal occurrence's dispatch is retired (CompleteDispatchAsync) instead of rescheduled +1min forever. - #4: Redis TryClaimAsync now guards the lease-steal CAS on the exact observed leaseExpiresUtc (mirroring TryReclaimExpiredAsync), so a concurrent same-owner renew invalidates the steal — no double-run. Conformance Leasing test now covers renew-defeats-steal. - #14: PerNode SkipIfRunning scope match no longer uses a fragile JobId EndsWith(":{scope}") (the default node id contains ':'); it extracts the scope precisely past the fixed-width timestamp. - #15: clarified the two attempt-budget knobs (ad-hoc total-attempts vs scheduled retries). Jobs suites green: 18 in-memory + 6 Redis conformance. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio.Redis/RedisJobRuntimeStore.cs | 17 ++++- .../Jobs/JobRuntimeStoreConformanceTests.cs | 17 ++++- src/Foundatio/Jobs/JobRuntime.cs | 19 ++++- src/Foundatio/Jobs/JobScheduler.cs | 25 ++++++- src/Foundatio/Messaging/MessageClientCore.cs | 5 +- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 73 +++++++++++++++++++ .../Queue/MessageQueueTests.cs | 35 +++++++++ 7 files changed, 184 insertions(+), 7 deletions(-) diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.cs index 8db55b072..ab1589cae 100644 --- a/src/Foundatio.Redis/RedisJobRuntimeStore.cs +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.cs @@ -102,6 +102,8 @@ public async Task> QueryAsync(JobQuery query, Cancellati var states = await LoadAsync(ids).ConfigureAwait(false); return states + // ScheduledForUtc must be filtered after hydration (it isn't indexed), mirroring GetExpiredProcessingAsync. + .Where(s => !query.ExcludeOccurrences || s.ScheduledForUtc is null) .OrderByDescending(s => s.LastUpdatedUtc) .Take(Math.Max(1, query.Limit)) .ToArray(); @@ -137,8 +139,19 @@ public async Task TryClaimAsync(string jobId, string nodeId, TimeSpan leas return false; var tx = _db.CreateTransaction(); - // Predicate on the owner we observed so a competing claim that lands first invalidates this one. - tx.AddCondition(String.IsNullOrEmpty(owner) ? Condition.HashNotExists(JobKey(jobId), "nodeId") : Condition.HashEqual(JobKey(jobId), "nodeId", owner)); + if (String.IsNullOrEmpty(owner)) + { + tx.AddCondition(Condition.HashNotExists(JobKey(jobId), "nodeId")); + } + else + { + // Stealing an expired lease: predicate on BOTH the observed owner and the exact lease value, so a + // concurrent renew by that owner (which rewrites leaseExpiresUtc) invalidates the steal and can't + // double-run. Mirrors TryReclaimExpiredAsync; the unguarded version could overwrite a freshly-renewed lease. + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "nodeId", owner)); + if (!current[1].IsNullOrEmpty) + tx.AddCondition(Condition.HashEqual(JobKey(jobId), "leaseExpiresUtc", current[1])); + } _ = tx.HashSetAsync(JobKey(jobId), [ new HashEntry("nodeId", nodeId), diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs index 22622028b..da54c6ba8 100644 --- a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -153,6 +153,14 @@ public virtual async Task Query_FiltersByNameStatusAndLimitAsync() // Limit is honored against the newest-first ordering, so the most recently updated row wins. var limited = await store.QueryAsync(new JobQuery { Limit = 1 }, ct); Assert.Equal("c", Assert.Single(limited).JobId); + + // ExcludeOccurrences filters out CRON occurrences (ScheduledForUtc set) so the generic worker's Queued query + // never claims scheduler-owned jobs. + await store.CreateIfAbsentAsync(NewJob(time, "d", "alpha", JobStatus.Queued) with { LastUpdatedUtc = t.AddSeconds(3), ScheduledForUtc = t }, ct); + var adHocQueued = await store.QueryAsync(new JobQuery { Status = JobStatus.Queued, ExcludeOccurrences = true }, ct); + Assert.Equal(new HashSet { "a", "c" }, adHocQueued.Select(j => j.JobId).ToHashSet()); // "d" excluded (occurrence) + var adHocAlpha = await store.QueryAsync(new JobQuery { Name = "alpha", ExcludeOccurrences = true }, ct); + Assert.Equal(new HashSet { "a", "b" }, adHocAlpha.Select(j => j.JobId).ToHashSet()); // "d" excluded (occurrence) } public virtual async Task Leasing_ClaimRenewReleaseAndStealAsync() @@ -183,7 +191,14 @@ public virtual async Task Leasing_ClaimRenewReleaseAndStealAsync() Assert.True(await store.RenewClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(10), ct)); Assert.Equal(time.GetUtcNow().AddMinutes(10), (await store.GetAsync("job-1", ct))!.LeaseExpiresUtc); - // Once the lease lapses, another node may steal the claim. + // A renewed lease is not stealable: after the lease would have lapsed the owner renews, so a competing steal + // must fail rather than act on a stale expired-lease observation (the steal CAS must see the renew → no double-run). + time.Advance(TimeSpan.FromMinutes(11)); + Assert.True(await store.RenewClaimAsync("job-1", "node-a", TimeSpan.FromMinutes(10), ct)); + Assert.False(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(5), ct)); + Assert.Equal("node-a", (await store.GetAsync("job-1", ct))!.NodeId); + + // Once the renewed lease itself lapses, another node may steal the claim. time.Advance(TimeSpan.FromMinutes(11)); Assert.True(await store.TryClaimAsync("job-1", "node-b", TimeSpan.FromMinutes(5), ct)); Assert.Equal("node-b", (await store.GetAsync("job-1", ct))!.NodeId); diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index fba07b1cd..f28f496d3 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -84,6 +84,13 @@ public sealed record JobQuery public string? Name { get; init; } public JobStatus? Status { get; init; } public int Limit { get; init; } = 100; + + /// + /// When true, CRON occurrences (jobs with set) are excluded. The job + /// scheduler is the sole executor of occurrences, so the generic worker must not claim them — otherwise it would + /// run them without the per-definition retry/dead-letter accounting that lives in the scheduler. + /// + public bool ExcludeOccurrences { get; init; } } public sealed record ScheduledDispatchState @@ -339,6 +346,9 @@ public Task> QueryAsync(JobQuery query, CancellationToke if (query.Status is { } status) results = results.Where(s => s.Status == status); + if (query.ExcludeOccurrences) + results = results.Where(s => s.ScheduledForUtc is null); + return Task.FromResult>(results .OrderByDescending(s => s.LastUpdatedUtc) .Take(Math.Max(1, query.Limit)) @@ -716,7 +726,9 @@ public async Task RunQueuedAsync(int limit = 100, CancellationToken cancell var queued = await _store.QueryAsync(new JobQuery { Status = JobStatus.Queued, - Limit = limit + Limit = limit, + // The scheduler owns CRON occurrences (retry/dead-letter accounting); the generic worker must skip them. + ExcludeOccurrences = true }, cancellationToken).ConfigureAwait(false); int completed = 0; @@ -752,6 +764,11 @@ public async Task RecoverStaleAsync(int maxAttempts, int limit = 100, Cance // node and its lease is still expired, so a worker that renewed between the scan and here is not yanked out // from under itself (no double-run). Attempts are incremented per run, so a job that keeps crashing is // dead-lettered once it has consumed its attempt budget instead of being re-queued forever. + // + // Budget semantics for ad-hoc (IJobClient) jobs: `maxAttempts` is the TOTAL number of attempts, so + // dead-letter at Attempt >= maxAttempts. (CRON occurrences use a different knob — ScheduledJobDefinition + // .MaxRetries, the number of retries AFTER the first run, i.e. total runs = MaxRetries + 1 — and are + // excluded from this path via GetExpiredProcessingAsync; the scheduler owns their recovery.) bool transitioned = state.Attempt >= maxAttempts ? await _store.TryReclaimExpiredAsync(state.JobId, now, state.NodeId, JobStatus.DeadLettered, new JobStatePatch { diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 079806955..cbc700823 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -230,7 +230,14 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = { if (!await TryPrepareOccurrenceForRunAsync(jobId, definition, utcNow, cancellationToken).ConfigureAwait(false)) { - await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); + // Retire (don't reschedule) the dispatch when the occurrence has reached a terminal state — e.g. it + // was dead-lettered in TryPrepareOccurrenceForRunAsync, or a worker completed it but crashed before + // CompleteDispatchAsync. Otherwise a terminal occurrence's dispatch would be re-claimed forever. + var pending = await _store.GetAsync(jobId, cancellationToken).ConfigureAwait(false); + if (pending is { Status: JobStatus.Completed or JobStatus.Cancelled or JobStatus.DeadLettered }) + await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); + else + await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); continue; } @@ -339,7 +346,21 @@ private static TimeSpan GetRetryBackoff(ScheduledJobDefinition definition, int a private async Task HasActiveOccurrenceAsync(string name, string scopeKey, CancellationToken cancellationToken) { var states = await _store.QueryAsync(new JobQuery { Name = name, Limit = 1000 }, cancellationToken).ConfigureAwait(false); - return states.Any(s => s.JobId.EndsWith($":{scopeKey}", StringComparison.Ordinal) && s.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing); + return states.Any(s => OccurrenceMatchesScope(s.JobId, name, scopeKey) && s.Status is JobStatus.Queued or JobStatus.Scheduled or JobStatus.Processing); + } + + // Exact scope match, not a JobId suffix test: an occurrence id is "{name}:{14-digit-timestamp}:{scopeKey}", and a + // scope key (a node id) can itself contain ':' (NodeIdentity.Current is "{machine}:{pid}:{token}"), so a naive + // EndsWith(":{scopeKey}") would let one node's occurrence count as another's. The query is already filtered to this + // name, so strip the literal "{name}:" prefix and the fixed-width timestamp, then compare the remainder exactly. + private static bool OccurrenceMatchesScope(string jobId, string name, string scopeKey) + { + string prefix = $"{name}:"; + if (!jobId.StartsWith(prefix, StringComparison.Ordinal)) + return false; + + var rest = jobId.AsSpan(prefix.Length); + return rest.Length >= 15 && rest[14] == ':' && rest[15..].SequenceEqual(scopeKey); } private string GetScopeKey(ScheduledJobDefinition definition) diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 774625448..357840890 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -969,7 +969,10 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c if (_runtimeStore is null) throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); - int nextAttempt = _entry.DeliveryCount + 1; + // Advance from the reconciled attempt count, not the raw transport DeliveryCount: the re-send produces a new + // transport message whose native DeliveryCount resets to 1, so basing the next attempt on DeliveryCount would + // pin it at 2 and redeliver forever. Attempts already takes the max of DeliveryCount and the carried header. + int nextAttempt = Attempts + 1; var headers = _entry.Headers.ToBuilder() .Set(KnownHeaders.Attempts, nextAttempt.ToString(CultureInfo.InvariantCulture)) .Build(); diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index 5965c8bf5..dbcdf9105 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -308,6 +308,79 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState Assert.Equal(1, probe.RunCount); } + [Fact] + public async Task RunQueuedAsync_DoesNotClaimScheduledOccurrencesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobSchedulerProbe(); + await using var serviceProvider = new ServiceCollection().AddSingleton(probe).BuildServiceProvider(); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + + // A CRON occurrence sitting in Queued (the scheduler transitioned it Scheduled->Queued) must NOT be claimed by + // the generic worker — only the scheduler runs occurrences, with its own retry/dead-letter accounting. + await store.CreateIfAbsentAsync(new JobState + { + JobId = "nightly:20260101000000:global", + Name = "nightly", + JobType = typeof(ScheduledProbeJob).FullName, + Status = JobStatus.Queued, + ScheduledForUtc = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero) + }, cancellationToken); + + Assert.Equal(0, await worker.RunQueuedAsync(cancellationToken: cancellationToken)); + Assert.Equal(0, probe.RunCount); + Assert.Equal(JobStatus.Queued, (await store.GetAsync("nightly:20260101000000:global", cancellationToken))!.Status); + } + + [Fact] + public async Task RunDueOccurrencesAsync_WhenOccurrenceIsTerminal_RetiresDispatchInsteadOfReschedulingAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + var processor = CreateProcessor(scheduler, store, "node-a"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + const string jobId = "nightly:20260101000000:global"; + + await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "nightly", Cron = "* * * * *", JobType = typeof(ScheduledProbeJob) }, cancellationToken); + // A worker completed the occurrence but crashed before retiring its dispatch: a terminal job with a live dispatch. + await store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = "nightly", Status = JobStatus.Completed, ScheduledForUtc = now.AddSeconds(-30) }, cancellationToken); + await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, Destination = "nightly", Body = Array.Empty(), DueUtc = now, JobId = jobId }, cancellationToken); + + await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken); + + // The dispatch for a terminal occurrence must be retired, not rescheduled +1min and re-claimed forever. + Assert.Empty(await store.ClaimDueDispatchesAsync(now.AddMinutes(5), 10, "node-b", TimeSpan.FromMinutes(5), cancellationToken)); + } + + [Fact] + public async Task EnqueueDueOccurrencesAsync_PerNodeScope_WithDelimiterInNodeId_DoesNotCrossMatchAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var scheduler = new InMemoryJobScheduler(); + var store = new InMemoryJobRuntimeStore(); + // Node ids that are suffix-confusable under a naive EndsWith(":{scope}") check — the default NodeIdentity contains ':'. + var nodeXB = CreateProcessor(scheduler, store, "x:b"); + var nodeB = CreateProcessor(scheduler, store, "b"); + var now = new DateTimeOffset(2026, 1, 1, 0, 0, 30, TimeSpan.Zero); + + await scheduler.ScheduleAsync(new ScheduledJobDefinition + { + Name = "per-node", + Cron = "* * * * *", + JobType = typeof(ScheduledProbeJob), + Scope = ScheduledJobScope.PerNode // default Overlap = SkipIfRunning, which runs the active-occurrence check + }, cancellationToken); + + Assert.Single(await nodeXB.EnqueueDueOccurrencesAsync(now, cancellationToken)); // creates "per-node:...:x:b" + // node "b" must still materialize its own occurrence; node "x:b"'s occurrence must not be mistaken for node "b"'s. + Assert.Single(await nodeB.EnqueueDueOccurrencesAsync(now, cancellationToken)); // creates "per-node:...:b" + + var states = await store.QueryAsync(new JobQuery { Name = "per-node", Limit = 100 }, cancellationToken); + Assert.Equal(2, states.Count); + } + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId, IMessageTransport? transport = null) { var serviceProvider = new ServiceCollection() diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 41c5879f7..c735a265d 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -334,6 +334,41 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough } + [Fact] + public async Task RejectAsync_RuntimeStoreRedelivery_AdvancesAttemptCountEachCycleAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + // BasicQueueTransport resets DeliveryCount to 1 on every (re)send and has no native redelivery delay, so each + // delayed reject re-schedules through the runtime store. The reconciled Attempts must keep advancing across + // redeliveries (1 -> 2 -> 3); a regression that bases the next attempt on the reset DeliveryCount would pin it at + // 2 and redeliver forever (never reaching MaxAttempts / dead-letter). + await using var transport = new BasicQueueTransport(); + await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); + var processor = CreateDispatchProcessor(store, transport); + var now = DateTimeOffset.UtcNow; + + await queue.EnqueueAsync(new PreviewWorkItem { Data = "loop" }, cancellationToken: cancellationToken); + + for (int expectedAttempt = 1; expectedAttempt <= 3; expectedAttempt++) + { + var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + Assert.NotNull(received); + Assert.Equal(expectedAttempt, received.Attempts); + Assert.Equal("loop", received.Message.Data); + + if (expectedAttempt < 3) + { + await received.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.FromMinutes(1) }, cancellationToken); + Assert.Equal(1, await processor.RunDueOccurrencesAsync(now.AddMinutes(expectedAttempt * 2), cancellationToken: cancellationToken)); + } + else + { + await received.CompleteAsync(cancellationToken); + } + } + } + [Fact] public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync() { From 6fae268406bdde927dd2d139425e1ab118b6d9f8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 00:12:35 -0500 Subject: [PATCH 21/57] Fix review #6: auto-register the job runtime pump with the store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A runtime store is inert without something draining it, yet UseInMemoryRuntime()/UseRuntimeStore() registered the store + processor but not the pump (it lived only behind a separate, undiscoverable AddJobRuntimeService() in the hosting package). The result: IJobClient jobs sat Queued forever and — worse — the messaging delayed-delivery fallback (which the same pump drains) silently disappeared. Move a JobRuntimePumpService (BackgroundService) into core and register it from RegisterJobServices, so configuring a store makes a hosted process run jobs and drain delayed messaging with no extra wiring; in a non-hosted process the IHostedService is simply never started. The hosting AddJobRuntimeService now carries its options onto the core pump instead of starting a second pump. This adds Microsoft.Extensions.Hosting.Abstractions (BackgroundService/ IHostedService) to the core package — a small, ubiquitous abstractions dependency, justified because the durable runtime is unusable without a pump. Flagging it for review; trivially revertible if you'd rather keep the pump exclusively in the hosting package. Tests: auto-registration + end-to-end run via the resolved pump; full jobs/queue/messaging regression green (201 pass, 4 pre-existing skips). Co-Authored-By: Claude Opus 4.8 --- .../Jobs/JobHostExtensions.cs | 16 +++- src/Foundatio/Foundatio.csproj | 3 + src/Foundatio/FoundatioServicesExtensions.cs | 10 ++ src/Foundatio/Jobs/JobRuntimePumpService.cs | 92 +++++++++++++++++++ .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 35 +++++++ 5 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/Foundatio/Jobs/JobRuntimePumpService.cs diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index fc709bbfb..aed419b0e 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -216,8 +216,22 @@ public static IServiceCollection AddJobRuntimeService(this IServiceCollection se { var options = new JobRuntimeServiceOptions(); configure?.Invoke(options); - services.AddSingleton(options); + // The core builder (AddFoundatio().Jobs.UseRuntimeStore()/UseInMemoryRuntime()) already registers the runtime + // pump when a store is configured. In that case carry these options onto the core pump rather than starting a + // second pump for the same store. This method then only needs to be called to tune the cadence/batch size. + if (services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimePumpService))) + { + services.AddSingleton(new JobRuntimePumpOptions + { + PollInterval = options.PollInterval, + BatchSize = options.BatchSize, + MaxJobAttempts = options.MaxJobAttempts + }); + return services; + } + + services.AddSingleton(options); if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimeService))) services.AddSingleton(); diff --git a/src/Foundatio/Foundatio.csproj b/src/Foundatio/Foundatio.csproj index b1ba08469..3fb6df955 100644 --- a/src/Foundatio/Foundatio.csproj +++ b/src/Foundatio/Foundatio.csproj @@ -9,5 +9,8 @@ + + diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 184b45574..9ce5779a2 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Foundatio.Caching; using Foundatio.Extensions; using Foundatio.Jobs; @@ -9,6 +10,7 @@ using Foundatio.Serializer; using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Foundatio; @@ -472,6 +474,14 @@ private void RegisterJobServices() sp.GetService(), transport: sp.GetService(), jobTypes: sp.GetRequiredService())); + + // A runtime store is inert without something draining it, so register the pump alongside the store: in a + // hosted process it runs jobs and the messaging delayed-delivery fallback automatically (no separate + // AddJobRuntimeService call); in a non-hosted process the IHostedService is simply never started. Guarded so + // repeated UseRuntimeStore/UseInMemoryRuntime calls don't stack multiple pumps. Options default unless + // AddJobRuntimeService (or a registered JobRuntimePumpOptions) overrides them. + if (!_services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimePumpService))) + _services.AddSingleton(); } } diff --git a/src/Foundatio/Jobs/JobRuntimePumpService.cs b/src/Foundatio/Jobs/JobRuntimePumpService.cs new file mode 100644 index 000000000..abd2190b7 --- /dev/null +++ b/src/Foundatio/Jobs/JobRuntimePumpService.cs @@ -0,0 +1,92 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Jobs; + +/// Cadence and batch size for the durable job-runtime pump. +public class JobRuntimePumpOptions +{ + /// How often the pump materializes CRON occurrences, dispatches due work, and runs queued jobs. Default 1s. + public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1); + + /// Maximum number of due dispatches and queued jobs claimed per iteration. Default 100. + public int BatchSize { get; set; } = 100; + + /// Maximum processing attempts for an ad-hoc job before a stale (lease-expired) instance is dead-lettered. Default 3. + public int MaxJobAttempts { get; set; } = 3; +} + +/// +/// Drives the durable job runtime (): materializes CRON occurrences, dispatches +/// delayed/scheduled work (including the messaging delayed-delivery fallback), recovers stale occurrences, and runs +/// jobs submitted via . Registered automatically whenever a runtime store is configured +/// (AddFoundatio().Jobs.UseInMemoryRuntime() / UseRuntimeStore()) so a configured store can never +/// silently accumulate work that nothing drains. In a non-hosted process (no generic host) it is simply never started. +/// +public class JobRuntimePumpService : BackgroundService +{ + private readonly JobScheduleProcessor _processor; + private readonly IJobWorker _worker; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly JobRuntimePumpOptions _options; + + public JobRuntimePumpService(JobScheduleProcessor processor, IJobWorker worker, TimeProvider? timeProvider = null, ILoggerFactory? loggerFactory = null, JobRuntimePumpOptions? options = null) + { + _processor = processor ?? throw new ArgumentNullException(nameof(processor)); + _worker = worker ?? throw new ArgumentNullException(nameof(worker)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _options = options ?? new JobRuntimePumpOptions(); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var now = _timeProvider.GetUtcNow(); + + // Materialize CRON occurrences due within the misfire window (deduped, idempotent). + await _processor.EnqueueDueOccurrencesAsync(now, stoppingToken).AnyContext(); + + // Claim and run due dispatches: CRON occurrences plus delayed queue/pub-sub messages, recovering + // occurrences whose processing lease expired and applying retry/dead-letter. + await _processor.RunDueOccurrencesAsync(now, _options.BatchSize, lease: null, stoppingToken).AnyContext(); + + // Recover ad-hoc (non-CRON) jobs whose processing lease expired (a worker crash mid-run). + await _worker.RecoverStaleAsync(_options.MaxJobAttempts, _options.BatchSize, stoppingToken).AnyContext(); + + // Run jobs submitted via IJobClient sitting in the Queued state. + await _worker.RunQueuedAsync(_options.BatchSize, stoppingToken).AnyContext(); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error pumping job runtime: {Message}", ex.Message); + } + + try + { + await _timeProvider.Delay(_options.PollInterval, stoppingToken).AnyContext(); + } + catch (OperationCanceledException) + { + break; + } + } + + _logger.LogInformation("Job runtime pump stopped"); + } +} diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index dbcdf9105..342336058 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -1,9 +1,12 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Foundatio; using Foundatio.Jobs; using Foundatio.Messaging; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Xunit; namespace Foundatio.Tests.Jobs; @@ -381,6 +384,38 @@ await scheduler.ScheduleAsync(new ScheduledJobDefinition Assert.Equal(2, states.Count); } + [Fact] + public async Task AddFoundatio_WithRuntimeStore_AutoRegistersAndRunsPumpAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var probe = new JobSchedulerProbe(); + var services = new ServiceCollection().AddSingleton(probe); + var foundatio = services.AddFoundatio(); + foundatio.Jobs.UseInMemoryRuntime(); + foundatio.Jobs.Register("probe"); + await using var provider = services.BuildServiceProvider(); + + // Configuring a runtime store auto-registers the pump — no separate AddJobRuntimeService — so a hosted process + // runs IJobClient-submitted jobs (and drains delayed messaging) without extra wiring. + var pump = Assert.Single(provider.GetServices().OfType()); + await pump.StartAsync(cancellationToken); + try + { + var handle = await provider.GetRequiredService().EnqueueAsync(cancellationToken: cancellationToken); + + JobState? state = null; + for (int i = 0; i < 100 && (state = await handle.GetStateAsync(cancellationToken))?.Status != JobStatus.Completed; i++) + await Task.Delay(50, cancellationToken); + + Assert.Equal(JobStatus.Completed, state?.Status); + Assert.Equal(1, probe.RunCount); + } + finally + { + await pump.StopAsync(cancellationToken); + } + } + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId, IMessageTransport? transport = null) { var serviceProvider = new ServiceCollection() From eb1674c69a9395ccfdea475a4115823fb05f9044 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 00:20:36 -0500 Subject: [PATCH 22/57] Fix review P0 #2: centralize subscription addressing, unbreak AWS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pub/sub subscription address ("{topic}/{subscription}") was smuggled through a single opaque string with no shared parser, so each transport re-derived the convention: Redis split on '/', but AWS fed the composite straight to SQS CreateQueue/GetQueueUrl — and '/' is illegal in an SQS queue name, so every AWS pub/sub subscribe/receive/complete failed (only via the PubSub facade; the conformance fan-out uses bare names, so it never surfaced). - New SubscriptionAddress helper (Format/TryParse) centralizes the convention and documents it: the source/Destination string is an OPAQUE provider-agnostic key (it contains '/'), to be mapped to native resources at EnsureAsync (which also supplies the topic via DestinationDeclaration.Source) — not assumed to be a legal broker name. PubSub and the Redis transport now both use it instead of re-deriving. - AWS encodes any non-legal logical name (i.e. a subscription composite) into a legal, deterministic, collision-free SQS name (sanitize + stable hash suffix); legal names pass through unchanged. Fixed the comment that asserted the broken "already conforms" invariant. Tests: SubscriptionAddress round-trip (core); the Redis PubSub-facade fan-out integration test exercises the composite end-to-end. The AWS path can't be exercised without LocalStack here; the encoding is correct by construction. Messaging regression green (68 pass, 1 pre-existing skip); Redis suite 22/22. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio.Aws/AwsMessageTransport.cs | 50 +++++++++++++++++-- .../Messaging/RedisStreamsMessageTransport.cs | 8 +-- src/Foundatio/Messaging/PubSub.cs | 9 +--- .../Messaging/SubscriptionAddress.cs | 43 ++++++++++++++++ .../InMemoryMessageTransportTests.cs | 16 ++++++ 5 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 src/Foundatio/Messaging/SubscriptionAddress.cs diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index 653a76030..4ed5f865f 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Concurrent; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Threading; @@ -343,9 +344,52 @@ private async Task ResolveTopicArnAsync(string name, CancellationToken c return response.TopicArn; } - // SQS queue names and SNS topic names allow alphanumerics, hyphens and underscores; the logical destination name - // already conforms, so we only prepend the configured prefix. - private string ResourceName(string logicalName) => _options.ResourcePrefix + logicalName; + // SQS queue / SNS topic names allow only [A-Za-z0-9_-] (max 80 chars). Most logical names already conform, but a + // pub/sub subscription's destination is the opaque "topic/subscription" key (see SubscriptionAddress) which + // contains '/'. Encode any illegal name deterministically and collision-free — sanitize, then append a short + // stable hash of the original — so EnsureAsync/ReceiveAsync/CompleteAsync all resolve the same queue from the same + // logical name. Legal names are returned unchanged (no behavior change for plain queues/topics). + private string ResourceName(string logicalName) => EncodeResourceName(_options.ResourcePrefix, logicalName); + + private static string EncodeResourceName(string prefix, string logicalName) + { + string candidate = prefix + logicalName; + if (IsResourceNameLegal(candidate)) + return candidate; + + string suffix = "-" + StableHash(candidate); + string sanitized = SanitizeResourceName(candidate); + if (sanitized.Length > 80 - suffix.Length) + sanitized = sanitized[..(80 - suffix.Length)]; + return sanitized + suffix; + } + + private static bool IsResourceNameLegal(string name) + { + if (name.Length is 0 or > 80) + return false; + foreach (char c in name) + { + if (!(Char.IsAsciiLetterOrDigit(c) || c is '-' or '_')) + return false; + } + + return true; + } + + private static string SanitizeResourceName(string name) + { + var builder = new StringBuilder(name.Length); + foreach (char c in name) + builder.Append(Char.IsAsciiLetterOrDigit(c) || c is '-' or '_' ? c : '-'); + return builder.ToString(); + } + + private static string StableHash(string value) + { + byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); // 8 hex chars + } private async Task GetQueueArnAsync(string queueUrl, CancellationToken ct) { diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 2c1d2bc25..6f1e0aa7d 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -348,10 +348,10 @@ private ResolvedSource Resolve(string source) if (_sources.TryGetValue(source, out var registered)) return registered; - // PubSub facade sources are "topic/subscription"; a bare name is a queue on the default group. - int slash = source.IndexOf('/'); - return slash > 0 - ? new ResolvedSource(StreamKey(source[..slash]), source[(slash + 1)..], "$") + // PubSub facade sources are "topic/subscription" (a consumer group on the topic stream); a bare name is a queue + // on the default group. Parse via the shared convention rather than re-deriving the split. + return SubscriptionAddress.TryParse(source, out string topic, out string subscription) + ? new ResolvedSource(StreamKey(topic), subscription, "$") : new ResolvedSource(StreamKey(source), _options.DefaultConsumerGroup, "0"); } diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 1b730b98d..6d2e663e2 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -162,7 +162,7 @@ private ListenerConfig BuildConfig(Type routeType, PubSubSubscriptionOptions opt Subscription = subscription, // The transport source is the topic-qualified subscription destination, not the bare subscription name, so // the same subscription identity used on two topics resolves to two distinct sources (and isolates). - Source = SubscriptionDestination(topic, subscription), + Source = SubscriptionAddress.Format(topic, subscription), Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{routeType.FullName ?? routeType.Name}", MessageType = routeType, AckMode = options.AckMode, @@ -185,13 +185,6 @@ private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken ca ], cancellationToken); } - // The topic-qualified subscription destination. The topic is part of the identity so the same subscription name on - // two topics does not collide on one transport source. (A provider can map this to its native subscription address.) - private static string SubscriptionDestination(string topic, string subscription) - { - return $"{topic}/{subscription}"; - } - private string GetTopic(Type messageType, string? topic) { return _core.Router.ResolveRoute(new MessageRouteContext diff --git a/src/Foundatio/Messaging/SubscriptionAddress.cs b/src/Foundatio/Messaging/SubscriptionAddress.cs new file mode 100644 index 000000000..07d923b35 --- /dev/null +++ b/src/Foundatio/Messaging/SubscriptionAddress.cs @@ -0,0 +1,43 @@ +using System; + +namespace Foundatio.Messaging; + +/// +/// The single, shared convention for addressing a pub/sub subscription as one transport destination string: +/// "{topic}/{subscription}". The topic is part of the identity so the same subscription name used on two topics +/// resolves to two distinct sources. +/// +/// +/// The resulting string (the source passed to receive/subscribe and carried as ) +/// is an opaque provider-agnostic key: because it contains '/' a transport must NOT assume it is a legal +/// broker resource name (e.g. an SQS queue name). Map it to native resources during +/// — which also supplies the structured topic via +/// — and treat it as a dictionary key thereafter, or parse it with +/// . Topic and subscription names must not contain '/'. Centralizing the convention here +/// (rather than each provider re-deriving it) keeps providers interoperable. +/// +public static class SubscriptionAddress +{ + /// Formats the topic-qualified subscription destination key. + public static string Format(string topic, string subscription) => $"{topic}/{subscription}"; + + /// + /// Splits a destination produced by into its topic and subscription. Returns false for a bare + /// (non-subscription) destination, leaving = the whole input and empty. + /// + public static bool TryParse(string destination, out string topic, out string subscription) + { + ArgumentNullException.ThrowIfNull(destination); + int slash = destination.IndexOf('/'); + if (slash <= 0 || slash >= destination.Length - 1) + { + topic = destination; + subscription = ""; + return false; + } + + topic = destination[..slash]; + subscription = destination[(slash + 1)..]; + return true; + } +} diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs index fccdf77cf..80f6f550a 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -15,6 +15,22 @@ protected override IMessageTransport CreateTransport() return new InMemoryMessageTransport(); } + [Fact] + public void SubscriptionAddress_FormatsAndParsesTopicAndSubscription() + { + string destination = SubscriptionAddress.Format("orders", "sub-a"); + Assert.Equal("orders/sub-a", destination); + + Assert.True(SubscriptionAddress.TryParse(destination, out string topic, out string subscription)); + Assert.Equal("orders", topic); + Assert.Equal("sub-a", subscription); + + // A bare (non-subscription) destination is not a subscription address. + Assert.False(SubscriptionAddress.TryParse("orders", out string bareTopic, out string bareSubscription)); + Assert.Equal("orders", bareTopic); + Assert.Equal("", bareSubscription); + } + [Fact] public void MessageHeaders_AreImmutableAndCaseInsensitive() { From e20d2eb944eb31521314e6df5b8f6552ced5a9c0 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 00:27:34 -0500 Subject: [PATCH 23/57] Fix review #5 (send error model) + #7 (shared header codec) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5: SendResult/SendItemResult modeled per-item failure (Success/ErrorCode/ IsRetryable/AllSucceeded) but no transport ever populated it — they all throw on failure — so the error model was dead, ambiguous surface and the core's failure branch was unreachable. Made send throw-on-failure normative: SendItemResult is now just the accepted MessageId, SendResult is the accepted ids, and the dead fields/branches are removed (core, JobScheduler materialize, conformance, all transports). Documented that a multi-message send is not atomic. #7: AWS and Redis each hand-rolled byte-for-byte-identical header JSON codecs with a latent case-semantics disagreement. Added a canonical MessageHeaders.SerializeToJson/DeserializeFromJson in core (bakes in the case-insensitive contract); both transports now use it. Added a round-trip test asserting case-insensitivity survives the wire. Regression green: messaging+queue 148 pass (3 pre-existing skips), Redis 22/22, in-memory transport 17 (1 skip). Co-Authored-By: Claude Opus 4.8 --- src/Foundatio.Aws/AwsMessageTransport.cs | 22 +++------------ .../Messaging/RedisStreamsMessageTransport.cs | 21 +++------------ .../MessageTransportConformanceTests.cs | 3 +-- src/Foundatio/Jobs/JobScheduler.cs | 6 ++--- .../Messaging/InMemoryMessageTransport.cs | 6 +---- src/Foundatio/Messaging/MessageClientCore.cs | 27 +++++++------------ src/Foundatio/Messaging/MessageHeaders.cs | 24 +++++++++++++++++ src/Foundatio/Messaging/MessageTransport.cs | 13 ++++++--- .../RedisJobStoreIntegrationTests.cs | 2 +- .../InMemoryMessageTransportTests.cs | 17 ++++++++++++ .../Queue/BasicQueueTransport.cs | 2 +- .../Queue/MessageQueueTests.cs | 6 ++--- 12 files changed, 75 insertions(+), 74 deletions(-) diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index 4ed5f865f..eb6ded841 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -91,7 +91,7 @@ public async Task SendAsync(string destination, IReadOnlyList new SnsMessageAttributeValue { DataType = "String", StringValue = value }) }, ct).ConfigureAwait(false); - items.Add(new SendItemResult { MessageId = response.MessageId, Success = true }); + items.Add(new SendItemResult { MessageId = response.MessageId }); } return new SendResult { Items = items }; @@ -112,7 +112,7 @@ public async Task SendAsync(string destination, IReadOnlyList BuildAttributes(Messag { var attributes = new Dictionary(StringComparer.Ordinal) { - [HeadersAttributeName] = stringAttribute(EncodeHeaders(headers)), + [HeadersAttributeName] = stringAttribute(MessageHeaders.SerializeToJson(headers)), [EncodingAttributeName] = stringAttribute(encoding) }; @@ -501,21 +501,7 @@ private static MessageHeaders FromSqsAttributes(Dictionary(StringComparer.Ordinal); - foreach (var header in headers) - map[header.Key] = header.Value; - return JsonSerializer.Serialize(map); - } - - private static MessageHeaders DecodeHeaders(string json) - { - var map = JsonSerializer.Deserialize>(json); - return map is null ? MessageHeaders.Empty : MessageHeaders.Create(map); + return MessageHeaders.DeserializeFromJson(value.StringValue); } private IAmazonSQS CreateSqsClient() diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 6f1e0aa7d..5f1e14ad2 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -74,7 +74,7 @@ public async Task SendAsync(string destination, IReadOnlyList GetBody(StreamEntry entry) return ReadOnlyMemory.Empty; } - private static string EncodeHeaders(MessageHeaders headers) - { - var map = new Dictionary(StringComparer.Ordinal); - foreach (var header in headers) - map[header.Key] = header.Value; - return JsonSerializer.Serialize(map); - } - - private static MessageHeaders DecodeHeaders(string? json) - { - if (String.IsNullOrEmpty(json)) - return MessageHeaders.Empty; - var map = JsonSerializer.Deserialize>(json); - return map is null ? MessageHeaders.Empty : MessageHeaders.Create(map); - } // Stream ids are "-"; the timestamp half is the broker enqueue time. private static DateTimeOffset? ParseStreamIdTime(RedisValue id) diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 9257a56e6..05f8805a7 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -42,9 +42,8 @@ public virtual async Task CanSendAndReceiveBatchAsync() CreateMessage("two", ("tenant", "acme")) ], new TransportSendOptions(), TestCancellationToken); - Assert.True(result.AllSucceeded); + // Send is throw-on-failure, so reaching here means both messages were accepted; assert the accepted ids. Assert.Equal(2, result.Items.Count); - Assert.All(result.Items, item => Assert.True(item.Success)); var entries = await pull.ReceiveAsync("orders", new ReceiveRequest { diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index cbc700823..395dc4e64 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -284,7 +284,7 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat if (_transport is null) throw new InvalidOperationException("A message transport is required to materialize scheduled queue and pub/sub dispatches."); - var result = await _transport.SendAsync(dispatch.Destination, [ + await _transport.SendAsync(dispatch.Destination, [ new TransportMessage { MessageId = dispatch.DispatchId, @@ -293,9 +293,7 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat } ], dispatch.Options with { DeliverAt = null }, cancellationToken).ConfigureAwait(false); - if (!result.AllSucceeded) - throw new MessageBusException($"Unable to materialize scheduled dispatch \"{dispatch.DispatchId}\" to \"{dispatch.Destination}\"."); - + // SendAsync is throw-on-failure; reaching here means the dispatch was materialized, so retire it. await _store.CompleteDispatchAsync(dispatch.DispatchId, _nodeId, cancellationToken).ConfigureAwait(false); } diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 0a755d323..62c2c45b6 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -68,11 +68,7 @@ public Task SendAsync(string destination, IReadOnlyList SendAsync(ScheduledDispatchKind kind, Type messageType if (await TryScheduleAsync(kind, destination, [transportMessage], sendOptions, cancellationToken).AnyContext()) return messageId; + // Send is throw-on-failure: SendChunkedAsync propagates any transport error, so a returned result means the + // message was accepted. Fall back to the pre-assigned id if the transport reported none. var items = await SendChunkedAsync(destination, [transportMessage], sendOptions, cancellationToken).AnyContext(); - var item = items.Count > 0 ? items[0] : null; - if (item is null || !item.Success) - throw _exceptionFactory($"Unable to send message to \"{destination}\": {item?.ErrorCode ?? "unknown error"}", null); - - return item.MessageId ?? messageId; + return (items.Count > 0 ? items[0].MessageId : null) ?? messageId; } public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable messages, Type? declaredType, MessageEnvelopeOptions options, Func resolveDestination, Func? ensureDestination, CancellationToken cancellationToken) @@ -164,10 +162,9 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable if (await TryScheduleAsync(kind, group.Key, group.Value, sendOptions, cancellationToken).AnyContext()) continue; - var items = await SendChunkedAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); - int failed = items.Count(i => !i.Success); - if (failed > 0) - throw _exceptionFactory($"Unable to send {failed} of {items.Count} messages to \"{group.Key}\".", null); + // Send is throw-on-failure (SendChunkedAsync propagates any transport error); a returned result means all + // messages in this destination group were accepted. + await SendChunkedAsync(group.Key, group.Value, sendOptions, cancellationToken).AnyContext(); } } @@ -638,15 +635,9 @@ private async Task> SendChunkedAsync(string destin private static void RecordSent(string destination, IReadOnlyList items) { - int sent = 0; - for (int index = 0; index < items.Count; index++) - { - if (items[index].Success) - sent++; - } - - if (sent > 0) - MessagingInstruments.Sent.Add(sent, new KeyValuePair("destination", destination)); + // Every returned item was accepted (send is throw-on-failure). + if (items.Count > 0) + MessagingInstruments.Sent.Add(items.Count, new KeyValuePair("destination", destination)); } private ISupportsPull RequirePull() diff --git a/src/Foundatio/Messaging/MessageHeaders.cs b/src/Foundatio/Messaging/MessageHeaders.cs index 1ba9686b6..a5e8ae045 100644 --- a/src/Foundatio/Messaging/MessageHeaders.cs +++ b/src/Foundatio/Messaging/MessageHeaders.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Frozen; using System.Collections.Generic; +using System.Text.Json; namespace Foundatio.Messaging; @@ -41,6 +42,29 @@ public static MessageHeaders Create(IEnumerable> he : new MessageHeaders(values.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase)); } + /// + /// The canonical on-the-wire encoding for headers (a JSON object). Transports should use this rather than rolling + /// their own so the round-trip semantics — notably case-insensitive keys (see ) — are identical + /// and contractually guaranteed across providers. + /// + public static string SerializeToJson(MessageHeaders headers) + { + ArgumentNullException.ThrowIfNull(headers); + var map = new Dictionary(StringComparer.Ordinal); + foreach (var header in headers) + map[header.Key] = header.Value; + return JsonSerializer.Serialize(map); + } + + /// Reads headers from the canonical encoding produced by . + public static MessageHeaders DeserializeFromJson(string? json) + { + if (String.IsNullOrEmpty(json)) + return Empty; + var map = JsonSerializer.Deserialize>(json); + return map is null ? Empty : Create(map); + } + public bool ContainsKey(string key) { return _headers.ContainsKey(key); diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 49e1c900f..10a2e3ae3 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -105,16 +105,21 @@ public sealed record MessageDestinationStats public sealed record SendItemResult { + /// The broker-assigned id of the accepted message. public string? MessageId { get; init; } - public required bool Success { get; init; } - public string? ErrorCode { get; init; } - public bool IsRetryable { get; init; } } +/// +/// The result of a successful : the accepted messages' ids, in order. +/// +/// +/// Send is throw-on-failure: a transport throws for any failure rather than returning a failed item, so every item in +/// was accepted. A multi-message send is NOT atomic — if a later message fails, earlier messages +/// may already have been delivered before the exception propagates. +/// public sealed record SendResult { public required IReadOnlyList Items { get; init; } - public bool AllSucceeded => Items.All(i => i.Success); } /// diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index e02396bf3..763c28ef4 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -255,7 +255,7 @@ public Task SendAsync(string destination, IReadOnlyList("Message.Type", "order.created"), + new KeyValuePair("tenant", "acme") + ]); + + // The shared codec both transports use preserves the case-insensitive contract across the wire. + var roundTripped = MessageHeaders.DeserializeFromJson(MessageHeaders.SerializeToJson(headers)); + Assert.Equal("order.created", roundTripped["MESSAGE.TYPE"]); + Assert.Equal("acme", roundTripped["tenant"]); + + Assert.Empty(MessageHeaders.DeserializeFromJson(null)); + Assert.Empty(MessageHeaders.DeserializeFromJson("")); + } + [Fact] public void MessageHeaders_AreImmutableAndCaseInsensitive() { diff --git a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs index 4fd7ed484..1fa97aaf4 100644 --- a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs +++ b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs @@ -29,7 +29,7 @@ public Task SendAsync(string destination, IReadOnlyList SendAsync(string destination, IReadOnlyList SendAsync(string destination, IReadOnlyList SendAsync(string destination, IReadOnlyList Date: Tue, 30 Jun 2026 00:39:28 -0500 Subject: [PATCH 24/57] Fix review #9/#16/#17/#22/#23: pull-loop, capability enforcement, misc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #9: the pull loop released one slot per processed entry but only acquired `claimed` slots, so a transport that ignores MaxMessages and over-returns over-released the concurrency semaphore (SemaphoreFull Exception / cap breach). Now processes at most `claimed` entries; any over-returned extras are left unsettled and redeliver. - #16: enforce a transport-advertised MaxMessageBytes on send with a clear error instead of letting an opaque broker rejection surface mid-send (it was declared but never read). + test. - #17: Redis dead-letter now records the reason header only when there is a reason (never an empty value), matching the in-memory reference. - #22: the in-memory transport no longer uses DeduplicationId as the message id (it's a dedup hint, not an identity) — distinct messages sharing a DeduplicationId were getting the same id, breaking settlement. - #23: cancellation-token defaulting is now consistent across the capability interfaces (all `= default`, matching IMessageTransport). Deferred (flagged): #8 at-most-once ack-ordering (no at-most-once transport exists to implement/test against — that transport tier was deferred); #13 the two-IQueue-type collision (entangled with the legacy-removal/migration boundary we're holding); #24 making dead-letter- read entries explicitly un-settleable (documentation). Messaging+queue 150 pass (3 pre-existing skips); Redis 22/22. Co-Authored-By: Claude Opus 4.8 --- .../Messaging/RedisStreamsMessageTransport.cs | 6 +++- .../Messaging/InMemoryMessageTransport.cs | 4 ++- src/Foundatio/Messaging/MessageClientCore.cs | 29 +++++++++++++++---- src/Foundatio/Messaging/MessageTransport.cs | 22 +++++++------- .../Queue/MessageQueueTests.cs | 18 ++++++++++-- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 5f1e14ad2..c8e44aca3 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -204,7 +204,11 @@ public async Task DeadLetterAsync(TransportEntry entry, string? reason, Cancella ThrowIfDisposed(); var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); - var headers = entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason ?? "").Build(); + // Match the in-memory reference: record the reason header only when there's a reason (never an empty value). + var headerBuilder = entry.Headers.ToBuilder(); + if (!String.IsNullOrEmpty(reason)) + headerBuilder.Set(KnownHeaders.DeadLetterReason, reason); + var headers = headerBuilder.Build(); await _db.StreamAddAsync(DeadKey(r.StreamKey), BuildFields(entry.Id, entry.Body, headers), messageId: null, maxLength: _options.MaxStreamLength, useApproximateMaxLength: true).ConfigureAwait(false); diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 62c2c45b6..43964a409 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -64,7 +64,9 @@ public Task SendAsync(string destination, IReadOnlyList= 0) + + for (int index = 0; index < toProcess; index++) { - var task = ProcessAndReleaseSlotAsync(entry, onMessage, source, slots, cancellationToken); + var task = ProcessAndReleaseSlotAsync(entries[index], onMessage, source, slots, cancellationToken); if (!task.IsCompleted) { inFlight[task] = 0; @@ -612,8 +616,21 @@ private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out private async Task> SendChunkedAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { + var info = _transport as ITransportInfo; + + // Enforce a transport-declared maximum message size up front with a clear error, rather than letting an opaque + // broker rejection surface mid-send (the limit is advertised, so honor it). + if (info?.MaxMessageBytes is { } maxBytes) + { + foreach (var message in messages) + { + if (message.Body.Length > maxBytes) + throw _exceptionFactory($"Message of {message.Body.Length} bytes exceeds transport \"{_transport.GetType().Name}\" maximum of {maxBytes} bytes for destination \"{destination}\".", null); + } + } + // Respect a transport-declared maximum batch size by splitting oversized sends into chunks. - int? maxBatchSize = (_transport as ITransportInfo)?.MaxBatchSize; + int? maxBatchSize = info?.MaxBatchSize; if (maxBatchSize is not { } limit || limit <= 0 || messages.Count <= limit) { var result = await _transport.SendAsync(destination, messages, options, cancellationToken).AnyContext(); diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 10a2e3ae3..1ba7fe621 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -171,12 +171,12 @@ public interface IMessageTransport : IAsyncDisposable public interface ISupportsPull : IMessageTransport { - Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct); + Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct = default); } public interface ISupportsPush : IMessageTransport { - Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct); + Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct = default); } public interface ISupportsRedeliveryDelay : IMessageTransport @@ -186,21 +186,21 @@ public interface ISupportsRedeliveryDelay : IMessageTransport // fallback instead of being silently clamped by the broker. TimeSpan? MaxRedeliveryDelay { get; } - Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct); + Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct = default); } public interface ISupportsDeadLetter : IMessageTransport { - Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct); + Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct = default); // Reads dead-lettered entries for a destination so callers can inspect raw payloads (including poison messages // that never deserialized) and the dead-letter reason header. Read entries are removed from the dead-letter store. - Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct); + Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct = default); } public interface ISupportsLockRenewal : IMessageTransport { - Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct); + Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct = default); } public interface ISupportsVisibilityTimeout : IMessageTransport @@ -210,12 +210,12 @@ public interface ISupportsVisibilityTimeout : IMessageTransport // unsatisfiable rather than relying on a silently clamped value. TimeSpan? MaxVisibilityTimeout { get; } - Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct); + Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default); } public interface ISupportsStats : IMessageTransport { - Task GetStatsAsync(string destination, CancellationToken ct); + Task GetStatsAsync(string destination, CancellationToken ct = default); } public interface ISupportsPriority : IMessageTransport { } @@ -232,9 +232,9 @@ public interface ISupportsExpiration : IMessageTransport { } public interface ISupportsProvisioning : IMessageTransport { - Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct); - Task DeleteAsync(string name, CancellationToken ct); - Task ExistsAsync(string name, CancellationToken ct); + Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct = default); + Task DeleteAsync(string name, CancellationToken ct = default); + Task ExistsAsync(string name, CancellationToken ct = default); } public interface IPushSubscription : IAsyncDisposable diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index dd8d79f31..7b0866699 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -334,6 +334,19 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough } + [Fact] + public async Task EnqueueAsync_ExceedingTransportMaxMessageBytes_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + // The transport advertises an 8-byte maximum; the core must enforce it up front with a clear error rather than + // let an opaque broker rejection surface mid-send. + await using var transport = new BatchLimitTransport(maxBatchSize: 10, maxMessageBytes: 8); + await using var queue = new MessageQueue(transport); + + await Assert.ThrowsAsync(async () => + await queue.EnqueueAsync(new PreviewWorkItem { Data = "a payload well over eight bytes" }, cancellationToken: cancellationToken)); + } + [Fact] public async Task RejectAsync_RuntimeStoreRedelivery_AdvancesAttemptCountEachCycleAsync() { @@ -816,9 +829,10 @@ private sealed class OtherWorkItem : IGroupedWorkItem private sealed class BatchLimitTransport : IMessageTransport, ITransportInfo { - public BatchLimitTransport(int maxBatchSize) + public BatchLimitTransport(int maxBatchSize, long? maxMessageBytes = null) { MaxBatchSize = maxBatchSize; + MaxMessageBytes = maxMessageBytes; } public List SendBatchSizes { get; } = new(); @@ -826,7 +840,7 @@ public BatchLimitTransport(int maxBatchSize) public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; public int? MaxBatchSize { get; } - public long? MaxMessageBytes => null; + public long? MaxMessageBytes { get; } public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { From 1bb4b9e1f237220288be9d467de1801d10fe68c8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 00:47:16 -0500 Subject: [PATCH 25/57] Fix review #11/#12: no-silent-skip conformance harness + coverage The conformance suites required each provider to re-declare every check as a [Fact] override, so a transport/store silently skipped any check the author forgot to override. Moved [Fact] onto the base virtual methods (MessageTransportConformanceTests, JobRuntimeStoreConformanceTests): concrete subclasses inherit and auto-run them, so a newly added base check runs against every provider with nothing to forget. Removed the redundant overrides from all subclasses (in-memory, AWS, Redis transport; in-memory + Redis store). Capability-unsupported checks self-skip via the base ISupports* gates; the one genuine provider opt-out (SQS DeleteMessage is idempotent, so the strict expired-receipt check) is now an explicit, visible Assert.Skip override rather than a silent omission. Added coverage (#12): a binary-body + case-insensitive-header round-trip conformance test that auto-runs on every provider (would catch a body/header codec divergence such as text-vs-base64). Verified discovery: in-memory transport 18 (15 conformance + 3 unit), Redis Streams 15 discovered / 11 run / 4 self-skip, AWS 16 discovered (skip without LocalStack), job-store conformance 6 on both stores. Co-Authored-By: Claude Opus 4.8 --- .../Jobs/JobRuntimeStoreConformanceTests.cs | 6 ++ .../MessageTransportConformanceTests.cs | 53 ++++++++++++ .../AwsMessageTransportConformanceTests.cs | 56 +++---------- .../RedisJobRuntimeStoreConformanceTests.cs | 22 +---- .../RedisStreamsTransportConformanceTests.cs | 30 ------- .../Jobs/InMemoryJobRuntimeStoreTests.cs | 21 +---- .../InMemoryMessageTransportTests.cs | 83 ------------------- 7 files changed, 75 insertions(+), 196 deletions(-) diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs index da54c6ba8..1869836d8 100644 --- a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -34,6 +34,7 @@ protected static JobState NewJob(TimeProvider time, string id, string name = "co return new JobState { JobId = id, Name = name, Status = status, CreatedUtc = now, LastUpdatedUtc = now }; } + [Fact] public virtual async Task JobLifecycle_RoundTripsAndTransitionsAsync() { var time = new FakeTimeProvider(); @@ -120,6 +121,7 @@ public virtual async Task JobLifecycle_RoundTripsAndTransitionsAsync() Assert.Null(await store.GetAsync("missing", ct)); } + [Fact] public virtual async Task Query_FiltersByNameStatusAndLimitAsync() { var time = new FakeTimeProvider(); @@ -163,6 +165,7 @@ public virtual async Task Query_FiltersByNameStatusAndLimitAsync() Assert.Equal(new HashSet { "a", "b" }, adHocAlpha.Select(j => j.JobId).ToHashSet()); // "d" excluded (occurrence) } + [Fact] public virtual async Task Leasing_ClaimRenewReleaseAndStealAsync() { var time = new FakeTimeProvider(); @@ -211,6 +214,7 @@ public virtual async Task Leasing_ClaimRenewReleaseAndStealAsync() Assert.Null(got.LeaseExpiresUtc); } + [Fact] public virtual async Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() { var time = new FakeTimeProvider(); @@ -255,6 +259,7 @@ JobState Processing(string id, DateTimeOffset lease, string node = "node-a", Dat Assert.Equal("node-b", (await store.GetAsync("reowned", ct))!.NodeId); } + [Fact] public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() { var time = new FakeTimeProvider(); @@ -343,6 +348,7 @@ public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() Assert.Equal("node-c", rescheduled.ClaimOwner); } + [Fact] public virtual async Task Concurrency_OptimisticControlElectsSingleWinnerAsync() { var time = new FakeTimeProvider(); diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 05f8805a7..2dd343bf2 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -24,6 +24,7 @@ protected virtual ValueTask CleanupTransportAsync(IMessageTransport transport) return transport.DisposeAsync(); } + [Fact] public virtual async Task CanSendAndReceiveBatchAsync() { var transport = CreateTransport(); @@ -84,6 +85,7 @@ public virtual async Task CanSendAndReceiveBatchAsync() } } + [Fact] public virtual async Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() { var transport = CreateTransport(); @@ -116,6 +118,7 @@ public virtual async Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsy } } + [Fact] public virtual async Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() { var transport = CreateTransport(); @@ -142,6 +145,7 @@ await Assert.ThrowsAsync(async () => } } + [Fact] public virtual async Task SubscribeAsync_DeliversPushMessagesAsync() { var transport = CreateTransport(); @@ -175,6 +179,7 @@ public virtual async Task SubscribeAsync_DeliversPushMessagesAsync() } } + [Fact] public virtual async Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() { var transport = CreateTransport(); @@ -209,6 +214,7 @@ await EnsureAsync(transport, } } + [Fact] public virtual async Task ReceiveAsync_RespectsPriorityAsync() { var transport = CreateTransport(); @@ -245,6 +251,7 @@ public virtual async Task ReceiveAsync_RespectsPriorityAsync() } } + [Fact] public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() { var transport = CreateTransport(); @@ -275,6 +282,7 @@ public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() } } + [Fact] public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() { var transport = CreateTransport(); @@ -302,6 +310,7 @@ public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() } } + [Fact] public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() { var transport = CreateTransport(); @@ -336,6 +345,7 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy await CleanupTransportIfNotNullAsync(transport); } } + [Fact] public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() { var transport = CreateTransport(); @@ -374,6 +384,7 @@ public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() } } + [Fact] public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() { var transport = CreateTransport(); @@ -413,6 +424,7 @@ public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayA } } + [Fact] public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() { var transport = CreateTransport(); @@ -451,6 +463,7 @@ public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() } } + [Fact] public virtual async Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() { var transport = CreateTransport(); @@ -479,6 +492,7 @@ public virtual async Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageA } } + [Fact] public virtual async Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() { var transport = CreateTransport(); @@ -508,6 +522,45 @@ public virtual async Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReason } } + [Fact] + public virtual async Task SendAsync_PreservesBinaryBodyAndCaseInsensitiveHeadersAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsPull pull) + { + Assert.Skip("Transport does not support pull receive (ISupportsPull)."); + return; + } + + try + { + await EnsureAsync(transport, new DestinationDeclaration { Name = "binary", Role = DestinationRole.Queue }); + + // Arbitrary, non-UTF-8 bytes with no content type must round-trip exactly (catches body-encoding bugs — a + // provider must not assume text), and header keys must round-trip case-insensitively across the wire. + byte[] payload = [0x00, 0x01, 0xFF, 0xFE, 0x10, 0x80, 0x7F]; + await transport.SendAsync("binary", [new TransportMessage + { + Body = payload, + Headers = MessageHeaders.Create([ + new KeyValuePair("tenant", "acme"), + new KeyValuePair("Mixed.Case", "x") + ]) + }], new TransportSendOptions(), TestCancellationToken); + + var entry = Assert.Single(await pull.ReceiveAsync("binary", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + Assert.Equal(payload, entry.Body.ToArray()); + Assert.Equal("acme", entry.Headers["tenant"]); + Assert.Equal("x", entry.Headers["MIXED.CASE"]); + + await transport.CompleteAsync(entry, TestCancellationToken); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + private async ValueTask CleanupTransportIfNotNullAsync(IMessageTransport? transport) { if (transport is not null) diff --git a/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs b/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs index 25af9746f..2f84d0227 100644 --- a/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs +++ b/tests/Foundatio.Aws.Tests/AwsMessageTransportConformanceTests.cs @@ -9,9 +9,10 @@ namespace Foundatio.Aws.Tests; /// /// Runs the shared transport conformance suite against AWS SQS/SNS. Set the environment variable /// FOUNDATIO_AWS_CONNECTION_STRING (e.g. serviceurl=http://localhost:4566;accesskey=test;secretkey=test;region=us-east-1 -/// for LocalStack, or real AWS credentials) to run; when it is not set every test is skipped. Capabilities SQS/SNS do -/// not support (priority, per-message expiration, push delivery, transport-native dead-letter) are skipped by the base -/// suite via their ISupports* capability checks. +/// for LocalStack, or real AWS credentials) to run; when it is not set every test is skipped. Inherits every base +/// [Fact], so a new conformance check runs against SQS/SNS automatically; capabilities SQS/SNS do not support +/// (priority, per-message expiration, push delivery, transport-native dead-letter) self-skip via their +/// ISupports* capability checks in the base suite. /// public class AwsMessageTransportConformanceTests : MessageTransportConformanceTests { @@ -32,45 +33,12 @@ public AwsMessageTransportConformanceTests(ITestOutputHelper output) : base(outp } [Fact] - public override Task CanSendAndReceiveBatchAsync() => base.CanSendAndReceiveBatchAsync(); - - [Fact] - public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() => base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); - - // CompleteAsync_WithExpiredReceipt is intentionally not run for SQS: DeleteMessage with a stale/used receipt - // handle is idempotent and does not raise — strict receipt validation is a transport-specific behavior, not part - // of the shared contract, so only transports that guarantee it (e.g. the in-memory reference) opt in. - - [Fact] - public override Task SubscribeAsync_DeliversPushMessagesAsync() => base.SubscribeAsync_DeliversPushMessagesAsync(); - - [Fact] - public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() => base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); - - [Fact] - public override Task ReceiveAsync_RespectsPriorityAsync() => base.ReceiveAsync_RespectsPriorityAsync(); - - [Fact] - public override Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() => base.SendAsync_WithDeliverAt_DelaysVisibilityAsync(); - - [Fact] - public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() => base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); - - [Fact] - public override Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() => base.ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync(); - - [Fact] - public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() => base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); - - [Fact] - public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() => base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); - - [Fact] - public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() => base.RenewLockAsync_ExtendsVisibilityWindowAsync(); - - [Fact] - public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() => base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); - - [Fact] - public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() => base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); + public override Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() + { + // Explicit, visible opt-out (not a silent skip): SQS DeleteMessage with a stale/used receipt handle is + // idempotent and does not raise. Strict receipt validation is transport-specific, not part of the shared + // contract, so SQS does not satisfy this check. + Assert.Skip("SQS DeleteMessage is idempotent for a stale receipt handle; strict receipt validation is not part of the shared contract."); + return Task.CompletedTask; + } } diff --git a/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs index 956c0c50f..c541f8f5c 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobRuntimeStoreConformanceTests.cs @@ -1,5 +1,4 @@ using System; -using System.Threading.Tasks; using Foundatio.Jobs; using Foundatio.Tests.Jobs; using Xunit; @@ -9,7 +8,8 @@ namespace Foundatio.Redis.Tests; /// /// Runs the shared conformance suite against a real Redis. Set /// FOUNDATIO_REDIS_CONNECTION_STRING (e.g. localhost:6399 for the bundled docker-compose Redis) to run; -/// when it is not set every test is skipped. Each test gets a unique key prefix so runs never collide. +/// when it is not set every test is skipped. Each test gets a unique key prefix so runs never collide. Inheriting the +/// base [Fact]s means a new conformance check automatically runs against Redis with no override to forget. /// public class RedisJobRuntimeStoreConformanceTests : JobRuntimeStoreConformanceTests { @@ -19,22 +19,4 @@ public RedisJobRuntimeStoreConformanceTests(ITestOutputHelper output) : base(out RedisTestConnection.Multiplexer is { } connection ? RedisTestConnection.CreateStore(connection, timeProvider) : null; // not configured -> the base suite skips every test - - [Fact] - public override Task JobLifecycle_RoundTripsAndTransitionsAsync() => base.JobLifecycle_RoundTripsAndTransitionsAsync(); - - [Fact] - public override Task Query_FiltersByNameStatusAndLimitAsync() => base.Query_FiltersByNameStatusAndLimitAsync(); - - [Fact] - public override Task Leasing_ClaimRenewReleaseAndStealAsync() => base.Leasing_ClaimRenewReleaseAndStealAsync(); - - [Fact] - public override Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() => base.StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync(); - - [Fact] - public override Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() => base.ScheduledDispatches_ClaimCompleteAndRescheduleAsync(); - - [Fact] - public override Task Concurrency_OptimisticControlElectsSingleWinnerAsync() => base.Concurrency_OptimisticControlElectsSingleWinnerAsync(); } diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs index 2fa849d88..c935c5b8b 100644 --- a/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportConformanceTests.cs @@ -1,5 +1,4 @@ using System; -using System.Threading.Tasks; using Foundatio.Messaging; using Foundatio.Tests.Messaging; using Xunit; @@ -28,33 +27,4 @@ public RedisStreamsTransportConformanceTests(ITestOutputHelper output) : base(ou }); } - [Fact] - public override Task CanSendAndReceiveBatchAsync() => base.CanSendAndReceiveBatchAsync(); - - [Fact] - public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() => base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); - - [Fact] - public override Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() => base.CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync(); - - [Fact] - public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() => base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); - - [Fact] - public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() => base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); - - [Fact] - public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() => base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); - - [Fact] - public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() => base.RenewLockAsync_ExtendsVisibilityWindowAsync(); - - [Fact] - public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() => base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); - - [Fact] - public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() => base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); - - [Fact] - public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() => base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); } diff --git a/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs b/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs index 7a596c9bf..6d6c1a212 100644 --- a/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs +++ b/tests/Foundatio.Tests/Jobs/InMemoryJobRuntimeStoreTests.cs @@ -1,31 +1,14 @@ using System; -using System.Threading.Tasks; using Foundatio.Jobs; using Xunit; namespace Foundatio.Tests.Jobs; +// Inherits every [Fact] from JobRuntimeStoreConformanceTests, so a new conformance check automatically runs here with +// no per-test override to forget. public class InMemoryJobRuntimeStoreTests : JobRuntimeStoreConformanceTests { public InMemoryJobRuntimeStoreTests(ITestOutputHelper output) : base(output) { } protected override IJobRuntimeStore CreateStore(TimeProvider timeProvider) => new InMemoryJobRuntimeStore(timeProvider); - - [Fact] - public override Task JobLifecycle_RoundTripsAndTransitionsAsync() => base.JobLifecycle_RoundTripsAndTransitionsAsync(); - - [Fact] - public override Task Query_FiltersByNameStatusAndLimitAsync() => base.Query_FiltersByNameStatusAndLimitAsync(); - - [Fact] - public override Task Leasing_ClaimRenewReleaseAndStealAsync() => base.Leasing_ClaimRenewReleaseAndStealAsync(); - - [Fact] - public override Task StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync() => base.StaleRecovery_ReclaimsExpiredButNotLiveOrCronAsync(); - - [Fact] - public override Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() => base.ScheduledDispatches_ClaimCompleteAndRescheduleAsync(); - - [Fact] - public override Task Concurrency_OptimisticControlElectsSingleWinnerAsync() => base.Concurrency_OptimisticControlElectsSingleWinnerAsync(); } diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs index 544133122..e63a064e1 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -72,87 +72,4 @@ public void MessageHeaders_AreImmutableAndCaseInsensitive() Assert.False(headers.ContainsKey("traceparent")); } - [Fact] - public override Task CanSendAndReceiveBatchAsync() - { - return base.CanSendAndReceiveBatchAsync(); - } - - [Fact] - public override Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync() - { - return base.AbandonAsync_RedeliversWithIncrementedDeliveryCountAsync(); - } - - [Fact] - public override Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync() - { - return base.CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredExceptionAsync(); - } - - [Fact] - public override Task SubscribeAsync_DeliversPushMessagesAsync() - { - return base.SubscribeAsync_DeliversPushMessagesAsync(); - } - - [Fact] - public override Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() - { - return base.SendAsync_ToTopic_FansOutToSubscriptionsAsync(); - } - - [Fact] - public override Task ReceiveAsync_RespectsPriorityAsync() - { - return base.ReceiveAsync_RespectsPriorityAsync(); - } - - [Fact] - public override Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() - { - return base.SendAsync_WithDeliverAt_DelaysVisibilityAsync(); - } - - [Fact] - public override Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() - { - return base.DeadLetterAsync_MovesEntryToDeadletterStatsAsync(); - } - - [Fact] - public override Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() - { - return base.ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync(); - } - - [Fact] - public override Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() - { - return base.ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync(); - } - - [Fact] - public override Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync() - { - return base.AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayAsync(); - } - - [Fact] - public override Task RenewLockAsync_ExtendsVisibilityWindowAsync() - { - return base.RenewLockAsync_ExtendsVisibilityWindowAsync(); - } - - [Fact] - public override Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync() - { - return base.CompetingConsumers_DoNotReceiveTheSameInFlightMessageAsync(); - } - - [Fact] - public override Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync() - { - return base.ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReasonAsync(); - } } From 5ec8f3a4e1b9eafc4843b69f4106930d4b03acdb Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 07:37:44 -0500 Subject: [PATCH 26/57] Add opt-out for the auto-registered job runtime pump Follow-up to the core-pump decision: the pump auto-starts under a host (so a configured store can't silently fail to drain), but advanced callers may want manual control (drive the processor/worker themselves, or run the pump on only some nodes). Adds JobRuntimePumpOptions.Enabled (default true); when false the hosted service is still registered but does nothing. Reachable from both wiring paths: - core: AddFoundatio().Jobs.ConfigureRuntimePump(o => o.Enabled = false) - hosting: AddJobRuntimeService(o => o.Enabled = false) (maps onto the core pump options) Safe default preserved; test asserts a disabled pump leaves a submitted job Queued and never runs it. Co-Authored-By: Claude Opus 4.8 --- .../Jobs/JobHostExtensions.cs | 1 + .../Jobs/JobRuntimeService.cs | 9 ++++++ src/Foundatio/FoundatioServicesExtensions.cs | 13 +++++++++ src/Foundatio/Jobs/JobRuntimePumpService.cs | 14 +++++++++ .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 29 +++++++++++++++++++ 5 files changed, 66 insertions(+) diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index aed419b0e..d42e388cd 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -224,6 +224,7 @@ public static IServiceCollection AddJobRuntimeService(this IServiceCollection se { services.AddSingleton(new JobRuntimePumpOptions { + Enabled = options.Enabled, PollInterval = options.PollInterval, BatchSize = options.BatchSize, MaxJobAttempts = options.MaxJobAttempts diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs index 0253c890a..8237ff0df 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobRuntimeService.cs @@ -15,6 +15,9 @@ namespace Foundatio.Extensions.Hosting.Jobs; /// public class JobRuntimeServiceOptions { + /// Whether the runtime pump runs. Default true; set false to take manual control of pumping. + public bool Enabled { get; set; } = true; + /// /// How often the runtime pump materializes CRON occurrences, dispatches due work, and runs queued jobs. /// Defaults to one second so sub-minute CRON schedules and short delays are honored. @@ -57,6 +60,12 @@ public JobRuntimeService(JobScheduleProcessor processor, IJobWorker worker, Time protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + if (!_options.Enabled) + { + _logger.LogInformation("Job runtime pump disabled (Enabled = false); not pumping the runtime store"); + return; + } + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); while (!stoppingToken.IsCancellationRequested) diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 9ce5779a2..d42d277ff 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -460,6 +460,19 @@ public FoundatioBuilder Register(string name) where TJob : IJob return _builder; } + /// + /// Tunes the auto-registered runtime pump (cadence, batch size, or + /// to opt out of automatic pumping and take manual control). + /// + public FoundatioBuilder ConfigureRuntimePump(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var options = new JobRuntimePumpOptions(); + configure(options); + _services.ReplaceSingleton(_ => options); + return _builder; + } + private void RegisterJobServices() { _services.ReplaceSingleton(sp => new JobTypeRegistry(sp.GetServices())); diff --git a/src/Foundatio/Jobs/JobRuntimePumpService.cs b/src/Foundatio/Jobs/JobRuntimePumpService.cs index abd2190b7..2ef580d22 100644 --- a/src/Foundatio/Jobs/JobRuntimePumpService.cs +++ b/src/Foundatio/Jobs/JobRuntimePumpService.cs @@ -11,6 +11,14 @@ namespace Foundatio.Jobs; /// Cadence and batch size for the durable job-runtime pump. public class JobRuntimePumpOptions { + /// + /// Whether the auto-registered runtime pump runs. Default true. Set false to take manual control of pumping (e.g. + /// drive / yourself, or run the pump on only some nodes); + /// the hosted service is then registered but does nothing. Configure via AddFoundatio().Jobs.ConfigureRuntimePump + /// or AddJobRuntimeService. + /// + public bool Enabled { get; set; } = true; + /// How often the pump materializes CRON occurrences, dispatches due work, and runs queued jobs. Default 1s. public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1); @@ -47,6 +55,12 @@ public JobRuntimePumpService(JobScheduleProcessor processor, IJobWorker worker, protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + if (!_options.Enabled) + { + _logger.LogInformation("Job runtime pump disabled (JobRuntimePumpOptions.Enabled = false); not pumping the runtime store"); + return; + } + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); while (!stoppingToken.IsCancellationRequested) diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index 342336058..db49c60d6 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -416,6 +416,35 @@ public async Task AddFoundatio_WithRuntimeStore_AutoRegistersAndRunsPumpAsync() } } + [Fact] + public async Task ConfigureRuntimePump_Disabled_DoesNotPumpAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var probe = new JobSchedulerProbe(); + var services = new ServiceCollection().AddSingleton(probe); + var foundatio = services.AddFoundatio(); + foundatio.Jobs.UseInMemoryRuntime(); + foundatio.Jobs.Register("probe"); + foundatio.Jobs.ConfigureRuntimePump(o => o.Enabled = false); // opt out of automatic pumping + await using var provider = services.BuildServiceProvider(); + + var pump = Assert.Single(provider.GetServices().OfType()); + await pump.StartAsync(cancellationToken); + try + { + var handle = await provider.GetRequiredService().EnqueueAsync(cancellationToken: cancellationToken); + + // With the pump disabled, the job is never claimed: it stays Queued and the job never runs. + await Task.Delay(300, cancellationToken); + Assert.Equal(JobStatus.Queued, (await handle.GetStateAsync(cancellationToken))!.Status); + Assert.Equal(0, probe.RunCount); + } + finally + { + await pump.StopAsync(cancellationToken); + } + } + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId, IMessageTransport? transport = null) { var serviceProvider = new ServiceCollection() From 09418c97dad917b0c5179f03c8e327af281142ab Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 30 Jun 2026 19:35:47 -0500 Subject: [PATCH 27/57] Verification follow-ups: single job pump regardless of order; test hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delta re-review of the review-fix series (adversarial, per-finding verified) came back clean — no regressions, no P0/P1. Two low-severity cleanups it surfaced: - Double-pump guard was order-dependent: calling AddJobRuntimeService before UseRuntimeStore registered both the hosting JobRuntimeService and the core JobRuntimePumpService against one store (each guard only checked for its own type; core can't see the hosting type across the package boundary). Collapsed to a single auto-registered pump: AddJobRuntimeService now only carries its options onto the core pump and never starts a second one, so any call order yields exactly one pump. (Not a correctness bug — claim/CAS prevented double-run — just redundant polling.) Added a hosting-first-order regression test. - Removed an orphaned `using System.Text.Json;` in the Redis Streams transport (left by the #7 header-codec move). Also hardened two poll-driven Redis integration tests (15s -> 30s): they flaked once under full-suite concurrency (many conformance tests hammer the same Redis); pass consistently with more headroom. Full matrix green: Foundatio.Tests 2001 (14 skip), Redis 27/27 (4 capability skips), AWS skips cleanly. Co-Authored-By: Claude Opus 4.8 --- .../Jobs/JobHostExtensions.cs | 27 +++++++------------ .../Messaging/RedisStreamsMessageTransport.cs | 1 - .../RedisStreamsTransportIntegrationTests.cs | 8 +++--- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 16 +++++++++++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index d42e388cd..9d1be3fd4 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -217,24 +217,17 @@ public static IServiceCollection AddJobRuntimeService(this IServiceCollection se var options = new JobRuntimeServiceOptions(); configure?.Invoke(options); - // The core builder (AddFoundatio().Jobs.UseRuntimeStore()/UseInMemoryRuntime()) already registers the runtime - // pump when a store is configured. In that case carry these options onto the core pump rather than starting a - // second pump for the same store. This method then only needs to be called to tune the cadence/batch size. - if (services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimePumpService))) + // Registering a runtime store (AddFoundatio().Jobs.UseRuntimeStore()/UseInMemoryRuntime()) is the precondition + // for this call, and that already auto-registers the single runtime pump (JobRuntimePumpService). So this method + // only carries options onto that pump — it never starts a second pump — which keeps a single pump regardless of + // the order AddJobRuntimeService and UseRuntimeStore are called in. + services.AddSingleton(new JobRuntimePumpOptions { - services.AddSingleton(new JobRuntimePumpOptions - { - Enabled = options.Enabled, - PollInterval = options.PollInterval, - BatchSize = options.BatchSize, - MaxJobAttempts = options.MaxJobAttempts - }); - return services; - } - - services.AddSingleton(options); - if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(JobRuntimeService))) - services.AddSingleton(); + Enabled = options.Enabled, + PollInterval = options.PollInterval, + BatchSize = options.BatchSize, + MaxJobAttempts = options.MaxJobAttempts + }); return services; } diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index c8e44aca3..927a85ea7 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -2,7 +2,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; using StackExchange.Redis; diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs index e4809ec52..aaa801c46 100644 --- a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs @@ -99,7 +99,7 @@ public async Task Core_RetriesFailedHandler_ThenDeadLettersAfterMaxAttemptsAsync await queue.EnqueueAsync(new RetryItem { Data = "retry" }, cancellationToken: ct); await queue.EnqueueAsync(new PoisonItem { Data = "poison" }, cancellationToken: ct); - await succeeded.Task.WaitAsync(TimeSpan.FromSeconds(15), ct); + await succeeded.Task.WaitAsync(TimeSpan.FromSeconds(30), ct); Assert.Equal(2, Volatile.Read(ref retryAttempts)); // The poison message lands in the dead-letter stream after exhausting its 2 attempts. @@ -148,8 +148,10 @@ public async Task PubSub_PublishToTopic_FansOutToEverySubscriptionAsync() await pubsub.PublishAsync(new FanItem { Data = "broadcast" }, cancellationToken: ct); - // Each named subscription is its own consumer group, so both receive an independent copy. - await Task.WhenAll(receivedByA.Task, receivedByB.Task).WaitAsync(TimeSpan.FromSeconds(15), ct); + // Each named subscription is its own consumer group, so both receive an independent copy. Delivery is + // poll-driven across two subscriptions (the core pull-fallback loop), so allow generous headroom for the whole + // conformance suite hammering the same Redis concurrently. + await Task.WhenAll(receivedByA.Task, receivedByB.Task).WaitAsync(TimeSpan.FromSeconds(30), ct); Assert.Equal("broadcast", await receivedByA.Task); Assert.Equal("broadcast", await receivedByB.Task); } diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index db49c60d6..d1c4b719a 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -445,6 +445,22 @@ public async Task ConfigureRuntimePump_Disabled_DoesNotPumpAsync() } } + [Fact] + public async Task AddJobRuntimeService_BeforeUseRuntimeStore_RegistersExactlyOnePumpAsync() + { + var services = new ServiceCollection().AddSingleton(new JobSchedulerProbe()); + // Hosting-first ordering must not stack a second pump: AddJobRuntimeService only tunes the single core pump. + Foundatio.Extensions.Hosting.Jobs.JobHostExtensions.AddJobRuntimeService(services, o => o.PollInterval = TimeSpan.FromMilliseconds(25)); + services.AddFoundatio().Jobs.UseInMemoryRuntime(); + await using var provider = services.BuildServiceProvider(); + + var hostedServices = provider.GetServices().ToList(); + Assert.Single(hostedServices.OfType()); + Assert.Empty(hostedServices.OfType()); + // The options passed to AddJobRuntimeService are carried onto that single pump. + Assert.Equal(TimeSpan.FromMilliseconds(25), provider.GetRequiredService().PollInterval); + } + private static JobScheduleProcessor CreateProcessor(IJobScheduler scheduler, IJobRuntimeStore store, string nodeId, IMessageTransport? transport = null) { var serviceProvider = new ServiceCollection() From 4f8e8c81465167bebd21e9903000cb9bfb2ffedf Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 10:34:08 -0500 Subject: [PATCH 28/57] Add Foundatio.MessagingSample: scaled messaging + jobs demo A minimal ASP.NET app showing the redesigned messaging and durable-jobs API in a real, scaled-out setup, wired from one clean AddFoundatio() chain: - Queue (competing consumers): POST /orders -> exactly one replica processes each order. - Pub/Sub (fan-out): POST /announcements -> every replica receives it (per-instance subscription). - Durable job: POST /reports -> runs on whichever replica's pump claims it; GET /reports/{id} shows status/progress. - CRON job: a heartbeat runs every minute, deduped to one replica per tick via the shared runtime store. Messaging runs on AWS SQS/SNS (LocalStack) and durable jobs on Redis; the transport is config-selectable (Messaging:Provider = Aws | Redis) without touching any queue/pub-sub code. Extends the existing Aspire AppHost with a LocalStack container and 3 replicas of the service so the distributed behavior is directly observable in the dashboard logs. Smoke-tested standalone against Redis: order consumed, announcement received, report job completed (progress 100), heartbeat ticked. Both solutions build; the sample is registered in Foundatio.slnx and Foundatio.All.slnx. Co-Authored-By: Claude Opus 4.8 --- Foundatio.All.slnx | 3 + Foundatio.slnx | 1 + .../Foundatio.AppHost.csproj | 1 + samples/Foundatio.AppHost/Program.cs | 20 ++++ .../Foundatio.MessagingSample.csproj | 16 +++ samples/Foundatio.MessagingSample/Jobs.cs | 39 ++++++++ samples/Foundatio.MessagingSample/Messages.cs | 24 +++++ .../MessagingWorkers.cs | 44 +++++++++ samples/Foundatio.MessagingSample/Program.cs | 99 +++++++++++++++++++ .../Properties/launchSettings.json | 12 +++ samples/Foundatio.MessagingSample/README.md | 48 +++++++++ .../appsettings.json | 12 +++ 12 files changed, 319 insertions(+) create mode 100644 samples/Foundatio.MessagingSample/Foundatio.MessagingSample.csproj create mode 100644 samples/Foundatio.MessagingSample/Jobs.cs create mode 100644 samples/Foundatio.MessagingSample/Messages.cs create mode 100644 samples/Foundatio.MessagingSample/MessagingWorkers.cs create mode 100644 samples/Foundatio.MessagingSample/Program.cs create mode 100644 samples/Foundatio.MessagingSample/Properties/launchSettings.json create mode 100644 samples/Foundatio.MessagingSample/README.md create mode 100644 samples/Foundatio.MessagingSample/appsettings.json diff --git a/Foundatio.All.slnx b/Foundatio.All.slnx index 356595f40..7782fc266 100644 --- a/Foundatio.All.slnx +++ b/Foundatio.All.slnx @@ -17,6 +17,9 @@ + + + diff --git a/Foundatio.slnx b/Foundatio.slnx index 7423951fd..3acc92a5b 100644 --- a/Foundatio.slnx +++ b/Foundatio.slnx @@ -2,6 +2,7 @@ + diff --git a/samples/Foundatio.AppHost/Foundatio.AppHost.csproj b/samples/Foundatio.AppHost/Foundatio.AppHost.csproj index 0fd055124..f5601f036 100644 --- a/samples/Foundatio.AppHost/Foundatio.AppHost.csproj +++ b/samples/Foundatio.AppHost/Foundatio.AppHost.csproj @@ -15,6 +15,7 @@ + diff --git a/samples/Foundatio.AppHost/Program.cs b/samples/Foundatio.AppHost/Program.cs index e5d8cc8ff..2193b9fef 100644 --- a/samples/Foundatio.AppHost/Program.cs +++ b/samples/Foundatio.AppHost/Program.cs @@ -24,4 +24,24 @@ u.Urls.Add(new ResourceUrlAnnotation { Url = "/jobs/run", DisplayText = "Run Job", Endpoint = u.GetEndpoint("http") }); }); +// LocalStack provides AWS SQS/SNS locally so the messaging sample's AWS transport works with no cloud account. +var localstack = builder.AddContainer("localstack", "localstack/localstack", "3") + .WithContainerName("Foundatio-LocalStack") + .WithEnvironment("SERVICES", "sqs,sns") + .WithEndpoint(port: 4566, targetPort: 4566, scheme: "http", name: "gateway") + .WithHttpHealthCheck("/_localstack/health", endpointName: "gateway"); + +// The redesigned messaging + durable-jobs sample, scaled to 3 replicas so you can watch the queue load-balance across +// instances, the pub/sub topic fan out to every instance, and durable/CRON jobs get claimed by a single instance. +// Messaging runs on AWS (SQS/SNS via LocalStack) and durable jobs on Redis; set Messaging__Provider=Redis to run the +// messaging on Redis Streams instead. +builder.AddProject("Foundatio-MessagingSample") + .WithExternalHttpEndpoints() + .WithReplicas(3) + .WithReference(cache) + .WaitFor(cache) + .WaitFor(localstack) + .WithEnvironment("Messaging__Provider", "Aws") + .WithEnvironment("Aws__ServiceUrl", localstack.GetEndpoint("gateway")); + await builder.Build().RunAsync(); diff --git a/samples/Foundatio.MessagingSample/Foundatio.MessagingSample.csproj b/samples/Foundatio.MessagingSample/Foundatio.MessagingSample.csproj new file mode 100644 index 000000000..f532dbd33 --- /dev/null +++ b/samples/Foundatio.MessagingSample/Foundatio.MessagingSample.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + diff --git a/samples/Foundatio.MessagingSample/Jobs.cs b/samples/Foundatio.MessagingSample/Jobs.cs new file mode 100644 index 000000000..dae620d35 --- /dev/null +++ b/samples/Foundatio.MessagingSample/Jobs.cs @@ -0,0 +1,39 @@ +using Foundatio.Jobs; + +namespace Foundatio.MessagingSample; + +/// +/// A durable, on-demand job (submitted via POST /reports). It runs on whichever instance's runtime pump claims +/// it, and reports progress through its so GET /reports/{id} can observe it. +/// +public sealed class GenerateReportJob(InstanceInfo instance, ILogger logger) : IJobWithExecutionContext +{ + public JobExecutionContext? ExecutionContext { get; set; } + + public async Task RunAsync(CancellationToken cancellationToken = default) + { + logger.LogInformation("[{Instance}] generating report {JobId}", instance.Id, ExecutionContext?.JobId); + + for (int percent = 25; percent <= 100; percent += 25) + { + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken); + if (ExecutionContext is { } context) + await context.ReportProgressAsync(percent, $"{percent}% complete", cancellationToken); + } + + return JobResult.Success; + } +} + +/// +/// A recurring job (see the CRON schedule wired in Program.cs). Every instance registers the same schedule, but the +/// shared runtime store dedupes each occurrence, so exactly one instance runs each tick. +/// +public sealed class HeartbeatJob(InstanceInfo instance, ILogger logger) : IJob +{ + public Task RunAsync(CancellationToken cancellationToken = default) + { + logger.LogInformation("[{Instance}] heartbeat {Time:HH:mm:ss}", instance.Id, DateTimeOffset.UtcNow); + return Task.FromResult(JobResult.Success); + } +} diff --git a/samples/Foundatio.MessagingSample/Messages.cs b/samples/Foundatio.MessagingSample/Messages.cs new file mode 100644 index 000000000..b37fefe65 --- /dev/null +++ b/samples/Foundatio.MessagingSample/Messages.cs @@ -0,0 +1,24 @@ +using Foundatio.Messaging; + +namespace Foundatio.MessagingSample; + +/// +/// A unit of work processed off a queue. The names the destination ("orders"); +/// with competing consumers, each order is handled by exactly one running instance. +/// +[MessageRoute("orders")] +public class ProcessOrder +{ + public string Product { get; set; } = ""; + public int Quantity { get; set; } = 1; +} + +/// +/// A broadcast event published to a topic ("announcements"). With a per-instance subscription, every running instance +/// receives its own copy. +/// +[MessageRoute("announcements")] +public class Announcement +{ + public string Text { get; set; } = ""; +} diff --git a/samples/Foundatio.MessagingSample/MessagingWorkers.cs b/samples/Foundatio.MessagingSample/MessagingWorkers.cs new file mode 100644 index 000000000..b4131e811 --- /dev/null +++ b/samples/Foundatio.MessagingSample/MessagingWorkers.cs @@ -0,0 +1,44 @@ +using Foundatio.Messaging; + +namespace Foundatio.MessagingSample; + +/// A short per-process id so you can see which instance handled each message/job when scaled to replicas. +public sealed record InstanceInfo(string Id); + +/// +/// Starts this instance's long-running queue consumer and pub/sub subscriber for the app's lifetime — the idiomatic +/// way to host Foundatio consumers in ASP.NET. Handlers auto-complete on success (); throwing +/// triggers the core's retry/dead-letter policy. +/// +public sealed class MessagingWorkers(IQueue queue, IPubSub pubSub, InstanceInfo instance, ILogger logger) : IHostedService +{ + private IMessageConsumer? _orderConsumer; + private IMessageSubscription? _announcementSubscription; + + public async Task StartAsync(CancellationToken cancellationToken) + { + // Competing consumers: the shared "orders" queue load-balances across every running instance, so each order is + // processed exactly once. Scale the service up and the work spreads out. + _orderConsumer = await queue.StartConsumerAsync((message, _) => + { + logger.LogInformation("[{Instance}] processed order: {Quantity} x {Product}", instance.Id, message.Message.Quantity, message.Message.Product); + return Task.CompletedTask; + }, cancellationToken: cancellationToken); + + // Fan-out: a per-instance subscription means every instance receives every announcement (broadcast). Using a + // shared subscription name here would instead load-balance the topic like the queue above. + _announcementSubscription = await pubSub.SubscribeAsync((message, _) => + { + logger.LogInformation("[{Instance}] announcement: {Text}", instance.Id, message.Message.Text); + return Task.CompletedTask; + }, new PubSubSubscriptionOptions { Subscription = instance.Id }, cancellationToken); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_orderConsumer is not null) + await _orderConsumer.DisposeAsync(); + if (_announcementSubscription is not null) + await _announcementSubscription.DisposeAsync(); + } +} diff --git a/samples/Foundatio.MessagingSample/Program.cs b/samples/Foundatio.MessagingSample/Program.cs new file mode 100644 index 000000000..282e437f5 --- /dev/null +++ b/samples/Foundatio.MessagingSample/Program.cs @@ -0,0 +1,99 @@ +using Amazon; +using Amazon.Runtime; +using Foundatio; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Foundatio.MessagingSample; +using StackExchange.Redis; + +var builder = WebApplication.CreateBuilder(args); + +// A short id so log lines make it obvious WHICH instance handled each message/job when scaled to multiple replicas. +var instance = new InstanceInfo(Guid.NewGuid().ToString("N")[..6]); +builder.Services.AddSingleton(instance); + +// One shared Redis connection: it backs the durable job runtime, and (when selected) the messaging transport too. +string redisConnectionString = builder.Configuration.GetConnectionString("Redis") ?? "localhost:6399"; +builder.Services.AddSingleton(_ => ConnectionMultiplexer.Connect(redisConnectionString)); + +// The messaging transport is chosen at startup — both AWS (SQS/SNS) and Redis (Streams) are wired, so you can flip +// Messaging:Provider and compare them without touching a line of the queue/pub-sub code below. +string transport = builder.Configuration["Messaging:Provider"] ?? "Redis"; + +builder.Services.AddFoundatio() + // Queues (competing consumers) and pub/sub (fan-out) both ride this single transport. + .Messaging.UseTransport(sp => transport.Equals("Aws", StringComparison.OrdinalIgnoreCase) + ? CreateAwsTransport(builder.Configuration) + : new RedisStreamsMessageTransport(new RedisStreamsMessageTransportOptions + { + ConnectionMultiplexer = sp.GetRequiredService() + })) + // Durable jobs live in Redis so any instance can claim and run them. UseRuntimeStore also auto-registers the pump + // that materializes CRON occurrences, drains scheduled work, and runs submitted jobs. + .Jobs.UseRuntimeStore(sp => new RedisJobRuntimeStore(new RedisJobRuntimeStoreOptions + { + ConnectionMultiplexer = sp.GetRequiredService() + })) + .Jobs.Register("generate-report") + .Jobs.Register("heartbeat"); + +// Hosts this instance's queue consumer + pub/sub subscriber for the app lifetime. +builder.Services.AddHostedService(); + +var app = builder.Build(); + +// Recurring job: every instance registers the same schedule, but the shared Redis store dedupes each occurrence, so +// exactly one instance runs each tick. +await app.Services.GetRequiredService().ScheduleAsync(new ScheduledJobDefinition +{ + Name = "heartbeat", + Cron = "* * * * *", // every minute + JobType = typeof(HeartbeatJob) +}); + +app.MapGet("/", (InstanceInfo i) => Results.Ok(new { service = "Foundatio messaging sample", instance = i.Id, transport })); + +// QUEUE — competing consumers: exactly one instance processes each order. +app.MapPost("/orders", async (ProcessOrder order, IQueue queue) => +{ + string id = await queue.EnqueueAsync(order); + return Results.Accepted(value: new { queued = id }); +}); + +// PUB/SUB — fan-out: every instance receives each announcement. +app.MapPost("/announcements", async (Announcement announcement, IPubSub pubSub) => +{ + await pubSub.PublishAsync(announcement); + return Results.Accepted(value: new { published = announcement.Text }); +}); + +// DURABLE JOB — submitted here, executed on whichever instance's runtime pump claims it. +app.MapPost("/reports", async (IJobClient jobs) => +{ + var handle = await jobs.EnqueueAsync(); + return Results.Accepted($"/reports/{handle.JobId}", new { jobId = handle.JobId }); +}); + +app.MapGet("/reports/{id}", async (string id, IJobMonitor monitor) => +{ + var state = await monitor.GetAsync(id); + return state is null + ? Results.NotFound() + : Results.Ok(new { state.JobId, status = state.Status.ToString(), state.Progress, state.ProgressMessage }); +}); + +app.Run(); + +// LocalStack (provisioned by the AppHost) provides AWS SQS/SNS locally and accepts any credentials. AutoCreateDestinations +// creates queues/topics on first use; ResourcePrefix keeps this sample's resources namespaced. +static AwsMessageTransport CreateAwsTransport(IConfiguration configuration) +{ + return new AwsMessageTransport(new AwsMessageTransportOptions + { + ServiceUrl = configuration["Aws:ServiceUrl"] ?? "http://localhost:4566", + Region = RegionEndpoint.USEast1, + Credentials = new BasicAWSCredentials("test", "test"), + AutoCreateDestinations = true, + ResourcePrefix = "fnd-sample-" + }); +} diff --git a/samples/Foundatio.MessagingSample/Properties/launchSettings.json b/samples/Foundatio.MessagingSample/Properties/launchSettings.json new file mode 100644 index 000000000..6c31385a1 --- /dev/null +++ b/samples/Foundatio.MessagingSample/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "applicationUrl": "http://localhost:5280", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/Foundatio.MessagingSample/README.md b/samples/Foundatio.MessagingSample/README.md new file mode 100644 index 000000000..a68f932aa --- /dev/null +++ b/samples/Foundatio.MessagingSample/README.md @@ -0,0 +1,48 @@ +# Foundatio.MessagingSample + +A minimal ASP.NET app that shows the redesigned Foundatio **messaging** (queues + pub/sub) and **durable jobs** in a +real, scaled-out setup. It runs under Aspire with **3 replicas**, so you can watch the distributed behavior: + +- **Queue (competing consumers)** — `POST /orders` enqueues work; exactly **one** replica processes each order. Scale + up and the work spreads out. +- **Pub/Sub (fan-out)** — `POST /announcements` publishes to a topic; **every** replica receives its own copy (each + uses a per-instance subscription). +- **Durable job** — `POST /reports` submits a job; whichever replica's runtime pump claims it runs it. Poll + `GET /reports/{id}` to watch its status/progress. +- **CRON job** — a `heartbeat` runs every minute; the shared runtime store dedupes occurrences so exactly one replica + runs each tick. + +Messaging runs on **AWS SQS/SNS** (via a LocalStack container) and durable jobs on **Redis** — both transports wired +from one clean `AddFoundatio()` chain in [`Program.cs`](Program.cs). The transport is selected by `Messaging:Provider` +(`Aws` or `Redis`), so you can flip it without touching any queue/pub-sub code. + +## Run it (Aspire) + +```sh +dotnet run --project samples/Foundatio.AppHost +``` + +The Aspire dashboard launches Redis + LocalStack and 3 replicas of the service. Open the service endpoint and: + +```sh +# fire several orders — watch them load-balance across the 3 replicas' logs +for i in $(seq 1 6); do curl -sX POST /orders -H 'content-type: application/json' -d "{\"product\":\"widget\",\"quantity\":$i}"; done + +# publish an announcement — every replica logs it +curl -sX POST /announcements -H 'content-type: application/json' -d '{"text":"hello all"}' + +# submit a durable job, then poll it +job=$(curl -sX POST /reports | jq -r .jobId); curl -s /reports/$job +``` + +The per-instance id in each log line (`[abc123] processed order: ...`) makes the distribution obvious. To run messaging +on Redis Streams instead of AWS, set `Messaging__Provider=Redis` on the service in the AppHost. + +## Run it standalone (no Aspire) + +Point it at a Redis instance (defaults to `localhost:6399`) and use the Redis transport: + +```sh +Messaging__Provider=Redis ConnectionStrings__Redis=localhost:6399 \ + dotnet run --project samples/Foundatio.MessagingSample +``` diff --git a/samples/Foundatio.MessagingSample/appsettings.json b/samples/Foundatio.MessagingSample/appsettings.json new file mode 100644 index 000000000..d8cacb4d3 --- /dev/null +++ b/samples/Foundatio.MessagingSample/appsettings.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "Messaging": { + "Provider": "Redis" + } +} From c704c93d4ce7be3e3d8261c4844266f7b92ac546 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 10:37:57 -0500 Subject: [PATCH 29/57] MessagingSample: richer CRON demo (Global vs PerNode scope + schedules) Expand the sample's scheduled-jobs story from a single heartbeat to a small, illustrative set on the new IJobScheduler API: - heartbeat (Global, * * * * *) -> one instance per tick - refresh-cache (PerNode, * * * * *) -> every instance per tick - sweep-stale-orders (Global, */2 * * * *) -> one instance, coarser schedule This shows the two things the redesigned scheduler adds over a plain cron: distributed dedup (Global = leader/singleton) and PerNode fan-out, plus schedule variety. Smoke-tested against Redis: heartbeat and refresh-cache both fire (materialized via the misfire window at startup). Co-Authored-By: Claude Opus 4.8 --- samples/Foundatio.MessagingSample/Jobs.cs | 32 +++++++++++++++++--- samples/Foundatio.MessagingSample/Program.cs | 18 +++++------ samples/Foundatio.MessagingSample/README.md | 6 ++-- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/samples/Foundatio.MessagingSample/Jobs.cs b/samples/Foundatio.MessagingSample/Jobs.cs index dae620d35..acffd9ebf 100644 --- a/samples/Foundatio.MessagingSample/Jobs.cs +++ b/samples/Foundatio.MessagingSample/Jobs.cs @@ -25,15 +25,37 @@ public async Task RunAsync(CancellationToken cancellationToken = defa } } -/// -/// A recurring job (see the CRON schedule wired in Program.cs). Every instance registers the same schedule, but the -/// shared runtime store dedupes each occurrence, so exactly one instance runs each tick. -/// +// The recurring (CRON) jobs below are scheduled in Program.cs. Every instance registers the same schedules, but each +// occurrence is materialized once into the shared runtime store, so scope decides how many instances run it: +// * Global (default) -> exactly ONE instance runs each tick (a leader/singleton task). +// * PerNode -> EVERY instance runs its own occurrence each tick (per-instance maintenance). + +/// Global, every minute: a simple liveness heartbeat that runs on a single instance per tick. public sealed class HeartbeatJob(InstanceInfo instance, ILogger logger) : IJob { public Task RunAsync(CancellationToken cancellationToken = default) { - logger.LogInformation("[{Instance}] heartbeat {Time:HH:mm:ss}", instance.Id, DateTimeOffset.UtcNow); + logger.LogInformation("[{Instance}] heartbeat {Time:HH:mm:ss} (one instance per tick)", instance.Id, DateTimeOffset.UtcNow); + return Task.FromResult(JobResult.Success); + } +} + +/// PerNode, every minute: each instance refreshes its own local state — so every instance runs this each tick. +public sealed class RefreshCacheJob(InstanceInfo instance, ILogger logger) : IJob +{ + public Task RunAsync(CancellationToken cancellationToken = default) + { + logger.LogInformation("[{Instance}] refreshed local cache (every instance per tick)", instance.Id); + return Task.FromResult(JobResult.Success); + } +} + +/// Global, every 2 minutes: a periodic maintenance sweep that runs on a single instance per tick. +public sealed class SweepStaleOrdersJob(InstanceInfo instance, ILogger logger) : IJob +{ + public Task RunAsync(CancellationToken cancellationToken = default) + { + logger.LogInformation("[{Instance}] swept stale orders (one instance per tick)", instance.Id); return Task.FromResult(JobResult.Success); } } diff --git a/samples/Foundatio.MessagingSample/Program.cs b/samples/Foundatio.MessagingSample/Program.cs index 282e437f5..3f134edce 100644 --- a/samples/Foundatio.MessagingSample/Program.cs +++ b/samples/Foundatio.MessagingSample/Program.cs @@ -35,21 +35,21 @@ ConnectionMultiplexer = sp.GetRequiredService() })) .Jobs.Register("generate-report") - .Jobs.Register("heartbeat"); + .Jobs.Register("heartbeat") + .Jobs.Register("refresh-cache") + .Jobs.Register("sweep-stale-orders"); // Hosts this instance's queue consumer + pub/sub subscriber for the app lifetime. builder.Services.AddHostedService(); var app = builder.Build(); -// Recurring job: every instance registers the same schedule, but the shared Redis store dedupes each occurrence, so -// exactly one instance runs each tick. -await app.Services.GetRequiredService().ScheduleAsync(new ScheduledJobDefinition -{ - Name = "heartbeat", - Cron = "* * * * *", // every minute - JobType = typeof(HeartbeatJob) -}); +// Recurring (CRON) jobs. Every instance registers the same schedules; the shared Redis store dedupes each occurrence, +// so Scope decides how many instances run it — Global = one instance per tick, PerNode = every instance per tick. +var scheduler = app.Services.GetRequiredService(); +await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "heartbeat", Cron = "* * * * *", JobType = typeof(HeartbeatJob) }); +await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "refresh-cache", Cron = "* * * * *", Scope = ScheduledJobScope.PerNode, JobType = typeof(RefreshCacheJob) }); +await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "sweep-stale-orders", Cron = "*/2 * * * *", JobType = typeof(SweepStaleOrdersJob) }); app.MapGet("/", (InstanceInfo i) => Results.Ok(new { service = "Foundatio messaging sample", instance = i.Id, transport })); diff --git a/samples/Foundatio.MessagingSample/README.md b/samples/Foundatio.MessagingSample/README.md index a68f932aa..51b229116 100644 --- a/samples/Foundatio.MessagingSample/README.md +++ b/samples/Foundatio.MessagingSample/README.md @@ -9,8 +9,10 @@ real, scaled-out setup. It runs under Aspire with **3 replicas**, so you can wat uses a per-instance subscription). - **Durable job** — `POST /reports` submits a job; whichever replica's runtime pump claims it runs it. Poll `GET /reports/{id}` to watch its status/progress. -- **CRON job** — a `heartbeat` runs every minute; the shared runtime store dedupes occurrences so exactly one replica - runs each tick. +- **CRON jobs** — scheduled recurring work, deduped through the shared runtime store so **scope** decides fan-out: + - `heartbeat` — Global, every minute → runs on **one** replica per tick (leader/singleton). + - `refresh-cache` — PerNode, every minute → runs on **every** replica per tick (per-instance maintenance). + - `sweep-stale-orders` — Global, every 2 minutes → a periodic maintenance sweep on one replica. Messaging runs on **AWS SQS/SNS** (via a LocalStack container) and durable jobs on **Redis** — both transports wired from one clean `AddFoundatio()` chain in [`Program.cs`](Program.cs). The transport is selected by `Messaging:Provider` From 393459555319069c399d4b2573a90ff4ba6ff18c Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 11:27:16 -0500 Subject: [PATCH 30/57] samples: drop legacy HostingSample; AppHost runs only the redesign sample Converge the sample story on the redesign: remove the legacy Foundatio.HostingSample (old IMessageBus + AddJob/AddCronJob/JobManager demo) and have the shared Aspire AppHost launch only the new Foundatio.MessagingSample (plus Redis + LocalStack). Removed from both solutions and the AppHost project reference. Co-Authored-By: Claude Opus 4.8 --- Foundatio.All.slnx | 1 - Foundatio.slnx | 1 - .../Foundatio.AppHost.csproj | 1 - samples/Foundatio.AppHost/Program.cs | 13 - .../Foundatio.HostingSample.csproj | 24 -- .../Jobs/EveryMinuteJob.cs | 33 --- .../Jobs/Sample1Job.cs | 42 ---- .../Jobs/Sample2Job.cs | 41 ---- .../Jobs/SampleLockJob.cs | 32 --- .../MyCriticalHealthCheck.cs | 18 -- samples/Foundatio.HostingSample/Program.cs | 222 ------------------ .../Properties/launchSettings.json | 27 --- .../ServiceDefaults.cs | 57 ----- .../Startup/MyStartupAction.cs | 42 ---- .../Startup/OtherStartupAction.cs | 25 -- .../appsettings.Development.json | 8 - .../Foundatio.HostingSample/appsettings.json | 10 - 17 files changed, 597 deletions(-) delete mode 100644 samples/Foundatio.HostingSample/Foundatio.HostingSample.csproj delete mode 100644 samples/Foundatio.HostingSample/Jobs/EveryMinuteJob.cs delete mode 100644 samples/Foundatio.HostingSample/Jobs/Sample1Job.cs delete mode 100644 samples/Foundatio.HostingSample/Jobs/Sample2Job.cs delete mode 100644 samples/Foundatio.HostingSample/Jobs/SampleLockJob.cs delete mode 100644 samples/Foundatio.HostingSample/MyCriticalHealthCheck.cs delete mode 100644 samples/Foundatio.HostingSample/Program.cs delete mode 100644 samples/Foundatio.HostingSample/Properties/launchSettings.json delete mode 100644 samples/Foundatio.HostingSample/ServiceDefaults.cs delete mode 100644 samples/Foundatio.HostingSample/Startup/MyStartupAction.cs delete mode 100644 samples/Foundatio.HostingSample/Startup/OtherStartupAction.cs delete mode 100644 samples/Foundatio.HostingSample/appsettings.Development.json delete mode 100644 samples/Foundatio.HostingSample/appsettings.json diff --git a/Foundatio.All.slnx b/Foundatio.All.slnx index 7782fc266..0dcdddb02 100644 --- a/Foundatio.All.slnx +++ b/Foundatio.All.slnx @@ -1,7 +1,6 @@ - diff --git a/Foundatio.slnx b/Foundatio.slnx index 3acc92a5b..f90428570 100644 --- a/Foundatio.slnx +++ b/Foundatio.slnx @@ -1,7 +1,6 @@ - diff --git a/samples/Foundatio.AppHost/Foundatio.AppHost.csproj b/samples/Foundatio.AppHost/Foundatio.AppHost.csproj index f5601f036..7dcbfef32 100644 --- a/samples/Foundatio.AppHost/Foundatio.AppHost.csproj +++ b/samples/Foundatio.AppHost/Foundatio.AppHost.csproj @@ -14,7 +14,6 @@ - diff --git a/samples/Foundatio.AppHost/Program.cs b/samples/Foundatio.AppHost/Program.cs index 2193b9fef..fb85c7cc3 100644 --- a/samples/Foundatio.AppHost/Program.cs +++ b/samples/Foundatio.AppHost/Program.cs @@ -11,19 +11,6 @@ .WithRedisInsight(b => b.WithEndpointProxySupport(false).WithContainerName("Foundatio-RedisInsight") .WithUrlForEndpoint("http", u => u.DisplayText = "Cache")); -builder.AddProject("Foundatio-HostingSample") - .WithExternalHttpEndpoints() - .WithReplicas(3) - .WithReference(cache) - .WaitFor(cache) - .WithArgs("all") - .WithUrls(u => - { - u.Urls.Clear(); - u.Urls.Add(new ResourceUrlAnnotation { Url = "/jobs/status", DisplayText = "Job Status", Endpoint = u.GetEndpoint("http") }); - u.Urls.Add(new ResourceUrlAnnotation { Url = "/jobs/run", DisplayText = "Run Job", Endpoint = u.GetEndpoint("http") }); - }); - // LocalStack provides AWS SQS/SNS locally so the messaging sample's AWS transport works with no cloud account. var localstack = builder.AddContainer("localstack", "localstack/localstack", "3") .WithContainerName("Foundatio-LocalStack") diff --git a/samples/Foundatio.HostingSample/Foundatio.HostingSample.csproj b/samples/Foundatio.HostingSample/Foundatio.HostingSample.csproj deleted file mode 100644 index f4dfa2769..000000000 --- a/samples/Foundatio.HostingSample/Foundatio.HostingSample.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - - net10.0 - False - false - REDIS - - - - - - - - - - - - - - - - - diff --git a/samples/Foundatio.HostingSample/Jobs/EveryMinuteJob.cs b/samples/Foundatio.HostingSample/Jobs/EveryMinuteJob.cs deleted file mode 100644 index e879b104a..000000000 --- a/samples/Foundatio.HostingSample/Jobs/EveryMinuteJob.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Caching; -using Foundatio.Jobs; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -public class EveryMinuteJob : IJob -{ - private readonly ICacheClient _cacheClient; - private readonly ILogger _logger; - - public EveryMinuteJob(ILoggerFactory loggerFactory, ICacheClient cacheClient) - { - _cacheClient = cacheClient; - _logger = loggerFactory.CreateLogger(); - } - - public async Task RunAsync(CancellationToken cancellationToken = default) - { - var runCount = await _cacheClient.IncrementAsync("EveryMinuteJob"); - - _logger.LogInformation("EveryMinuteJob Run Count={Count} Thread={ManagedThreadId}", runCount, Thread.CurrentThread.ManagedThreadId); - - await Task.Delay(TimeSpan.FromSeconds(30)); - - _logger.LogInformation("EveryMinuteJob Complete"); - - return JobResult.Success; - } -} diff --git a/samples/Foundatio.HostingSample/Jobs/Sample1Job.cs b/samples/Foundatio.HostingSample/Jobs/Sample1Job.cs deleted file mode 100644 index bea203a60..000000000 --- a/samples/Foundatio.HostingSample/Jobs/Sample1Job.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Jobs; -using Foundatio.Resilience; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -[Job(Description = "Sample 1 job", Interval = "5s", IterationLimit = 5)] -public class Sample1Job : IJob -{ - private readonly IResiliencePolicy _policy; - private readonly ILogger _logger; - private int _iterationCount = 0; - - public Sample1Job(IResiliencePolicyProvider provider, ILoggerFactory loggerFactory) - { - // get policy for Sample1Job and if not found, try to get policy for IJob, then fallback to default policy - _policy = provider.GetPolicy(); - _logger = loggerFactory.CreateLogger(); - } - - public async Task RunAsync(CancellationToken cancellationToken = default) - { - return await _policy.ExecuteAsync(async _ => - { - int count = Interlocked.Increment(ref _iterationCount); - _logger.LogTrace("Sample1Job Run #{IterationCount} Thread={ManagedThreadId}", _iterationCount, Thread.CurrentThread.ManagedThreadId); - - if (count < 3) - { - _logger.LogInformation("Sample1Job Run #{IterationCount} Thread={ManagedThreadId} - Simulating failure", _iterationCount, Thread.CurrentThread.ManagedThreadId); - throw new InvalidOperationException("Simulated failure"); - } - - await Task.Delay(5000, cancellationToken); - - return JobResult.Success; - }, cancellationToken); - } -} diff --git a/samples/Foundatio.HostingSample/Jobs/Sample2Job.cs b/samples/Foundatio.HostingSample/Jobs/Sample2Job.cs deleted file mode 100644 index dc7fbdccd..000000000 --- a/samples/Foundatio.HostingSample/Jobs/Sample2Job.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Jobs; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -[Job(Description = "Sample 2 job", Interval = "15s", IterationLimit = 24)] -public class Sample2Job : IJob, IHealthCheck -{ - private readonly ILogger _logger; - private int _iterationCount = 0; - private DateTime? _lastRun = null; - - public Sample2Job(ILoggerFactory loggerFactory) - { - _logger = loggerFactory.CreateLogger(); - } - - public Task RunAsync(CancellationToken cancellationToken = default) - { - _lastRun = DateTime.UtcNow; - Interlocked.Increment(ref _iterationCount); - _logger.LogTrace("Sample2Job Run #{IterationCount} Thread={ManagedThreadId}", _iterationCount, Thread.CurrentThread.ManagedThreadId); - - return Task.FromResult(JobResult.Success); - } - - public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) - { - if (!_lastRun.HasValue) - return Task.FromResult(HealthCheckResult.Healthy("Job has not been run yet.")); - - if (DateTime.UtcNow.Subtract(_lastRun.Value) > TimeSpan.FromSeconds(5)) - return Task.FromResult(HealthCheckResult.Unhealthy("Job has not run in the last 5 seconds.")); - - return Task.FromResult(HealthCheckResult.Healthy("Job has run in the last 5 seconds.")); - } -} diff --git a/samples/Foundatio.HostingSample/Jobs/SampleLockJob.cs b/samples/Foundatio.HostingSample/Jobs/SampleLockJob.cs deleted file mode 100644 index 9cae2fc83..000000000 --- a/samples/Foundatio.HostingSample/Jobs/SampleLockJob.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Caching; -using Foundatio.Jobs; -using Foundatio.Lock; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -[Job(Description = "Sample lock job", Interval = "5s")] -public class SampleLockJob : JobWithLockBase -{ - private readonly ILockProvider _lockProvider; - - public SampleLockJob(ICacheClient cache, ILoggerFactory loggerFactory) : base(loggerFactory) - { - _lockProvider = new ThrottlingLockProvider(cache, 1, TimeSpan.FromMinutes(1), _timeProvider, _resiliencePolicyProvider, loggerFactory); - } - - protected override Task GetLockAsync(CancellationToken cancellationToken = default) - { - return _lockProvider.AcquireAsync(nameof(SampleLockJob), TimeSpan.FromMinutes(15), cancellationToken); - } - - protected override Task RunInternalAsync(JobContext context) - { - _logger.LogTrace("SampleLockJob Run Thread={ManagedThreadId}", Thread.CurrentThread.ManagedThreadId); - - return Task.FromResult(JobResult.Success); - } -} diff --git a/samples/Foundatio.HostingSample/MyCriticalHealthCheck.cs b/samples/Foundatio.HostingSample/MyCriticalHealthCheck.cs deleted file mode 100644 index 7d3d7ac84..000000000 --- a/samples/Foundatio.HostingSample/MyCriticalHealthCheck.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Diagnostics.HealthChecks; - -namespace Foundatio.HostingSample; - -public class MyCriticalHealthCheck : IHealthCheck -{ - private static DateTime _startTime = DateTime.Now; - - public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken()) - { - return DateTime.Now.Subtract(_startTime) > TimeSpan.FromSeconds(3) ? - Task.FromResult(HealthCheckResult.Healthy("Critical resource is available.")) - : Task.FromResult(HealthCheckResult.Unhealthy("Critical resource not available.")); - } -} diff --git a/samples/Foundatio.HostingSample/Program.cs b/samples/Foundatio.HostingSample/Program.cs deleted file mode 100644 index 8560a9786..000000000 --- a/samples/Foundatio.HostingSample/Program.cs +++ /dev/null @@ -1,222 +0,0 @@ -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Foundatio; -using Foundatio.Extensions.Hosting.Jobs; -using Foundatio.Extensions.Hosting.Startup; -using Foundatio.HostingSample; -using Foundatio.Resilience; -using Foundatio.Serializer; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Json; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -#if REDIS -using Microsoft.Extensions.Configuration; -using Foundatio.Redis; -using StackExchange.Redis; -#endif - -bool all = args.Contains("all", StringComparer.OrdinalIgnoreCase); -bool sample1 = all || args.Contains("sample1", StringComparer.OrdinalIgnoreCase); -bool sample2 = all || args.Contains("sample2", StringComparer.OrdinalIgnoreCase); -bool everyMinute = all || args.Contains("everyMinute", StringComparer.OrdinalIgnoreCase); -bool evenMinutes = all || args.Contains("evenMinutes", StringComparer.OrdinalIgnoreCase); - -var builder = WebApplication.CreateBuilder(args); - -// configure Foundatio services -builder.Services.AddFoundatio() - .Storage.UseFolder() - .Caching.UseInMemory() - .Locking.UseCache() - .Messaging.UseInMemory() - .AddSerializer(sp => new SystemTextJsonSerializer(sp.GetRequiredService>().Value.SerializerOptions)) - .AddResilience(b => b.WithPolicy(p => p.WithMaxAttempts(5).WithLinearDelay().WithJitter())); - -ConfigureServices(); - -// shutdown the host if no jobs are running, cron jobs are not considered running jobs -builder.Services.AddJobLifetimeService(); - -// inserts a startup action that does not complete until the critical health checks are healthy -// gets inserted as 1st startup action so that any other startup actions don't run until the critical resources are available -builder.Services.AddStartupActionToWaitForHealthChecks("Critical"); - -builder.Services.AddHealthChecks().AddCheck("My Critical Resource", tags: ["Critical"]); - -// add health check that does not return healthy until the startup actions have completed -// useful for readiness checks -builder.Services.AddHealthChecks().AddCheckForStartupActions("Critical"); - -// this gets added automatically by any AddJob call, but we might not be running any jobs, and we need it for doing dynamic jobs -builder.Services.AddJobScheduler(); - -if (everyMinute) - builder.Services.AddDistributedCronJob("* * * * *"); - -builder.Services.AddCronJob(b => b.Name("Tokyo").CronSchedule("44 4 * * *").CronTimeZone("Asia/Tokyo").JobAction(async sp => -{ - var logger = sp.GetRequiredService>(); - logger.LogInformation("Tokyo 4:44am Run Thread={ManagedThreadId}", Thread.CurrentThread.ManagedThreadId); - await Task.Delay(TimeSpan.FromSeconds(5)); -})); - -if (evenMinutes) - builder.Services.AddCronJob("EvenMinutes", "*/2 * * * *", async sp => - { - var logger = sp.GetRequiredService>(); - logger.LogInformation("EvenMinuteJob Run Thread={ManagedThreadId}", Thread.CurrentThread.ManagedThreadId); - await Task.Delay(TimeSpan.FromSeconds(30)); - logger.LogInformation("EvenMinuteJob Complete"); - }); - -if (sample1) - builder.Services.AddJob("Sample1", sp => new Sample1Job(sp.GetService(), sp.GetService()), o => o.ApplyDefaults().WaitForStartupActions().InitialDelay(TimeSpan.FromSeconds(4))); - -builder.Services.AddJob(o => o.WaitForStartupActions()); - -if (sample2) -{ - builder.Services.AddHealthChecks().AddCheck("Sample2Job"); - builder.Services.AddJob(o => o.WaitForStartupActions()); -} - -// if you don't specify priority, actions will automatically be assigned an incrementing priority starting at 0 -builder.Services.AddStartupAction("Test1", async sp => -{ - var logger = sp.GetRequiredService>(); - logger.LogTrace("Running startup 1 action"); - for (int i = 0; i < 3; i++) - { - await Task.Delay(100); - logger.LogTrace("Running startup 1 action..."); - } - - logger.LogTrace("Done running startup 1 action"); -}); - -// then these startup actions will run concurrently since they both have the same priority -builder.Services.AddStartupAction(priority: 100); -builder.Services.AddStartupAction(priority: 100); - -/*builder.Services.AddStartupAction("Test2", async sp => -{ - var logger = sp.GetRequiredService>(); - logger.LogTrace("Running startup 2 action"); - for (int i = 0; i < 2; i++) - { - await Task.Delay(50); - logger.LogTrace("Running startup 2 action..."); - } - //throw new ApplicationException("Boom goes the startup"); - logger.LogTrace("Done running startup 2 action"); -});*/ - -//s.AddStartupAction("Boom", () => throw new ApplicationException("Boom goes the startup")); - -var app = builder.Build(); - -app.MapGet("/", () => "Foundatio!"); - -app.MapGet("/jobs/status", (IJobManager jobManager, string name = null, bool? running = null, bool history = true) => - { - if (!String.IsNullOrEmpty(name)) - return Results.Ok(jobManager.GetJobStatus(name, includeHistory: history)); - - if (running.HasValue && running.Value) - return Results.Ok(jobManager.GetJobStatus(true, includeHistory: history)); - - return Results.Ok(jobManager.GetJobStatus(includeHistory: history)); - }); - -app.MapGet("/jobs/run", async (IJobManager jobManager, string name) => - { - if (String.IsNullOrWhiteSpace(name)) - return Results.BadRequest("Job name is required."); - - await jobManager.RunJobAsync(name); - - return Results.Accepted($"Job {name} started successfully."); - }); - -app.MapGet("/jobs/enable", (IJobManager jobManager, string name) => - { - if (String.IsNullOrWhiteSpace(name)) - return Results.BadRequest("Job name is required."); - - jobManager.Update(name, c => c.Enabled()); - - return Results.Ok($"Job {name} enabled successfully."); - }); - -app.MapGet("/jobs/disable", (IJobManager jobManager, string name) => - { - if (String.IsNullOrWhiteSpace(name)) - return Results.BadRequest("Job name is required."); - - jobManager.Update(name, c => c.Disabled()); - - return Results.Ok($"Job {name} disabled successfully."); - }); - -app.MapGet("/jobs/schedule", (IJobManager jobManager, string name, string cron) => -{ - if (String.IsNullOrWhiteSpace(name)) - return Results.BadRequest("Job name is required."); - - jobManager.Update(name, c => c.CronSchedule(cron)); - - return Results.Ok($"Job {name} updated successfully."); -}); - -app.MapGet("/jobs/release", async (IJobManager jobManager, string name) => -{ - if (String.IsNullOrWhiteSpace(name)) - return Results.BadRequest("Job name is required."); - - await jobManager.ReleaseLockAsync(name); - - return Results.Ok($"Job {name} lock released successfully."); -}); - -app.UseHealthChecks("/health"); -app.UseReadyHealthChecks("Critical"); - -// this middleware will return Service Unavailable until the startup actions have completed -app.UseWaitForStartupActionsBeforeServingRequests(); - -// add mvc or other request middleware after the UseWaitForStartupActionsBeforeServingRequests call - -app.Run(); - -void ConfigureServices() -{ - builder.Services.AddLogging(opt => - { - opt.AddSimpleConsole(c => c.TimestampFormat = "[HH:mm:ss] "); - }); - - builder.AddServiceDefaults(); - builder.Services.ConfigureHttpJsonOptions(o => { o.SerializerOptions.WriteIndented = true; }); - -#if REDIS - builder.Services.AddSingleton(sp => - { - var connectionString = builder.Configuration.GetConnectionString("Redis")!; - connectionString += ",abortConnect=false"; - return ConnectionMultiplexer.Connect(connectionString); - // enable redis logging - //return ConnectionMultiplexer.Connect(connectionString, o => o.LoggerFactory = sp.GetRequiredService()); - }); - - // distributed cache and messaging using redis (replaces in memory cache) - builder.Services.AddFoundatio() - .Caching.UseRedis() - .Messaging.UseRedis(); -#endif -} diff --git a/samples/Foundatio.HostingSample/Properties/launchSettings.json b/samples/Foundatio.HostingSample/Properties/launchSettings.json deleted file mode 100644 index f520048a1..000000000 --- a/samples/Foundatio.HostingSample/Properties/launchSettings.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/launchsettings.json", - "profiles": { - "http": { - "commandName": "Project", - "commandLineArgs": "all", - "dotnetRunMessages": true, - "launchBrowser": false, - "launchUrl": "jobstatus", - "applicationUrl": "http://localhost:5324", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "https": { - "commandName": "Project", - "commandLineArgs": "all", - "dotnetRunMessages": true, - "launchBrowser": false, - "launchUrl": "jobstatus", - "applicationUrl": "https://localhost:7580;http://localhost:5324", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/samples/Foundatio.HostingSample/ServiceDefaults.cs b/samples/Foundatio.HostingSample/ServiceDefaults.cs deleted file mode 100644 index fbd3284d7..000000000 --- a/samples/Foundatio.HostingSample/ServiceDefaults.cs +++ /dev/null @@ -1,57 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using OpenTelemetry; -using OpenTelemetry.Metrics; -using OpenTelemetry.Trace; - -namespace Microsoft.Extensions.Hosting; - -public static class Extensions -{ - public static TBuilder AddServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder - { - builder.ConfigureOpenTelemetry(); - - return builder; - } - - public static TBuilder ConfigureOpenTelemetry(this TBuilder builder) where TBuilder : IHostApplicationBuilder - { - builder.Logging.AddOpenTelemetry(logging => - { - logging.IncludeFormattedMessage = true; - logging.IncludeScopes = true; - }); - - builder.Services.AddOpenTelemetry() - .WithMetrics(metrics => - { - metrics.AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddRuntimeInstrumentation() - .AddMeter("Foundatio"); - }) - .WithTracing(tracing => - { - tracing.AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddSource("Foundatio"); - }); - - builder.AddOpenTelemetryExporters(); - - return builder; - } - - private static TBuilder AddOpenTelemetryExporters(this TBuilder builder) where TBuilder : IHostApplicationBuilder - { - var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); - - if (useOtlpExporter) - { - builder.Services.AddOpenTelemetry().UseOtlpExporter(); - } - - return builder; - } -} diff --git a/samples/Foundatio.HostingSample/Startup/MyStartupAction.cs b/samples/Foundatio.HostingSample/Startup/MyStartupAction.cs deleted file mode 100644 index 8a87e059d..000000000 --- a/samples/Foundatio.HostingSample/Startup/MyStartupAction.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Caching; -using Foundatio.Extensions.Hosting.Jobs; -using Foundatio.Extensions.Hosting.Startup; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -public class MyStartupAction : IStartupAction -{ - private readonly IJobManager _jobManager; - private readonly ICacheClient _cacheClient; - private readonly ILogger _logger; - - public MyStartupAction(IJobManager jobManager, ICacheClient cacheClient, ILogger logger) - { - _jobManager = jobManager; - _cacheClient = cacheClient; - _logger = logger; - } - - public async Task RunAsync(CancellationToken cancellationToken = default) - { - // set next run to be far in the past so it runs immediately - await _cacheClient.SetAsync("jobs:every_minute:nextrun", DateTime.UtcNow.AddDays(-1)); - - for (int i = 0; i < 5; i++) - { - _logger.LogTrace("MyStartupAction Run Thread={ManagedThreadId}", Thread.CurrentThread.ManagedThreadId); - await Task.Delay(500); - } - - _jobManager.AddOrUpdate("MyJob", j => j.CronSchedule("* * * * *").JobAction(async () => - { - _logger.LogInformation("Running MyJob"); - await Task.Delay(1000); - _logger.LogInformation("MyJob Complete"); - })); - } -} diff --git a/samples/Foundatio.HostingSample/Startup/OtherStartupAction.cs b/samples/Foundatio.HostingSample/Startup/OtherStartupAction.cs deleted file mode 100644 index 96cce9a33..000000000 --- a/samples/Foundatio.HostingSample/Startup/OtherStartupAction.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Extensions.Hosting.Startup; -using Microsoft.Extensions.Logging; - -namespace Foundatio.HostingSample; - -public class OtherStartupAction : IStartupAction -{ - private readonly ILogger _logger; - - public OtherStartupAction(ILogger logger) - { - _logger = logger; - } - - public async Task RunAsync(CancellationToken cancellationToken = default) - { - for (int i = 0; i < 5; i++) - { - _logger.LogTrace("OtherStartupAction Run Thread={ManagedThreadId}", Thread.CurrentThread.ManagedThreadId); - await Task.Delay(900); - } - } -} diff --git a/samples/Foundatio.HostingSample/appsettings.Development.json b/samples/Foundatio.HostingSample/appsettings.Development.json deleted file mode 100644 index a34cd70c5..000000000 --- a/samples/Foundatio.HostingSample/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/samples/Foundatio.HostingSample/appsettings.json b/samples/Foundatio.HostingSample/appsettings.json deleted file mode 100644 index 49bf8624f..000000000 --- a/samples/Foundatio.HostingSample/appsettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "Foundatio": "Information" - } - }, - "AllowedHosts": "*" -} From bdddc0a78bb219642a36dda67109f14d4a9e561d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 11:49:28 -0500 Subject: [PATCH 31/57] Isolate legacy message bus into Foundatio.Messaging.Legacy namespace Move the legacy IMessageBus surface (IMessageBus/IMessagePublisher/ IMessageSubscriber, MessageBusBase, InMemory/Null bus, Message, Shared/InMemory options) into Foundatio.Messaging.Legacy so it no longer collides with the redesigned messaging contract (IMessageTransport/IQueue/ IPubSub) that keeps the Foundatio.Messaging namespace. Referencers that still consume the legacy bus (HybridCacheClient, CacheLockProvider, WorkItemJob, the DI extensions, the hosting scheduled-job services, and the test harness/tests) gain a `using Foundatio.Messaging.Legacy;`. Repoint the Xunit logger-base doc crefs at the new IPubSub.SubscribeAsync{T}. No behavior change: full solution builds clean; the in-memory messaging, hybrid-cache, lock, and work-item-job suites stay green (448 passed, 1 skip). Co-Authored-By: Claude Opus 4.8 --- src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs | 1 + src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs | 1 + src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs | 1 + src/Foundatio.TestHarness/Jobs/WithLockingJob.cs | 1 + src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs | 1 + src/Foundatio.TestHarness/Queue/QueueTestBase.cs | 1 + src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs | 2 +- src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs | 2 +- src/Foundatio.Xunit/Logging/TestLoggerBase.cs | 2 +- src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs | 2 +- src/Foundatio/Caching/HybridAwareCacheClient.cs | 1 + src/Foundatio/Caching/HybridCacheClient.cs | 1 + src/Foundatio/FoundatioServicesExtensions.cs | 1 + src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs | 1 + src/Foundatio/Lock/CacheLockProvider.cs | 1 + src/Foundatio/Messaging/IMessageBus.cs | 2 +- src/Foundatio/Messaging/IMessagePublisher.cs | 2 +- src/Foundatio/Messaging/IMessageSubscriber.cs | 2 +- src/Foundatio/Messaging/InMemoryMessageBus.cs | 2 +- src/Foundatio/Messaging/InMemoryMessageBusOptions.cs | 2 +- src/Foundatio/Messaging/Message.cs | 2 +- src/Foundatio/Messaging/MessageBusBase.cs | 2 +- src/Foundatio/Messaging/NullMessageBus.cs | 2 +- src/Foundatio/Messaging/SharedMessageBusOptions.cs | 2 +- tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs | 1 + tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs | 1 + tests/Foundatio.Tests/Locks/InMemoryLockTests.cs | 1 + tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs | 1 + tests/Foundatio.Tests/Messaging/MessageTests.cs | 1 + tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs | 1 + 30 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs index 7ba51027a..4ddb44601 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs @@ -9,6 +9,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs index b7b4ea619..a35564a4d 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs @@ -5,6 +5,7 @@ using Foundatio.Caching; using Foundatio.Extensions.Hosting.Startup; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; diff --git a/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs b/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs index 5963d16c9..01cebc5ca 100644 --- a/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs +++ b/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs @@ -4,6 +4,7 @@ using Foundatio.AsyncEx; using Foundatio.Caching; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Microsoft.Extensions.Logging; using Xunit; diff --git a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs index 9bbfd120f..c56e4e995 100644 --- a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs @@ -5,6 +5,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Microsoft.Extensions.Logging; using Xunit; diff --git a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs index 523fa5bf3..38abf2689 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs @@ -7,6 +7,7 @@ using Exceptionless; using Foundatio.AsyncEx; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Foundatio.Tests.Serializer; using Foundatio.Tests.Utility; diff --git a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs index 9c8ed42df..0c7a7922c 100644 --- a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs +++ b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs @@ -10,6 +10,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Serializer; using Foundatio.Tests.Extensions; diff --git a/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs b/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs index 58597e3bd..7e0fc9fe2 100644 --- a/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs +++ b/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs @@ -27,7 +27,7 @@ protected TestLoggerBase(ITestOutputHelper output, TestLoggerFixture fixture) /// /// Gets a cancellation token that is cancelled when the current test completes or /// when the test run is aborted/timed out. Pass this token to - /// + /// /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs b/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs index 9dc993a44..e4ffd1b7d 100644 --- a/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs +++ b/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs @@ -22,7 +22,7 @@ protected TestWithLoggingBase(ITestOutputHelper output) /// /// Gets a cancellation token that is cancelled when the current test completes or /// when the test run is aborted/timed out. Pass this token to - /// + /// /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit/Logging/TestLoggerBase.cs b/src/Foundatio.Xunit/Logging/TestLoggerBase.cs index 2e1b29c02..6fca07cb6 100644 --- a/src/Foundatio.Xunit/Logging/TestLoggerBase.cs +++ b/src/Foundatio.Xunit/Logging/TestLoggerBase.cs @@ -26,7 +26,7 @@ protected TestLoggerBase(ITestOutputHelper output, TestLoggerFixture fixture) /// /// Gets a cancellation token that is cancelled when the current test completes. - /// Pass this token to + /// Pass this token to /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs b/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs index 4189b92c0..16fe79d70 100644 --- a/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs +++ b/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs @@ -21,7 +21,7 @@ protected TestWithLoggingBase(ITestOutputHelper output) /// /// Gets a cancellation token that is cancelled when the current test completes. - /// Pass this token to + /// Pass this token to /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio/Caching/HybridAwareCacheClient.cs b/src/Foundatio/Caching/HybridAwareCacheClient.cs index 81532fc98..4cda2aa9d 100644 --- a/src/Foundatio/Caching/HybridAwareCacheClient.cs +++ b/src/Foundatio/Caching/HybridAwareCacheClient.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Threading.Tasks; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio/Caching/HybridCacheClient.cs b/src/Foundatio/Caching/HybridCacheClient.cs index e248b49e1..596aef5d0 100644 --- a/src/Foundatio/Caching/HybridCacheClient.cs +++ b/src/Foundatio/Caching/HybridCacheClient.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Foundatio.AsyncEx; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index d42d277ff..787ee32a0 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -5,6 +5,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Resilience; using Foundatio.Serializer; diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs index 5d37547e6..e2a70a967 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Serializer; using Foundatio.Utility; diff --git a/src/Foundatio/Lock/CacheLockProvider.cs b/src/Foundatio/Lock/CacheLockProvider.cs index aae3d4d14..5edcf2fb2 100644 --- a/src/Foundatio/Lock/CacheLockProvider.cs +++ b/src/Foundatio/Lock/CacheLockProvider.cs @@ -7,6 +7,7 @@ using Foundatio.AsyncEx; using Foundatio.Caching; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio/Messaging/IMessageBus.cs b/src/Foundatio/Messaging/IMessageBus.cs index 1567da0d5..90cc8e056 100644 --- a/src/Foundatio/Messaging/IMessageBus.cs +++ b/src/Foundatio/Messaging/IMessageBus.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; /// /// Represents a message bus that supports both publishing and subscribing to messages. diff --git a/src/Foundatio/Messaging/IMessagePublisher.cs b/src/Foundatio/Messaging/IMessagePublisher.cs index 1961d85f0..531a4efcd 100644 --- a/src/Foundatio/Messaging/IMessagePublisher.cs +++ b/src/Foundatio/Messaging/IMessagePublisher.cs @@ -2,7 +2,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; /// /// Publishes messages to all subscribers listening for the message type. diff --git a/src/Foundatio/Messaging/IMessageSubscriber.cs b/src/Foundatio/Messaging/IMessageSubscriber.cs index a5c107d01..0d2ec7279 100644 --- a/src/Foundatio/Messaging/IMessageSubscriber.cs +++ b/src/Foundatio/Messaging/IMessageSubscriber.cs @@ -2,7 +2,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; /// /// Subscribes to messages published on the message bus. diff --git a/src/Foundatio/Messaging/InMemoryMessageBus.cs b/src/Foundatio/Messaging/InMemoryMessageBus.cs index 7cd2132e4..a7a511f1c 100644 --- a/src/Foundatio/Messaging/InMemoryMessageBus.cs +++ b/src/Foundatio/Messaging/InMemoryMessageBus.cs @@ -6,7 +6,7 @@ using Foundatio.Utility; using Microsoft.Extensions.Logging; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; public class InMemoryMessageBus : MessageBusBase { diff --git a/src/Foundatio/Messaging/InMemoryMessageBusOptions.cs b/src/Foundatio/Messaging/InMemoryMessageBusOptions.cs index 1387eee31..7a7433614 100644 --- a/src/Foundatio/Messaging/InMemoryMessageBusOptions.cs +++ b/src/Foundatio/Messaging/InMemoryMessageBusOptions.cs @@ -1,4 +1,4 @@ -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; public class InMemoryMessageBusOptions : SharedMessageBusOptions { } diff --git a/src/Foundatio/Messaging/Message.cs b/src/Foundatio/Messaging/Message.cs index eeaad8fc5..e943c9962 100644 --- a/src/Foundatio/Messaging/Message.cs +++ b/src/Foundatio/Messaging/Message.cs @@ -3,7 +3,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; /// /// Represents a message received from the message bus with metadata and raw payload. diff --git a/src/Foundatio/Messaging/MessageBusBase.cs b/src/Foundatio/Messaging/MessageBusBase.cs index aff97ce4b..85b1943b2 100644 --- a/src/Foundatio/Messaging/MessageBusBase.cs +++ b/src/Foundatio/Messaging/MessageBusBase.cs @@ -13,7 +13,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; public abstract class MessageBusBase : IMessageBus, IHaveLogger, IHaveLoggerFactory, IHaveTimeProvider, IHaveResiliencePolicyProvider, IDisposable, IAsyncDisposable where TOptions : SharedMessageBusOptions { diff --git a/src/Foundatio/Messaging/NullMessageBus.cs b/src/Foundatio/Messaging/NullMessageBus.cs index cee2fa42a..2eeb6dba6 100644 --- a/src/Foundatio/Messaging/NullMessageBus.cs +++ b/src/Foundatio/Messaging/NullMessageBus.cs @@ -2,7 +2,7 @@ using System.Threading; using System.Threading.Tasks; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; public class NullMessageBus : IMessageBus { diff --git a/src/Foundatio/Messaging/SharedMessageBusOptions.cs b/src/Foundatio/Messaging/SharedMessageBusOptions.cs index 94a15423c..5f3bcc499 100644 --- a/src/Foundatio/Messaging/SharedMessageBusOptions.cs +++ b/src/Foundatio/Messaging/SharedMessageBusOptions.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -namespace Foundatio.Messaging; +namespace Foundatio.Messaging.Legacy; public class SharedMessageBusOptions : SharedOptions { diff --git a/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs b/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs index bf29b93c0..5a41bc927 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.Logging; using Xunit; diff --git a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs index 9b1cd4226..28eadbf68 100644 --- a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs +++ b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs @@ -9,6 +9,7 @@ using Foundatio.AsyncEx; using Foundatio.Jobs; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Tests.Extensions; using Foundatio.Xunit; diff --git a/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs b/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs index de9c2d073..645d9c847 100644 --- a/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs +++ b/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs @@ -3,6 +3,7 @@ using Foundatio.Caching; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Xunit; namespace Foundatio.Tests.Locks; diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs index d94a0a9c8..7f2010848 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using Foundatio.AsyncEx; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Xunit; diff --git a/tests/Foundatio.Tests/Messaging/MessageTests.cs b/tests/Foundatio.Tests/Messaging/MessageTests.cs index 67cc60a76..67de43086 100644 --- a/tests/Foundatio.Tests/Messaging/MessageTests.cs +++ b/tests/Foundatio.Tests/Messaging/MessageTests.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.InteropServices; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Xunit; namespace Foundatio.Tests.Messaging; diff --git a/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs b/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs index 95ac75e3e..e698317f7 100644 --- a/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs +++ b/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs @@ -6,6 +6,7 @@ using Foundatio.Caching; using Foundatio.Lock; using Foundatio.Messaging; +using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; using Foundatio.Xunit; From d3c5e726831bd1039e07912c9a4e2effd8e33e61 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 12:02:04 -0500 Subject: [PATCH 32/57] Isolate legacy job types into Foundatio.Jobs.Legacy namespace Move the legacy job surface (JobRunner, JobBase/JobContext, JobOptions, IQueueJob/QueueJobBase/QueueEntryContext, JobWithLockBase, JobAttribute, and the WorkItemJob family) into Foundatio.Jobs.Legacy so the redesigned durable- jobs API (IJobRuntimeStore/IJobClient/IJobWorker/IJobScheduler, JobState, ScheduledJobDefinition, JobExecutionContext) owns a clean Foundatio.Jobs surface. IJob.cs is split: the shared IJob contract, the new IJobWithExecutionContext, and the shared TryRunAsync helper (used by the new JobWorker) stay in Foundatio.Jobs; the legacy IJobWithOptions and the continuous-run RunContinuousAsync extensions move to Foundatio.Jobs.Legacy (LegacyJobRunExtensions). Referencers that still consume the legacy job types (the hosting job infra, the test-harness job/queue fixtures, and the job tests) gain a `using Foundatio.Jobs.Legacy;`. No behavior change: full solution builds clean; the Jobs test namespace stays green (56 passed, 1 manual-only skip). Co-Authored-By: Claude Opus 4.8 --- .../Jobs/HostedJobOptions.cs | 2 +- .../Jobs/HostedJobService.cs | 1 + .../Jobs/JobHostExtensions.cs | 1 + .../Jobs/JobManager.cs | 1 + .../Jobs/JobOptionsBuilder.cs | 1 + .../Jobs/HelloWorldJob.cs | 1 + .../Jobs/JobQueueTestsBase.cs | 1 + .../Jobs/SampleQueueJob.cs | 1 + .../Jobs/ThrottledJob.cs | 1 + .../Jobs/WithDependencyJob.cs | 1 + .../Jobs/WithLockingJob.cs | 1 + .../Queue/QueueTestBase.cs | 1 + src/Foundatio/Jobs/IJob.cs | 108 +--------------- src/Foundatio/Jobs/IQueueJob.cs | 2 +- src/Foundatio/Jobs/JobAttribute.cs | 2 +- src/Foundatio/Jobs/JobBase.cs | 2 +- src/Foundatio/Jobs/JobContext.cs | 2 +- src/Foundatio/Jobs/JobOptions.cs | 2 +- src/Foundatio/Jobs/JobRunner.cs | 2 +- src/Foundatio/Jobs/JobWithLockBase.cs | 2 +- src/Foundatio/Jobs/LegacyJobRunExtensions.cs | 115 ++++++++++++++++++ src/Foundatio/Jobs/QueueEntryContext.cs | 2 +- src/Foundatio/Jobs/QueueJobBase.cs | 2 +- .../Jobs/WorkItemJob/WorkItemContext.cs | 2 +- .../Jobs/WorkItemJob/WorkItemData.cs | 2 +- .../Jobs/WorkItemJob/WorkItemHandlers.cs | 2 +- src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs | 2 +- .../WorkItemJob/WorkItemQueueExtensions.cs | 2 +- .../Jobs/WorkItemJob/WorkItemStatus.cs | 2 +- tests/Foundatio.Tests/Jobs/JobTests.cs | 1 + .../Foundatio.Tests/Jobs/WorkItemJobTests.cs | 1 + 31 files changed, 145 insertions(+), 123 deletions(-) create mode 100644 src/Foundatio/Jobs/LegacyJobRunExtensions.cs diff --git a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs index ce9488ed7..c0055d78c 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs @@ -2,7 +2,7 @@ namespace Foundatio.Extensions.Hosting.Jobs; -public class HostedJobOptions : JobOptions +public class HostedJobOptions : Foundatio.Jobs.Legacy.JobOptions { public bool WaitForStartupActions { get; set; } } diff --git a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs index d5d7b2eba..31f564ce3 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Foundatio.Extensions.Hosting.Startup; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index 9d1be3fd4..c84d81d94 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs index d84fe2d8d..166e60aff 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs index 9adb13a94..e605d6b00 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs @@ -1,5 +1,6 @@ using System; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; namespace Foundatio.Extensions.Hosting.Jobs; diff --git a/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs b/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs index b79a2a0ff..7e7161a67 100644 --- a/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs +++ b/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Microsoft.Extensions.Logging; namespace Foundatio.Tests.Jobs; diff --git a/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs b/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs index 3d1844582..281974130 100644 --- a/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs +++ b/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs @@ -7,6 +7,7 @@ using Exceptionless; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Queues; using Foundatio.Xunit; diff --git a/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs b/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs index c5dfde3de..0fccf829d 100644 --- a/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs +++ b/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Exceptionless; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Queues; using Foundatio.Resilience; diff --git a/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs b/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs index bdbfb56af..3a2de04dc 100644 --- a/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs +++ b/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs b/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs index 054b123a0..a8188fec0 100644 --- a/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Microsoft.Extensions.Logging; namespace Foundatio.Tests.Jobs; diff --git a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs index c56e4e995..a877d48b7 100644 --- a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Messaging; using Foundatio.Messaging.Legacy; diff --git a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs index 0c7a7922c..e70a8a2c4 100644 --- a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs +++ b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs @@ -8,6 +8,7 @@ using Foundatio.AsyncEx; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Messaging; using Foundatio.Messaging.Legacy; diff --git a/src/Foundatio/Jobs/IJob.cs b/src/Foundatio/Jobs/IJob.cs index 5d222322d..bf5c8ef70 100644 --- a/src/Foundatio/Jobs/IJob.cs +++ b/src/Foundatio/Jobs/IJob.cs @@ -1,9 +1,7 @@ using System; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Foundatio.Utility; -using Microsoft.Extensions.Logging; namespace Foundatio.Jobs; @@ -21,22 +19,11 @@ public interface IJob Task RunAsync(CancellationToken cancellationToken = default); } -/// -/// A job that exposes configurable options for execution behavior. -/// -public interface IJobWithOptions : IJob -{ - /// - /// Gets or sets the options controlling job execution (name, interval, iteration limit). - /// - JobOptions? Options { get; set; } -} - /// /// A durable job that wants its — job id, attempt number, and store-backed progress, /// lease heartbeat, and cooperative cancellation checks. The durable runtime sets on the /// job instance before invoking it. Jobs that use the context should be registered as transient (a fresh instance per -/// run), since the context is per-run state, matching . +/// run), since the context is per-run state, matching . /// public interface IJobWithExecutionContext : IJob { @@ -60,97 +47,4 @@ public static async Task TryRunAsync(this IJob job, CancellationToken return JobResult.FromException(ex); } } - - /// - /// Runs the job continuously until the cancellation token is set or the iteration limit is reached. - /// - /// Returns the iteration count for normal jobs. For queue-based jobs this will be the number of items processed successfully. - public static Task RunContinuousAsync(this IJob job, TimeSpan? interval = null, int iterationLimit = -1, - CancellationToken cancellationToken = default, Func>? continuationCallback = null) - { - var options = JobOptions.GetDefaults(job); - options.Interval = interval; - options.IterationLimit = iterationLimit; - return RunContinuousAsync(job, options, cancellationToken, continuationCallback); - } - - /// - /// Runs the job continuously until the cancellation token is set or the iteration limit is reached. - /// - /// Returns the iteration count for normal jobs. For queue based jobs this will be the amount of items processed successfully. - public static async Task RunContinuousAsync(this IJob job, JobOptions options, CancellationToken cancellationToken = default, Func>? continuationCallback = null) - { - int iterations = 0; - var logger = job.GetLogger(); - - int queueItemsProcessed = 0; - bool isQueueJob = job.GetType().GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IQueueJob<>)); - - string jobId = Guid.NewGuid().ToString("N").Substring(0, 10); - using var jobScope = logger.BeginScope(s => s.Property("job.name", options.Name ?? String.Empty).Property("job.id", jobId)); - logger.LogInformation("Starting continuous job type {JobName} on machine {MachineName}...", options.Name, Environment.MachineName); - - while (!cancellationToken.IsCancellationRequested) - { - using var activity = FoundatioDiagnostics.ActivitySource.StartActivity($"Job: {options.Name}"); - - string jobRunId = Guid.NewGuid().ToString("N").Substring(0, 10); - using var _ = logger.BeginScope(s => s.Property("job.run_id", jobRunId)); - var result = await job.TryRunAsync(cancellationToken).AnyContext(); - logger.LogJobResult(result, options.Name); - - iterations++; - if (isQueueJob && result.IsSuccess) - queueItemsProcessed++; - - if (cancellationToken.IsCancellationRequested || (options.IterationLimit > -1 && options.IterationLimit <= iterations)) - break; - - if (result.Error != null) - { - await job.GetTimeProvider().SafeDelay(TimeSpan.FromMilliseconds(Math.Max((int)(options.Interval?.TotalMilliseconds ?? 0), 100)), cancellationToken).AnyContext(); - } - else if (options.Interval.HasValue && options.Interval.Value > TimeSpan.Zero) - { - await job.GetTimeProvider().SafeDelay(options.Interval.Value, cancellationToken).AnyContext(); - } - - // needed to yield back a task for jobs that aren't async - await Task.Yield(); - - if (cancellationToken.IsCancellationRequested) - break; - - if (continuationCallback is null) - continue; - - try - { - if (!await continuationCallback().AnyContext()) - break; - } - catch (Exception ex) - { - logger.LogError(ex, "Error in continuation callback: {Message}", ex.Message); - } - } - - if (cancellationToken.IsCancellationRequested) - logger.LogTrace("Job cancellation requested"); - - if (options.IterationLimit > 0) - { - logger.LogInformation( - "Stopping continuous job type {JobName} on machine {MachineName}: Job ran {Iterations} times (Limit={IterationLimit})", - options.Name, Environment.MachineName, iterations, options.IterationLimit); - } - else - { - logger.LogInformation( - "Stopping continuous job type {JobName} on machine {MachineName}: Job ran {Iterations} times", - options.Name, Environment.MachineName, iterations); - } - - return isQueueJob ? queueItemsProcessed : iterations; - } } diff --git a/src/Foundatio/Jobs/IQueueJob.cs b/src/Foundatio/Jobs/IQueueJob.cs index 2f396e3b2..ef7eafb76 100644 --- a/src/Foundatio/Jobs/IQueueJob.cs +++ b/src/Foundatio/Jobs/IQueueJob.cs @@ -5,7 +5,7 @@ using Foundatio.Utility; using Microsoft.Extensions.Logging; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; /// /// A job that processes items from a queue. Each invocation of diff --git a/src/Foundatio/Jobs/JobAttribute.cs b/src/Foundatio/Jobs/JobAttribute.cs index 8b32f5ab4..b39b99fdc 100644 --- a/src/Foundatio/Jobs/JobAttribute.cs +++ b/src/Foundatio/Jobs/JobAttribute.cs @@ -1,6 +1,6 @@ using System; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = true)] public class JobAttribute : Attribute diff --git a/src/Foundatio/Jobs/JobBase.cs b/src/Foundatio/Jobs/JobBase.cs index ead6e3b48..e5b4711ec 100644 --- a/src/Foundatio/Jobs/JobBase.cs +++ b/src/Foundatio/Jobs/JobBase.cs @@ -6,7 +6,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public abstract class JobBase : IJob, IHaveLogger, IHaveLoggerFactory, IHaveTimeProvider, IHaveResiliencePolicyProvider { diff --git a/src/Foundatio/Jobs/JobContext.cs b/src/Foundatio/Jobs/JobContext.cs index c955f2657..2550bbdd6 100644 --- a/src/Foundatio/Jobs/JobContext.cs +++ b/src/Foundatio/Jobs/JobContext.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using Foundatio.Lock; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class JobContext { diff --git a/src/Foundatio/Jobs/JobOptions.cs b/src/Foundatio/Jobs/JobOptions.cs index 8960a0fbd..c0d175687 100644 --- a/src/Foundatio/Jobs/JobOptions.cs +++ b/src/Foundatio/Jobs/JobOptions.cs @@ -3,7 +3,7 @@ using Foundatio.Extensions; using Foundatio.Utility; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class JobOptions { diff --git a/src/Foundatio/Jobs/JobRunner.cs b/src/Foundatio/Jobs/JobRunner.cs index 0437123f4..c7f121028 100644 --- a/src/Foundatio/Jobs/JobRunner.cs +++ b/src/Foundatio/Jobs/JobRunner.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class JobRunner { diff --git a/src/Foundatio/Jobs/JobWithLockBase.cs b/src/Foundatio/Jobs/JobWithLockBase.cs index 8af43866d..44d9af51a 100644 --- a/src/Foundatio/Jobs/JobWithLockBase.cs +++ b/src/Foundatio/Jobs/JobWithLockBase.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public abstract class JobWithLockBase : IJobWithOptions, IHaveLogger, IHaveLoggerFactory, IHaveTimeProvider, IHaveResiliencePolicyProvider { diff --git a/src/Foundatio/Jobs/LegacyJobRunExtensions.cs b/src/Foundatio/Jobs/LegacyJobRunExtensions.cs new file mode 100644 index 000000000..c73321158 --- /dev/null +++ b/src/Foundatio/Jobs/LegacyJobRunExtensions.cs @@ -0,0 +1,115 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; +using Microsoft.Extensions.Logging; + +namespace Foundatio.Jobs.Legacy; + +/// +/// A job that exposes configurable options for execution behavior. +/// +public interface IJobWithOptions : IJob +{ + /// + /// Gets or sets the options controlling job execution (name, interval, iteration limit). + /// + JobOptions? Options { get; set; } +} + +public static class LegacyJobExtensions +{ + /// + /// Runs the job continuously until the cancellation token is set or the iteration limit is reached. + /// + /// Returns the iteration count for normal jobs. For queue-based jobs this will be the number of items processed successfully. + public static Task RunContinuousAsync(this IJob job, TimeSpan? interval = null, int iterationLimit = -1, + CancellationToken cancellationToken = default, Func>? continuationCallback = null) + { + var options = JobOptions.GetDefaults(job); + options.Interval = interval; + options.IterationLimit = iterationLimit; + return RunContinuousAsync(job, options, cancellationToken, continuationCallback); + } + + /// + /// Runs the job continuously until the cancellation token is set or the iteration limit is reached. + /// + /// Returns the iteration count for normal jobs. For queue based jobs this will be the amount of items processed successfully. + public static async Task RunContinuousAsync(this IJob job, JobOptions options, CancellationToken cancellationToken = default, Func>? continuationCallback = null) + { + int iterations = 0; + var logger = job.GetLogger(); + + int queueItemsProcessed = 0; + bool isQueueJob = job.GetType().GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(IQueueJob<>)); + + string jobId = Guid.NewGuid().ToString("N").Substring(0, 10); + using var jobScope = logger.BeginScope(s => s.Property("job.name", options.Name ?? String.Empty).Property("job.id", jobId)); + logger.LogInformation("Starting continuous job type {JobName} on machine {MachineName}...", options.Name, Environment.MachineName); + + while (!cancellationToken.IsCancellationRequested) + { + using var activity = FoundatioDiagnostics.ActivitySource.StartActivity($"Job: {options.Name}"); + + string jobRunId = Guid.NewGuid().ToString("N").Substring(0, 10); + using var _ = logger.BeginScope(s => s.Property("job.run_id", jobRunId)); + var result = await job.TryRunAsync(cancellationToken).AnyContext(); + logger.LogJobResult(result, options.Name); + + iterations++; + if (isQueueJob && result.IsSuccess) + queueItemsProcessed++; + + if (cancellationToken.IsCancellationRequested || (options.IterationLimit > -1 && options.IterationLimit <= iterations)) + break; + + if (result.Error != null) + { + await job.GetTimeProvider().SafeDelay(TimeSpan.FromMilliseconds(Math.Max((int)(options.Interval?.TotalMilliseconds ?? 0), 100)), cancellationToken).AnyContext(); + } + else if (options.Interval.HasValue && options.Interval.Value > TimeSpan.Zero) + { + await job.GetTimeProvider().SafeDelay(options.Interval.Value, cancellationToken).AnyContext(); + } + + // needed to yield back a task for jobs that aren't async + await Task.Yield(); + + if (cancellationToken.IsCancellationRequested) + break; + + if (continuationCallback is null) + continue; + + try + { + if (!await continuationCallback().AnyContext()) + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Error in continuation callback: {Message}", ex.Message); + } + } + + if (cancellationToken.IsCancellationRequested) + logger.LogTrace("Job cancellation requested"); + + if (options.IterationLimit > 0) + { + logger.LogInformation( + "Stopping continuous job type {JobName} on machine {MachineName}: Job ran {Iterations} times (Limit={IterationLimit})", + options.Name, Environment.MachineName, iterations, options.IterationLimit); + } + else + { + logger.LogInformation( + "Stopping continuous job type {JobName} on machine {MachineName}: Job ran {Iterations} times", + options.Name, Environment.MachineName, iterations); + } + + return isQueueJob ? queueItemsProcessed : iterations; + } +} diff --git a/src/Foundatio/Jobs/QueueEntryContext.cs b/src/Foundatio/Jobs/QueueEntryContext.cs index 04ba268a8..dce43f5ef 100644 --- a/src/Foundatio/Jobs/QueueEntryContext.cs +++ b/src/Foundatio/Jobs/QueueEntryContext.cs @@ -4,7 +4,7 @@ using Foundatio.Queues; using Foundatio.Utility; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class QueueEntryContext : JobContext where T : class { diff --git a/src/Foundatio/Jobs/QueueJobBase.cs b/src/Foundatio/Jobs/QueueJobBase.cs index fb1f520d4..e0f451eaa 100644 --- a/src/Foundatio/Jobs/QueueJobBase.cs +++ b/src/Foundatio/Jobs/QueueJobBase.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public abstract class QueueJobBase : IQueueJob, IHaveLogger, IHaveLoggerFactory, IHaveTimeProvider, IHaveResiliencePolicyProvider where T : class { diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemContext.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemContext.cs index 355d779ed..7d043ef8a 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemContext.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemContext.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Foundatio.Lock; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class WorkItemContext { diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemData.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemData.cs index 987ee5d32..d0004930e 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemData.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemData.cs @@ -1,7 +1,7 @@ using Foundatio.Metrics; using Foundatio.Queues; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class WorkItemData : IHaveSubMetricName, IHaveUniqueIdentifier { diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemHandlers.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemHandlers.cs index 0b9082673..48158268c 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemHandlers.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemHandlers.cs @@ -8,7 +8,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class WorkItemHandlers { diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs index e2a70a967..129e71d17 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs @@ -11,7 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; [Job(Description = "Processes adhoc work item queues entries")] public class WorkItemJob : IQueueJob, IHaveLogger, IHaveLoggerFactory diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemQueueExtensions.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemQueueExtensions.cs index 72f3704b8..c89dbcf91 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemQueueExtensions.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemQueueExtensions.cs @@ -5,7 +5,7 @@ using Foundatio.Serializer; using Foundatio.Utility; -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public static class WorkItemQueueExtensions { diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemStatus.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemStatus.cs index 1ddad0248..433858b09 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemStatus.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemStatus.cs @@ -1,4 +1,4 @@ -namespace Foundatio.Jobs; +namespace Foundatio.Jobs.Legacy; public class WorkItemStatus { diff --git a/tests/Foundatio.Tests/Jobs/JobTests.cs b/tests/Foundatio.Tests/Jobs/JobTests.cs index 93e12c694..d3a04f974 100644 --- a/tests/Foundatio.Tests/Jobs/JobTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Xunit; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs index 28eadbf68..75906b43e 100644 --- a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs +++ b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs @@ -8,6 +8,7 @@ using Exceptionless; using Foundatio.AsyncEx; using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Queues; From 4a289b56e5dc774636f964359b2639fdc36663ce Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 12:07:53 -0500 Subject: [PATCH 33/57] Isolate legacy hosting job infra into Foundatio.Extensions.Hosting.Jobs.Legacy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the legacy in-process hosting job types — JobManager/IJobManager, the HostedJobService runner, the in-process ScheduledJobService CRON scheduler and its ScheduledJobInstance state, Cron, DynamicJob, and the HostedJob/ScheduledJob options+builders+registration — into Foundatio.Extensions.Hosting.Jobs.Legacy. Split JobHostExtensions: the new durable-runtime entry point (AddJobRuntimeService) stays in Foundatio.Extensions.Hosting.Jobs; the legacy registration API (AddJob/AddCronJob/AddDistributedCronJob/AddJobScheduler/ AddJobLifetimeService) moves to LegacyJobHostExtensions in the .Legacy namespace. The one durable-runtime bridge, JobRuntimeService, also stays. No external consumers referenced the legacy hosting types or DI methods, so the only fixups were an internal cref. No behavior change: full solution builds clean; the Jobs test namespace stays green (56 passed, 1 manual-only skip). Co-Authored-By: Claude Opus 4.8 --- src/Foundatio.Extensions.Hosting/Jobs/Cron.cs | 2 +- .../Jobs/DynamicJob.cs | 2 +- .../Jobs/HostedJobOptions.cs | 2 +- .../Jobs/HostedJobService.cs | 2 +- .../Jobs/JobHostExtensions.cs | 210 +---------------- .../Jobs/JobManager.cs | 2 +- .../Jobs/JobOptionsBuilder.cs | 2 +- .../Jobs/LegacyJobHostExtensions.cs | 217 ++++++++++++++++++ .../Jobs/ScheduledJobInstance.cs | 2 +- .../Jobs/ScheduledJobOptions.cs | 2 +- .../Jobs/ScheduledJobOptionsBuilder.cs | 2 +- .../Jobs/ScheduledJobRegistration.cs | 2 +- .../Jobs/ScheduledJobService.cs | 4 +- .../ShutdownHostIfNoJobsRunningService.cs | 2 +- 14 files changed, 231 insertions(+), 222 deletions(-) create mode 100644 src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs diff --git a/src/Foundatio.Extensions.Hosting/Jobs/Cron.cs b/src/Foundatio.Extensions.Hosting/Jobs/Cron.cs index ffc193481..e13bbcb5a 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/Cron.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/Cron.cs @@ -15,7 +15,7 @@ using System; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; /// /// Helper class that provides common values for the cron expressions. diff --git a/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs b/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs index 1eda91928..48ce072cd 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs @@ -4,7 +4,7 @@ using Foundatio.Jobs; using Foundatio.Utility; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; internal class DynamicJob : IJob { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs index c0055d78c..664065cf1 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobOptions.cs @@ -1,6 +1,6 @@ using Foundatio.Jobs; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class HostedJobOptions : Foundatio.Jobs.Legacy.JobOptions { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs index 31f564ce3..0105be37f 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class HostedJobService : IHostedService, IJobStatus, IDisposable { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs index c84d81d94..9d53b68bf 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobHostExtensions.cs @@ -1,212 +1,11 @@ -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System; using Foundatio.Jobs; -using Foundatio.Jobs.Legacy; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; namespace Foundatio.Extensions.Hosting.Jobs; public static class JobHostExtensions { - public static IServiceCollection AddJob(this IServiceCollection services, HostedJobOptions jobOptions) - { - if (jobOptions.JobFactory == null) - throw new ArgumentNullException(nameof(jobOptions), "jobOptions.JobFactory is required"); - - return services.AddTransient(s => new HostedJobService(s, jobOptions, s.GetRequiredService())); - } - - public static IServiceCollection AddJob(this IServiceCollection services, HostedJobOptions? jobOptions = null) where T : class, IJob - { - services.AddTransient(); - return services.AddTransient(s => - { - if (jobOptions == null) - { - jobOptions = new HostedJobOptions(); - jobOptions.ApplyDefaults(); - } - - jobOptions.Name ??= JobOptions.GetDefaultJobName(typeof(T)); - jobOptions.JobFactory ??= sp => sp.GetRequiredService(); - - return new HostedJobService(s, jobOptions, s.GetRequiredService()); - }); - } - - public static IServiceCollection AddJob(this IServiceCollection services, Action configureJobOptions) where T : class, IJob - { - var jobOptionsBuilder = new HostedJobOptionsBuilder(); - jobOptionsBuilder.ApplyDefaults(); - jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddJob(jobOptionsBuilder.Target); - } - - public static IServiceCollection AddJob(this IServiceCollection services, Action configureJobOptions) - { - var jobOptionsBuilder = new HostedJobOptionsBuilder(); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddJob(jobOptionsBuilder.Target); - } - - public static IServiceCollection AddJob(this IServiceCollection services, string name, Func jobFactory, Action configureJobOptions) - { - var jobOptionsBuilder = new HostedJobOptionsBuilder(); - jobOptionsBuilder.Name(name).JobFactory(jobFactory); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddJob(jobOptionsBuilder.Target); - } - - /// - /// Legacy/compat. This registers the in-process , which runs CRON occurrences - /// in-process and does not materialize durable, recoverable occurrences. The forward path in the redesigned runtime - /// is the durable scheduler — register it with services.AddFoundatio().Jobs.UseInMemoryRuntime() plus - /// , which materializes durable occurrences with retry, recovery, and - /// dead-lettering. Routing this default API onto the durable scheduler is a planned follow-up. - /// - public static IServiceCollection AddCronJob(this IServiceCollection services, ScheduledJobOptions jobOptions) - { - if (jobOptions.JobFactory == null) - throw new ArgumentNullException(nameof(jobOptions), "jobOptions.JobFactory is required"); - - services.AddJobScheduler(); - - return services.AddTransient(s => new ScheduledJobRegistration(jobOptions)); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, Action configureJobOptions) - { - var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddCronJob(jobOptionsBuilder.Target); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string cronSchedule, Action? configureJobOptions = null) where T : class, IJob - { - services.AddTransient(); - var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); - jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))).CronSchedule(cronSchedule).JobFactory(sp => sp.GetRequiredService()); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddCronJob(jobOptionsBuilder.Target); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, action))); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, _) => action(xp)))); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => action()))); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, ct) => - { - action(xp, ct); - return Task.CompletedTask; - }))); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, ct) => - { - action(ct); - return Task.CompletedTask; - }))); - } - - public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => - { - action(); - return Task.CompletedTask; - }))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string cronSchedule, Action? configureJobOptions = null) where T : class, IJob - { - services.AddTransient(); - var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); - jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))).Distributed().CronSchedule(cronSchedule).JobFactory(sp => sp.GetRequiredService()); - configureJobOptions?.Invoke(jobOptionsBuilder); - return services.AddCronJob(jobOptionsBuilder.Target); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, action))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, _) => action(xp)))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => action()))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, ct) => - { - action(xp, ct); - return Task.CompletedTask; - }))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, ct) => - { - action(ct); - return Task.CompletedTask; - }))); - } - - public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) - { - return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => - { - action(); - return Task.CompletedTask; - }))); - } - - /// - /// Legacy/compat: registers the in-process CRON scheduler. For durable, - /// recoverable CRON occurrences use the redesigned runtime (AddFoundatio().Jobs.UseInMemoryRuntime() + - /// ) instead. - /// - public static IServiceCollection AddJobScheduler(this IServiceCollection services) - { - if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(ScheduledJobService))) - services.AddTransient(); - - if (!services.Any(s => s.ServiceType == typeof(JobManager) && s.ImplementationType == typeof(JobManager))) - services.AddSingleton(); - - if (!services.Any(s => s.ServiceType == typeof(IJobManager) && s.ImplementationType == typeof(JobManager))) - services.AddSingleton(sp => sp.GetRequiredService()); - - return services; - } - /// /// Registers the hosted pump that drives the durable job runtime (): /// materializing CRON occurrences, dispatching delayed/scheduled work, recovering stale occurrences, and running @@ -232,11 +31,4 @@ public static IServiceCollection AddJobRuntimeService(this IServiceCollection se return services; } - - public static IServiceCollection AddJobLifetimeService(this IServiceCollection services) - { - services.AddSingleton(); - services.AddSingleton(x => x.GetRequiredService()); - return services; - } } diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs index 166e60aff..49c94cd3e 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs @@ -11,7 +11,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public interface IJobManager { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs index e605d6b00..d0f414239 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs @@ -2,7 +2,7 @@ using Foundatio.Jobs; using Foundatio.Jobs.Legacy; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class HostedJobOptionsBuilder { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs new file mode 100644 index 000000000..6f8633331 --- /dev/null +++ b/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs @@ -0,0 +1,217 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; + +public static class LegacyJobHostExtensions +{ + public static IServiceCollection AddJob(this IServiceCollection services, HostedJobOptions jobOptions) + { + if (jobOptions.JobFactory == null) + throw new ArgumentNullException(nameof(jobOptions), "jobOptions.JobFactory is required"); + + return services.AddTransient(s => new HostedJobService(s, jobOptions, s.GetRequiredService())); + } + + public static IServiceCollection AddJob(this IServiceCollection services, HostedJobOptions? jobOptions = null) where T : class, IJob + { + services.AddTransient(); + return services.AddTransient(s => + { + if (jobOptions == null) + { + jobOptions = new HostedJobOptions(); + jobOptions.ApplyDefaults(); + } + + jobOptions.Name ??= JobOptions.GetDefaultJobName(typeof(T)); + jobOptions.JobFactory ??= sp => sp.GetRequiredService(); + + return new HostedJobService(s, jobOptions, s.GetRequiredService()); + }); + } + + public static IServiceCollection AddJob(this IServiceCollection services, Action configureJobOptions) where T : class, IJob + { + var jobOptionsBuilder = new HostedJobOptionsBuilder(); + jobOptionsBuilder.ApplyDefaults(); + jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddJob(jobOptionsBuilder.Target); + } + + public static IServiceCollection AddJob(this IServiceCollection services, Action configureJobOptions) + { + var jobOptionsBuilder = new HostedJobOptionsBuilder(); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddJob(jobOptionsBuilder.Target); + } + + public static IServiceCollection AddJob(this IServiceCollection services, string name, Func jobFactory, Action configureJobOptions) + { + var jobOptionsBuilder = new HostedJobOptionsBuilder(); + jobOptionsBuilder.Name(name).JobFactory(jobFactory); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddJob(jobOptionsBuilder.Target); + } + + /// + /// Legacy/compat. This registers the in-process , which runs CRON occurrences + /// in-process and does not materialize durable, recoverable occurrences. The forward path in the redesigned runtime + /// is the durable scheduler — register it with services.AddFoundatio().Jobs.UseInMemoryRuntime() plus + /// , which materializes durable + /// occurrences with retry, recovery, and dead-lettering. Routing this default API onto the durable scheduler is a + /// planned follow-up. + /// + public static IServiceCollection AddCronJob(this IServiceCollection services, ScheduledJobOptions jobOptions) + { + if (jobOptions.JobFactory == null) + throw new ArgumentNullException(nameof(jobOptions), "jobOptions.JobFactory is required"); + + services.AddJobScheduler(); + + return services.AddTransient(s => new ScheduledJobRegistration(jobOptions)); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, Action configureJobOptions) + { + var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddCronJob(jobOptionsBuilder.Target); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string cronSchedule, Action? configureJobOptions = null) where T : class, IJob + { + services.AddTransient(); + var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); + jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))).CronSchedule(cronSchedule).JobFactory(sp => sp.GetRequiredService()); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddCronJob(jobOptionsBuilder.Target); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, action))); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, _) => action(xp)))); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => action()))); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, ct) => + { + action(xp, ct); + return Task.CompletedTask; + }))); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, ct) => + { + action(ct); + return Task.CompletedTask; + }))); + } + + public static IServiceCollection AddCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => + { + action(); + return Task.CompletedTask; + }))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string cronSchedule, Action? configureJobOptions = null) where T : class, IJob + { + services.AddTransient(); + var jobOptionsBuilder = new ScheduledJobOptionsBuilder(); + jobOptionsBuilder.Name(JobOptions.GetDefaultJobName(typeof(T))).Distributed().CronSchedule(cronSchedule).JobFactory(sp => sp.GetRequiredService()); + configureJobOptions?.Invoke(jobOptionsBuilder); + return services.AddCronJob(jobOptionsBuilder.Target); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, action))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, _) => action(xp)))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Func action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => action()))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (xp, ct) => + { + action(xp, ct); + return Task.CompletedTask; + }))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, ct) => + { + action(ct); + return Task.CompletedTask; + }))); + } + + public static IServiceCollection AddDistributedCronJob(this IServiceCollection services, string name, string cronSchedule, Action action) + { + return services.AddCronJob(o => o.Name(name).Distributed().CronSchedule(cronSchedule).JobFactory(sp => new DynamicJob(sp, (_, _) => + { + action(); + return Task.CompletedTask; + }))); + } + + /// + /// Legacy/compat: registers the in-process CRON scheduler. For durable, + /// recoverable CRON occurrences use the redesigned runtime (AddFoundatio().Jobs.UseInMemoryRuntime() + + /// ) instead. + /// + public static IServiceCollection AddJobScheduler(this IServiceCollection services) + { + if (!services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(ScheduledJobService))) + services.AddTransient(); + + if (!services.Any(s => s.ServiceType == typeof(JobManager) && s.ImplementationType == typeof(JobManager))) + services.AddSingleton(); + + if (!services.Any(s => s.ServiceType == typeof(IJobManager) && s.ImplementationType == typeof(JobManager))) + services.AddSingleton(sp => sp.GetRequiredService()); + + return services; + } + + public static IServiceCollection AddJobLifetimeService(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService()); + return services; + } +} diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs index 4ddb44601..069dc2b27 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs @@ -15,7 +15,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; internal class ScheduledJobInstance { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs index b1279a284..23cb622b7 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs @@ -3,7 +3,7 @@ using System.Runtime.CompilerServices; using Foundatio.Jobs; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class ScheduledJobOptions : INotifyPropertyChanged { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs index 5fda06683..5b88c37f4 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Foundatio.Jobs; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class ScheduledJobOptionsBuilder { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobRegistration.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobRegistration.cs index 6d7269e75..862a0889c 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobRegistration.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobRegistration.cs @@ -1,4 +1,4 @@ -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class ScheduledJobRegistration { diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs index a35564a4d..cec478544 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs @@ -12,10 +12,10 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; /// -/// Legacy/compat in-process CRON scheduler used by . +/// Legacy/compat in-process CRON scheduler used by . /// It runs occurrences in-process and does not materialize durable, recoverable occurrences. The redesigned runtime's /// durable scheduler (JobScheduleProcessor driven by ) is the forward path. /// diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ShutdownHostIfNoJobsRunningService.cs b/src/Foundatio.Extensions.Hosting/Jobs/ShutdownHostIfNoJobsRunningService.cs index bb5a6c74d..1dd387ad9 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ShutdownHostIfNoJobsRunningService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ShutdownHostIfNoJobsRunningService.cs @@ -10,7 +10,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Foundatio.Extensions.Hosting.Jobs; +namespace Foundatio.Extensions.Hosting.Jobs.Legacy; public class ShutdownHostIfNoJobsRunningService : IHostedService, IDisposable { From 99167538656c7af68f8df4158b7cc204ccad2cba Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 12:21:26 -0500 Subject: [PATCH 34/57] Drop new->legacy doc reference from IJobWithExecutionContext The shared IJobWithExecutionContext doc comment cross-referenced the legacy Foundatio.Jobs.Legacy.IJobWithOptions purely to keep the cref resolving after the jobs isolation. Reword to plain prose so the new/shared Foundatio.Jobs surface carries no documentation dependency on the legacy namespace. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio/Jobs/IJob.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Foundatio/Jobs/IJob.cs b/src/Foundatio/Jobs/IJob.cs index bf5c8ef70..2d3185a49 100644 --- a/src/Foundatio/Jobs/IJob.cs +++ b/src/Foundatio/Jobs/IJob.cs @@ -23,7 +23,7 @@ public interface IJob /// A durable job that wants its — job id, attempt number, and store-backed progress, /// lease heartbeat, and cooperative cancellation checks. The durable runtime sets on the /// job instance before invoking it. Jobs that use the context should be registered as transient (a fresh instance per -/// run), since the context is per-run state, matching . +/// run), since the context is per-run state. /// public interface IJobWithExecutionContext : IJob { From 471c5d0f75a8d0ea11aeeb7b4372d15085d98375 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 13:32:57 -0500 Subject: [PATCH 35/57] Merge IJobWithExecutionContext into a single context-based IJob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new durable-jobs contract is now one interface: IJob.RunAsync(JobExecutionContext context). Every run is handed its context (cancellation token, job id, attempt, and store-backed progress/heartbeat helpers) and uses what it needs — no separate IJobWithExecutionContext marker to remember. JobExecutionContext gains a public "detached" constructor so a job can be run outside the durable runtime (tests, one-off invocations); its store-backed helpers become no-ops there. Because the old IJob (RunAsync(CancellationToken)) was shared with the legacy job system, legacy is forked onto its own self-contained contract: Foundatio.Jobs.Legacy gains its own IJob and its own JobResult/JobResultExtensions, plus a legacy TryRunAsync. The legacy job types, hosting runners, and legacy tests bind to those (dropping the now-ambiguous `using Foundatio.Jobs;`), leaving the new Foundatio.Jobs surface clean. Full solution builds (net8.0 + net10.0, warnings-as-errors); Jobs test namespace green (56 passed, 1 manual-only skip). Sample jobs updated to the new signature; the fuller sample rewrite (declarative handlers, fluent providers) follows. Co-Authored-By: Claude Opus 4.8 --- samples/Foundatio.MessagingSample/Jobs.cs | 20 ++--- .../Jobs/DynamicJob.cs | 2 +- .../Jobs/HostedJobService.cs | 1 - .../Jobs/JobManager.cs | 1 - .../Jobs/JobOptionsBuilder.cs | 1 - .../Jobs/LegacyJobHostExtensions.cs | 1 - .../Jobs/ScheduledJobInstance.cs | 2 +- .../Jobs/ScheduledJobOptions.cs | 2 +- .../Jobs/ScheduledJobOptionsBuilder.cs | 2 +- .../Jobs/HelloWorldJob.cs | 1 - .../Jobs/JobQueueTestsBase.cs | 1 - .../Jobs/SampleQueueJob.cs | 1 - .../Jobs/ThrottledJob.cs | 1 - .../Jobs/WithDependencyJob.cs | 1 - .../Jobs/WithLockingJob.cs | 1 - .../Queue/QueueTestBase.cs | 1 - src/Foundatio/Jobs/IJob.cs | 28 +++--- src/Foundatio/Jobs/JobRuntime.cs | 38 ++++++--- src/Foundatio/Jobs/LegacyJob.cs | 14 +++ src/Foundatio/Jobs/LegacyJobResult.cs | 85 +++++++++++++++++++ src/Foundatio/Jobs/LegacyJobRunExtensions.cs | 19 +++++ .../RedisJobStoreIntegrationTests.cs | 8 +- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 17 ++-- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 8 +- tests/Foundatio.Tests/Jobs/JobTests.cs | 1 - 25 files changed, 182 insertions(+), 75 deletions(-) create mode 100644 src/Foundatio/Jobs/LegacyJob.cs create mode 100644 src/Foundatio/Jobs/LegacyJobResult.cs diff --git a/samples/Foundatio.MessagingSample/Jobs.cs b/samples/Foundatio.MessagingSample/Jobs.cs index acffd9ebf..6ec9c7c04 100644 --- a/samples/Foundatio.MessagingSample/Jobs.cs +++ b/samples/Foundatio.MessagingSample/Jobs.cs @@ -6,19 +6,16 @@ namespace Foundatio.MessagingSample; /// A durable, on-demand job (submitted via POST /reports). It runs on whichever instance's runtime pump claims /// it, and reports progress through its so GET /reports/{id} can observe it. /// -public sealed class GenerateReportJob(InstanceInfo instance, ILogger logger) : IJobWithExecutionContext +public sealed class GenerateReportJob(InstanceInfo instance, ILogger logger) : IJob { - public JobExecutionContext? ExecutionContext { get; set; } - - public async Task RunAsync(CancellationToken cancellationToken = default) + public async Task RunAsync(JobExecutionContext context) { - logger.LogInformation("[{Instance}] generating report {JobId}", instance.Id, ExecutionContext?.JobId); + logger.LogInformation("[{Instance}] generating report {JobId}", instance.Id, context.JobId); for (int percent = 25; percent <= 100; percent += 25) { - await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken); - if (ExecutionContext is { } context) - await context.ReportProgressAsync(percent, $"{percent}% complete", cancellationToken); + await Task.Delay(TimeSpan.FromMilliseconds(250), context.CancellationToken); + await context.ReportProgressAsync(percent, $"{percent}% complete", context.CancellationToken); } return JobResult.Success; @@ -29,11 +26,12 @@ public async Task RunAsync(CancellationToken cancellationToken = defa // occurrence is materialized once into the shared runtime store, so scope decides how many instances run it: // * Global (default) -> exactly ONE instance runs each tick (a leader/singleton task). // * PerNode -> EVERY instance runs its own occurrence each tick (per-instance maintenance). +// A job uses its JobExecutionContext when it wants progress/heartbeat/identity, or ignores it (as these do). /// Global, every minute: a simple liveness heartbeat that runs on a single instance per tick. public sealed class HeartbeatJob(InstanceInfo instance, ILogger logger) : IJob { - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { logger.LogInformation("[{Instance}] heartbeat {Time:HH:mm:ss} (one instance per tick)", instance.Id, DateTimeOffset.UtcNow); return Task.FromResult(JobResult.Success); @@ -43,7 +41,7 @@ public Task RunAsync(CancellationToken cancellationToken = default) /// PerNode, every minute: each instance refreshes its own local state — so every instance runs this each tick. public sealed class RefreshCacheJob(InstanceInfo instance, ILogger logger) : IJob { - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { logger.LogInformation("[{Instance}] refreshed local cache (every instance per tick)", instance.Id); return Task.FromResult(JobResult.Success); @@ -53,7 +51,7 @@ public Task RunAsync(CancellationToken cancellationToken = default) /// Global, every 2 minutes: a periodic maintenance sweep that runs on a single instance per tick. public sealed class SweepStaleOrdersJob(InstanceInfo instance, ILogger logger) : IJob { - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { logger.LogInformation("[{Instance}] swept stale orders (one instance per tick)", instance.Id); return Task.FromResult(JobResult.Success); diff --git a/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs b/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs index 48ce072cd..cfe7ee563 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/DynamicJob.cs @@ -1,7 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; -using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Utility; namespace Foundatio.Extensions.Hosting.Jobs.Legacy; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs index 0105be37f..48867dd00 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/HostedJobService.cs @@ -2,7 +2,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Extensions.Hosting.Startup; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs index 49c94cd3e..ebf8daa5f 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobManager.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs index d0f414239..0e9d13bfd 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/JobOptionsBuilder.cs @@ -1,5 +1,4 @@ using System; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; namespace Foundatio.Extensions.Hosting.Jobs.Legacy; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs b/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs index 6f8633331..ca428697a 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/LegacyJobHostExtensions.cs @@ -2,7 +2,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs index 069dc2b27..b30a1a7cb 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Cronos; -using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Messaging; using Foundatio.Messaging.Legacy; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs index 23cb622b7..95559f577 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptions.cs @@ -1,7 +1,7 @@ using System; using System.ComponentModel; using System.Runtime.CompilerServices; -using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; namespace Foundatio.Extensions.Hosting.Jobs.Legacy; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs index 5b88c37f4..2b104f27e 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobOptionsBuilder.cs @@ -1,7 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; -using Foundatio.Jobs; +using Foundatio.Jobs.Legacy; namespace Foundatio.Extensions.Hosting.Jobs.Legacy; diff --git a/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs b/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs index 7e7161a67..4ca876c5b 100644 --- a/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs +++ b/src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs @@ -1,7 +1,6 @@ using System; using System.Threading; using System.Threading.Tasks; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs b/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs index 281974130..83fd6c5f8 100644 --- a/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs +++ b/src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Exceptionless; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Queues; diff --git a/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs b/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs index 0fccf829d..d03aac214 100644 --- a/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs +++ b/src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs @@ -2,7 +2,6 @@ using System.Threading; using System.Threading.Tasks; using Exceptionless; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Queues; diff --git a/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs b/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs index 3a2de04dc..5d5e84e1d 100644 --- a/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs +++ b/src/Foundatio.TestHarness/Jobs/ThrottledJob.cs @@ -2,7 +2,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs b/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs index a8188fec0..b205b123c 100644 --- a/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs @@ -1,5 +1,4 @@ using System.Threading.Tasks; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs index a877d48b7..6e9435108 100644 --- a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs @@ -2,7 +2,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Messaging; diff --git a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs index e70a8a2c4..977f0db21 100644 --- a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs +++ b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs @@ -7,7 +7,6 @@ using Exceptionless; using Foundatio.AsyncEx; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Lock; using Foundatio.Messaging; diff --git a/src/Foundatio/Jobs/IJob.cs b/src/Foundatio/Jobs/IJob.cs index 2d3185a49..23ac3ace6 100644 --- a/src/Foundatio/Jobs/IJob.cs +++ b/src/Foundatio/Jobs/IJob.cs @@ -1,42 +1,34 @@ using System; -using System.Threading; using System.Threading.Tasks; using Foundatio.Utility; namespace Foundatio.Jobs; /// -/// Represents a unit of background work that can be executed once or continuously. -/// Implement this interface to create custom jobs for scheduled tasks, queue processing, or maintenance operations. +/// Represents a unit of background work run by the durable job runtime. Every run is handed a +/// carrying its cancellation token, identity, attempt number, and store-backed +/// progress/heartbeat helpers — a job uses what it needs and ignores the rest. /// public interface IJob { /// /// Executes the job's work. /// - /// Token to signal that the job should stop. + /// The execution context for this run (cancellation, identity, progress, heartbeat). /// A result indicating success, failure, or cancellation. - Task RunAsync(CancellationToken cancellationToken = default); -} - -/// -/// A durable job that wants its — job id, attempt number, and store-backed progress, -/// lease heartbeat, and cooperative cancellation checks. The durable runtime sets on the -/// job instance before invoking it. Jobs that use the context should be registered as transient (a fresh instance per -/// run), since the context is per-run state. -/// -public interface IJobWithExecutionContext : IJob -{ - JobExecutionContext? ExecutionContext { get; set; } + Task RunAsync(JobExecutionContext context); } public static class JobExtensions { - public static async Task TryRunAsync(this IJob job, CancellationToken cancellationToken = default) + /// + /// Runs the job, converting cancellation and unhandled exceptions into a instead of throwing. + /// + public static async Task TryRunAsync(this IJob job, JobExecutionContext context) { try { - return await job.RunAsync(cancellationToken).AnyContext(); + return await job.RunAsync(context).AnyContext(); } catch (OperationCanceledException) { diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index f28f496d3..2bcedf7bf 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -214,13 +214,14 @@ public Task RequestCancellationAsync(CancellationToken cancellationToken = } /// -/// Passed to a durable job that implements . Gives the running job its identity -/// and attempt number, plus store-backed progress reporting, lease heartbeat (for long runs), and cooperative -/// cancellation checks — the parts of that are useful from inside job code. +/// Passed to a job on each run. Gives the running job its identity and attempt number, plus store-backed progress +/// reporting, lease heartbeat (for long runs), and cooperative cancellation checks — the parts of +/// that are useful from inside job code. When a job is run outside the durable runtime +/// (for example directly in a test), the store-backed helpers are no-ops and cancellation reflects the supplied token. /// public sealed class JobExecutionContext { - private readonly IJobRuntimeStore _store; + private readonly IJobRuntimeStore? _store; private readonly string _nodeId; private readonly TimeSpan _lease; @@ -234,19 +235,33 @@ internal JobExecutionContext(string jobId, int attempt, CancellationToken cancel _lease = lease; } + /// + /// Creates a detached context for running a job outside the durable runtime (tests or one-off invocations). + /// Progress reporting and lease renewal are no-ops; cancellation reflects . + /// + public JobExecutionContext(CancellationToken cancellationToken = default, string? jobId = null, int attempt = 1) + { + JobId = jobId ?? Guid.NewGuid().ToString("N"); + Attempt = attempt; + CancellationToken = cancellationToken; + _store = null; + _nodeId = String.Empty; + _lease = TimeSpan.Zero; + } + public string JobId { get; } public int Attempt { get; } public CancellationToken CancellationToken { get; } public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) - => _store.SetProgressAsync(JobId, percent, message, cancellationToken); + => _store?.SetProgressAsync(JobId, percent, message, cancellationToken) ?? Task.CompletedTask; // Extends the worker's lease so a long-but-alive run is not reclaimed as stale. public Task RenewLeaseAsync(CancellationToken cancellationToken = default) - => _store.RenewClaimAsync(JobId, _nodeId, _lease, cancellationToken); + => _store?.RenewClaimAsync(JobId, _nodeId, _lease, cancellationToken) ?? Task.FromResult(true); public Task IsCancellationRequestedAsync(CancellationToken cancellationToken = default) - => _store.IsCancellationRequestedAsync(JobId, cancellationToken); + => _store?.IsCancellationRequestedAsync(JobId, cancellationToken) ?? Task.FromResult(CancellationToken.IsCancellationRequested); } public interface IJobMonitor @@ -821,12 +836,11 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc var jobType = ResolveJobType(state); var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); - // Hand the job its execution context (progress, heartbeat, cancellation, identity) when it opts in. The - // store was already incremented to this attempt by the Queued -> Processing transition above. - if (job is IJobWithExecutionContext contextual) - contextual.ExecutionContext = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease); + // Hand the job its execution context (identity, attempt, progress, heartbeat, cancellation). The store was + // already incremented to this attempt by the Queued -> Processing transition above. + var context = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease); - var result = await job.TryRunAsync(linkedCancellationTokenSource.Token).ConfigureAwait(false); + var result = await job.TryRunAsync(context).ConfigureAwait(false); var completedAt = _timeProvider.GetUtcNow(); if (result.IsCancelled) diff --git a/src/Foundatio/Jobs/LegacyJob.cs b/src/Foundatio/Jobs/LegacyJob.cs new file mode 100644 index 000000000..0d41e3b40 --- /dev/null +++ b/src/Foundatio/Jobs/LegacyJob.cs @@ -0,0 +1,14 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Jobs.Legacy; + +/// +/// The legacy job contract, run once or continuously by the legacy and hosted runners. +/// Superseded by the durable-runtime (which is handed a +/// per run); kept for compatibility. +/// +public interface IJob +{ + Task RunAsync(CancellationToken cancellationToken = default); +} diff --git a/src/Foundatio/Jobs/LegacyJobResult.cs b/src/Foundatio/Jobs/LegacyJobResult.cs new file mode 100644 index 000000000..850e7b5ca --- /dev/null +++ b/src/Foundatio/Jobs/LegacyJobResult.cs @@ -0,0 +1,85 @@ +using System; +using Microsoft.Extensions.Logging; + +namespace Foundatio.Jobs.Legacy; + +public class JobResult +{ + public bool IsCancelled { get; set; } + public Exception? Error { get; set; } + public string Message { get; set; } = String.Empty; + public bool IsSuccess { get; set; } + + public static readonly JobResult None = new() + { + IsSuccess = true + }; + + public static readonly JobResult Cancelled = new() + { + IsCancelled = true + }; + + public static readonly JobResult Success = new() + { + IsSuccess = true + }; + + public static JobResult FromException(Exception exception, string? message = null) + { + return new JobResult + { + Error = exception, + IsSuccess = false, + Message = message ?? exception.Message + }; + } + + public static JobResult CancelledWithMessage(string message) + { + return new JobResult + { + IsCancelled = true, + Message = message + }; + } + + public static JobResult SuccessWithMessage(string message) + { + return new JobResult + { + IsSuccess = true, + Message = message + }; + } + + public static JobResult FailedWithMessage(string message) + { + return new JobResult + { + IsSuccess = false, + Message = message + }; + } +} + +public static class JobResultExtensions +{ + public static void LogJobResult(this ILogger logger, JobResult result, string? jobName) + { + if (result is null) + { + logger.LogError("Null job run result for {JobName}", jobName); + return; + } + + if (result.IsCancelled) + logger.LogWarning(result.Error, "Job run {JobName} cancelled: {Message}", jobName, result.Message); + else if (!result.IsSuccess) + logger.LogError(result.Error, "Job run {JobName} failed: {Message}", jobName, result.Message); + else if (!String.IsNullOrEmpty(result.Message)) + logger.LogInformation("Job run {JobName} succeeded: {Message}", jobName, result.Message); + else + logger.LogDebug("Job run {JobName} succeeded", jobName); + } +} diff --git a/src/Foundatio/Jobs/LegacyJobRunExtensions.cs b/src/Foundatio/Jobs/LegacyJobRunExtensions.cs index c73321158..d42564a3e 100644 --- a/src/Foundatio/Jobs/LegacyJobRunExtensions.cs +++ b/src/Foundatio/Jobs/LegacyJobRunExtensions.cs @@ -20,6 +20,25 @@ public interface IJobWithOptions : IJob public static class LegacyJobExtensions { + /// + /// Runs the job, converting cancellation and unhandled exceptions into a instead of throwing. + /// + public static async Task TryRunAsync(this IJob job, CancellationToken cancellationToken = default) + { + try + { + return await job.RunAsync(cancellationToken).AnyContext(); + } + catch (OperationCanceledException) + { + return JobResult.Cancelled; + } + catch (Exception ex) + { + return JobResult.FromException(ex); + } + } + /// /// Runs the job continuously until the cancellation token is set or the iteration limit is reached. /// diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index 763c28ef4..25e528d90 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -212,9 +212,9 @@ private sealed class Probe private sealed class ProbeJob(Probe probe) : IJob { - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { - cancellationToken.ThrowIfCancellationRequested(); + context.CancellationToken.ThrowIfCancellationRequested(); probe.Record(); return Task.FromResult(JobResult.Success); } @@ -222,9 +222,9 @@ public Task RunAsync(CancellationToken cancellationToken = default) private sealed class FailingJob : IJob { - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { - cancellationToken.ThrowIfCancellationRequested(); + context.CancellationToken.ThrowIfCancellationRequested(); return Task.FromResult(JobResult.FromException(new InvalidOperationException("boom"))); } } diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index 1f3c6a7d6..0df723d3f 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -326,9 +326,9 @@ public SuccessfulTrackedJob(JobRuntimeProbe probe) _probe = probe; } - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { - cancellationToken.ThrowIfCancellationRequested(); + context.CancellationToken.ThrowIfCancellationRequested(); _probe.RecordRun(); return Task.FromResult(JobResult.Success); } @@ -343,13 +343,13 @@ public CancellableTrackedJob(JobRuntimeProbe probe) _probe = probe; } - public async Task RunAsync(CancellationToken cancellationToken = default) + public async Task RunAsync(JobExecutionContext context) { _probe.Started.TrySetResult(); try { - await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); + await Task.Delay(TimeSpan.FromMinutes(1), context.CancellationToken); return JobResult.Success; } catch (OperationCanceledException) @@ -360,14 +360,11 @@ public async Task RunAsync(CancellationToken cancellationToken = defa } } - private sealed class ProgressJob : IJobWithExecutionContext + private sealed class ProgressJob : IJob { - public JobExecutionContext? ExecutionContext { get; set; } - - public async Task RunAsync(CancellationToken cancellationToken = default) + public async Task RunAsync(JobExecutionContext context) { - var context = ExecutionContext!; - await context.ReportProgressAsync(75, $"{context.JobId}:{context.Attempt}", cancellationToken); + await context.ReportProgressAsync(75, $"{context.JobId}:{context.Attempt}", context.CancellationToken); return JobResult.Success; } } diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index d1c4b719a..ddbcc18e5 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -491,9 +491,9 @@ public FailingScheduledJob(JobSchedulerProbe probe) _probe = probe; } - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { - cancellationToken.ThrowIfCancellationRequested(); + context.CancellationToken.ThrowIfCancellationRequested(); _probe.RecordRun(); return Task.FromResult(JobResult.FromException(new InvalidOperationException("failed"))); } @@ -508,9 +508,9 @@ public ScheduledProbeJob(JobSchedulerProbe probe) _probe = probe; } - public Task RunAsync(CancellationToken cancellationToken = default) + public Task RunAsync(JobExecutionContext context) { - cancellationToken.ThrowIfCancellationRequested(); + context.CancellationToken.ThrowIfCancellationRequested(); _probe.RecordRun(); return Task.FromResult(JobResult.Success); } diff --git a/tests/Foundatio.Tests/Jobs/JobTests.cs b/tests/Foundatio.Tests/Jobs/JobTests.cs index d3a04f974..94eb34faa 100644 --- a/tests/Foundatio.Tests/Jobs/JobTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobTests.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Xunit; using Microsoft.Extensions.DependencyInjection; From 0f3c9fe3ab52c884ebc5fbe60e38bcba4a5c1903 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 13:53:41 -0500 Subject: [PATCH 36/57] Declarative message handlers + fluent provider/CRON registration; idiomatic sample MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Messaging handlers are now declarative: AddFoundatio().Messaging.AddQueueHandler() (competing consumers) and AddBroadcastHandler() (fan-out via a per-instance subscription), plus delegate overloads. Handlers implement IMessageHandler and are resolved from DI in a per-message scope; a single auto-registered hosted service starts and stops them. Programmatic StartConsumerAsync/SubscribeAsync remain for dynamic use. Fluent provider + CRON registration for clean, idiomatic startup code: - .Messaging.UseAws() (Foundatio.Aws) — SQS/SNS, binding ServiceUrl/Region/ResourcePrefix/ credentials from an "Aws" config section, overridable via a lambda. - .Messaging.UseRedis() / .Jobs.UseRedis() (Foundatio.Redis) — Redis Streams transport and Redis job runtime store, sharing one IConnectionMultiplexer (from DI or a connection string / the "Redis" configuration entry). - .Jobs.AddCronJob(cron, o => ...) — registers a durable CRON schedule (CronJobOptions for scope/overlap/etc.); the runtime pump schedules all registered definitions on start, so no manual IJobScheduler.ScheduleAsync call is needed. The Aspire sample is rewritten to this surface: one AddFoundatio() chain with UseAws() + declarative handlers + UseRedis() + AddCronJob<>(); the hand-written MessagingWorkers and the dynamic UseTransport switch are gone. New DeclarativeRegistrationTests cover handler hosting/dispatch (queue class, queue delegate, broadcast) and AddCronJob registration + pump scheduling. Full solution builds (net8.0 + net10.0, warnings-as-errors); in-memory suite green (2004 passed), live Redis suite green (27 passed). Co-Authored-By: Claude Opus 4.8 --- samples/Foundatio.AppHost/Program.cs | 10 +- samples/Foundatio.MessagingSample/Handlers.cs | 32 ++++ .../MessagingWorkers.cs | 44 ----- samples/Foundatio.MessagingSample/Program.cs | 78 ++------- .../appsettings.json | 5 +- .../AwsFoundatioBuilderExtensions.cs | 46 ++++++ .../RedisFoundatioBuilderExtensions.cs | 53 ++++++ src/Foundatio/FoundatioServicesExtensions.cs | 110 +++++++++++++ src/Foundatio/Jobs/JobRuntimePumpService.cs | 25 ++- src/Foundatio/Jobs/JobScheduler.cs | 28 ++++ src/Foundatio/Messaging/IMessageHandler.cs | 16 ++ .../Messaging/MessageHandlerHostedService.cs | 60 +++++++ .../DeclarativeRegistrationTests.cs | 154 ++++++++++++++++++ 13 files changed, 547 insertions(+), 114 deletions(-) create mode 100644 samples/Foundatio.MessagingSample/Handlers.cs delete mode 100644 samples/Foundatio.MessagingSample/MessagingWorkers.cs create mode 100644 src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs create mode 100644 src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs create mode 100644 src/Foundatio/Messaging/IMessageHandler.cs create mode 100644 src/Foundatio/Messaging/MessageHandlerHostedService.cs create mode 100644 tests/Foundatio.Tests/DeclarativeRegistrationTests.cs diff --git a/samples/Foundatio.AppHost/Program.cs b/samples/Foundatio.AppHost/Program.cs index fb85c7cc3..93ebe54aa 100644 --- a/samples/Foundatio.AppHost/Program.cs +++ b/samples/Foundatio.AppHost/Program.cs @@ -20,15 +20,17 @@ // The redesigned messaging + durable-jobs sample, scaled to 3 replicas so you can watch the queue load-balance across // instances, the pub/sub topic fan out to every instance, and durable/CRON jobs get claimed by a single instance. -// Messaging runs on AWS (SQS/SNS via LocalStack) and durable jobs on Redis; set Messaging__Provider=Redis to run the -// messaging on Redis Streams instead. +// Messaging runs on AWS (SQS/SNS via LocalStack) and durable jobs on Redis. WithReference(cache) supplies the "Redis" +// connection string UseRedis() reads; the Aws__* settings point UseAws() at LocalStack (which accepts any credentials). builder.AddProject("Foundatio-MessagingSample") .WithExternalHttpEndpoints() .WithReplicas(3) .WithReference(cache) .WaitFor(cache) .WaitFor(localstack) - .WithEnvironment("Messaging__Provider", "Aws") - .WithEnvironment("Aws__ServiceUrl", localstack.GetEndpoint("gateway")); + .WithEnvironment("Aws__ServiceUrl", localstack.GetEndpoint("gateway")) + .WithEnvironment("Aws__AccessKey", "test") + .WithEnvironment("Aws__SecretKey", "test") + .WithEnvironment("Aws__ResourcePrefix", "fnd-sample-"); await builder.Build().RunAsync(); diff --git a/samples/Foundatio.MessagingSample/Handlers.cs b/samples/Foundatio.MessagingSample/Handlers.cs new file mode 100644 index 000000000..3aec7d30e --- /dev/null +++ b/samples/Foundatio.MessagingSample/Handlers.cs @@ -0,0 +1,32 @@ +using Foundatio.Messaging; + +namespace Foundatio.MessagingSample; + +/// A short per-process id so you can see which instance handled each message/job when scaled to replicas. +public sealed record InstanceInfo(string Id); + +/// +/// Handles orders off the queue — registered with AddQueueHandler, so exactly one running instance processes +/// each order (competing consumers). Resolved from DI per message; throwing would trigger retry/dead-letter. +/// +public sealed class ProcessOrderHandler(InstanceInfo instance, ILogger logger) : IMessageHandler +{ + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + logger.LogInformation("[{Instance}] processed order: {Quantity} x {Product}", instance.Id, message.Message.Quantity, message.Message.Product); + return Task.CompletedTask; + } +} + +/// +/// Handles announcements — registered with AddBroadcastHandler, so every running instance receives its own copy +/// (fan-out via a per-instance subscription). +/// +public sealed class AnnouncementHandler(InstanceInfo instance, ILogger logger) : IMessageHandler +{ + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + logger.LogInformation("[{Instance}] announcement: {Text}", instance.Id, message.Message.Text); + return Task.CompletedTask; + } +} diff --git a/samples/Foundatio.MessagingSample/MessagingWorkers.cs b/samples/Foundatio.MessagingSample/MessagingWorkers.cs deleted file mode 100644 index b4131e811..000000000 --- a/samples/Foundatio.MessagingSample/MessagingWorkers.cs +++ /dev/null @@ -1,44 +0,0 @@ -using Foundatio.Messaging; - -namespace Foundatio.MessagingSample; - -/// A short per-process id so you can see which instance handled each message/job when scaled to replicas. -public sealed record InstanceInfo(string Id); - -/// -/// Starts this instance's long-running queue consumer and pub/sub subscriber for the app's lifetime — the idiomatic -/// way to host Foundatio consumers in ASP.NET. Handlers auto-complete on success (); throwing -/// triggers the core's retry/dead-letter policy. -/// -public sealed class MessagingWorkers(IQueue queue, IPubSub pubSub, InstanceInfo instance, ILogger logger) : IHostedService -{ - private IMessageConsumer? _orderConsumer; - private IMessageSubscription? _announcementSubscription; - - public async Task StartAsync(CancellationToken cancellationToken) - { - // Competing consumers: the shared "orders" queue load-balances across every running instance, so each order is - // processed exactly once. Scale the service up and the work spreads out. - _orderConsumer = await queue.StartConsumerAsync((message, _) => - { - logger.LogInformation("[{Instance}] processed order: {Quantity} x {Product}", instance.Id, message.Message.Quantity, message.Message.Product); - return Task.CompletedTask; - }, cancellationToken: cancellationToken); - - // Fan-out: a per-instance subscription means every instance receives every announcement (broadcast). Using a - // shared subscription name here would instead load-balance the topic like the queue above. - _announcementSubscription = await pubSub.SubscribeAsync((message, _) => - { - logger.LogInformation("[{Instance}] announcement: {Text}", instance.Id, message.Message.Text); - return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = instance.Id }, cancellationToken); - } - - public async Task StopAsync(CancellationToken cancellationToken) - { - if (_orderConsumer is not null) - await _orderConsumer.DisposeAsync(); - if (_announcementSubscription is not null) - await _announcementSubscription.DisposeAsync(); - } -} diff --git a/samples/Foundatio.MessagingSample/Program.cs b/samples/Foundatio.MessagingSample/Program.cs index 3f134edce..9f93f8522 100644 --- a/samples/Foundatio.MessagingSample/Program.cs +++ b/samples/Foundatio.MessagingSample/Program.cs @@ -1,66 +1,36 @@ -using Amazon; -using Amazon.Runtime; using Foundatio; using Foundatio.Jobs; using Foundatio.Messaging; using Foundatio.MessagingSample; -using StackExchange.Redis; var builder = WebApplication.CreateBuilder(args); // A short id so log lines make it obvious WHICH instance handled each message/job when scaled to multiple replicas. -var instance = new InstanceInfo(Guid.NewGuid().ToString("N")[..6]); -builder.Services.AddSingleton(instance); - -// One shared Redis connection: it backs the durable job runtime, and (when selected) the messaging transport too. -string redisConnectionString = builder.Configuration.GetConnectionString("Redis") ?? "localhost:6399"; -builder.Services.AddSingleton(_ => ConnectionMultiplexer.Connect(redisConnectionString)); - -// The messaging transport is chosen at startup — both AWS (SQS/SNS) and Redis (Streams) are wired, so you can flip -// Messaging:Provider and compare them without touching a line of the queue/pub-sub code below. -string transport = builder.Configuration["Messaging:Provider"] ?? "Redis"; +builder.Services.AddSingleton(new InstanceInfo(Guid.NewGuid().ToString("N")[..6])); builder.Services.AddFoundatio() - // Queues (competing consumers) and pub/sub (fan-out) both ride this single transport. - .Messaging.UseTransport(sp => transport.Equals("Aws", StringComparison.OrdinalIgnoreCase) - ? CreateAwsTransport(builder.Configuration) - : new RedisStreamsMessageTransport(new RedisStreamsMessageTransportOptions - { - ConnectionMultiplexer = sp.GetRequiredService() - })) - // Durable jobs live in Redis so any instance can claim and run them. UseRuntimeStore also auto-registers the pump - // that materializes CRON occurrences, drains scheduled work, and runs submitted jobs. - .Jobs.UseRuntimeStore(sp => new RedisJobRuntimeStore(new RedisJobRuntimeStoreOptions - { - ConnectionMultiplexer = sp.GetRequiredService() - })) - .Jobs.Register("generate-report") - .Jobs.Register("heartbeat") - .Jobs.Register("refresh-cache") - .Jobs.Register("sweep-stale-orders"); - -// Hosts this instance's queue consumer + pub/sub subscriber for the app lifetime. -builder.Services.AddHostedService(); + // Messaging on AWS (SQS/SNS). Handlers are registered declaratively; Foundatio hosts them and dispatches to them — + // no hand-written IHostedService. Swap UseAws() for UseRedis() to run messaging on Redis Streams instead. + .Messaging.UseAws() + .Messaging.AddQueueHandler() // competing consumers: one instance per order + .Messaging.AddBroadcastHandler() // fan-out: every instance gets each announcement + // Durable jobs on Redis so any instance can claim them. The pump (auto-registered) runs submitted jobs and + // materializes the CRON schedules below — no manual scheduling call. + .Jobs.UseRedis() + .Jobs.Register("generate-report") // on-demand, submitted via POST /reports + .Jobs.AddCronJob("* * * * *") // Global: one instance per tick + .Jobs.AddCronJob("* * * * *", o => o.Scope = ScheduledJobScope.PerNode) // every instance per tick + .Jobs.AddCronJob("*/2 * * * *"); // Global: periodic sweep var app = builder.Build(); -// Recurring (CRON) jobs. Every instance registers the same schedules; the shared Redis store dedupes each occurrence, -// so Scope decides how many instances run it — Global = one instance per tick, PerNode = every instance per tick. -var scheduler = app.Services.GetRequiredService(); -await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "heartbeat", Cron = "* * * * *", JobType = typeof(HeartbeatJob) }); -await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "refresh-cache", Cron = "* * * * *", Scope = ScheduledJobScope.PerNode, JobType = typeof(RefreshCacheJob) }); -await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "sweep-stale-orders", Cron = "*/2 * * * *", JobType = typeof(SweepStaleOrdersJob) }); - -app.MapGet("/", (InstanceInfo i) => Results.Ok(new { service = "Foundatio messaging sample", instance = i.Id, transport })); +app.MapGet("/", (InstanceInfo instance) => Results.Ok(new { service = "Foundatio messaging sample", instance = instance.Id })); -// QUEUE — competing consumers: exactly one instance processes each order. +// QUEUE — competing consumers: exactly one instance processes each order (handled by ProcessOrderHandler). app.MapPost("/orders", async (ProcessOrder order, IQueue queue) => -{ - string id = await queue.EnqueueAsync(order); - return Results.Accepted(value: new { queued = id }); -}); + Results.Accepted(value: new { queued = await queue.EnqueueAsync(order) })); -// PUB/SUB — fan-out: every instance receives each announcement. +// PUB/SUB — fan-out: every instance receives each announcement (handled by AnnouncementHandler). app.MapPost("/announcements", async (Announcement announcement, IPubSub pubSub) => { await pubSub.PublishAsync(announcement); @@ -83,17 +53,3 @@ }); app.Run(); - -// LocalStack (provisioned by the AppHost) provides AWS SQS/SNS locally and accepts any credentials. AutoCreateDestinations -// creates queues/topics on first use; ResourcePrefix keeps this sample's resources namespaced. -static AwsMessageTransport CreateAwsTransport(IConfiguration configuration) -{ - return new AwsMessageTransport(new AwsMessageTransportOptions - { - ServiceUrl = configuration["Aws:ServiceUrl"] ?? "http://localhost:4566", - Region = RegionEndpoint.USEast1, - Credentials = new BasicAWSCredentials("test", "test"), - AutoCreateDestinations = true, - ResourcePrefix = "fnd-sample-" - }); -} diff --git a/samples/Foundatio.MessagingSample/appsettings.json b/samples/Foundatio.MessagingSample/appsettings.json index d8cacb4d3..10f68b8c8 100644 --- a/samples/Foundatio.MessagingSample/appsettings.json +++ b/samples/Foundatio.MessagingSample/appsettings.json @@ -5,8 +5,5 @@ "Microsoft.AspNetCore": "Warning" } }, - "AllowedHosts": "*", - "Messaging": { - "Provider": "Redis" - } + "AllowedHosts": "*" } diff --git a/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs b/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs new file mode 100644 index 000000000..d428f695d --- /dev/null +++ b/src/Foundatio.Aws/AwsFoundatioBuilderExtensions.cs @@ -0,0 +1,46 @@ +using System; +using Amazon; +using Amazon.Runtime; +using Foundatio.Messaging; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Foundatio; + +public static class AwsFoundatioBuilderExtensions +{ + /// + /// Runs messaging (queues on SQS, pub/sub on SNS+SQS) over AWS. With no arguments it relies on the AWS SDK's default + /// region and credential resolution; common settings (ServiceUrl, Region, ResourcePrefix, AccessKey/SecretKey) are + /// also bound from an "Aws" configuration section when present, and can override + /// anything. Point ServiceUrl at LocalStack to run without a cloud account. + /// + public static FoundatioBuilder UseAws(this FoundatioBuilder.MessagingBuilder builder, Action? configure = null) + { + return builder.UseTransport(sp => + { + var options = new AwsMessageTransportOptions(); + BindFromConfiguration(options, sp.GetService()?.GetSection("Aws")); + configure?.Invoke(options); + return new AwsMessageTransport(options); + }); + } + + private static void BindFromConfiguration(AwsMessageTransportOptions options, IConfiguration? section) + { + if (section is null) + return; + + if (section["ServiceUrl"] is { Length: > 0 } serviceUrl) + options.ServiceUrl = serviceUrl; + + if (section["Region"] is { Length: > 0 } region) + options.Region = RegionEndpoint.GetBySystemName(region); + + if (section["ResourcePrefix"] is { Length: > 0 } prefix) + options.ResourcePrefix = prefix; + + if (section["AccessKey"] is { Length: > 0 } accessKey && section["SecretKey"] is { Length: > 0 } secretKey) + options.Credentials = new BasicAWSCredentials(accessKey, secretKey); + } +} diff --git a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs new file mode 100644 index 000000000..71d7b040e --- /dev/null +++ b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs @@ -0,0 +1,53 @@ +using System; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StackExchange.Redis; + +namespace Foundatio; + +public static class RedisFoundatioBuilderExtensions +{ + /// + /// Backs the durable job runtime with Redis. Uses an already registered in DI, + /// otherwise connects using or the "Redis" connection string from configuration + /// (falling back to localhost). The connection is shared with . + /// + public static FoundatioBuilder UseRedis(this FoundatioBuilder.JobsBuilder builder, Action? configure = null, string? connectionString = null) + { + EnsureConnection(((IFoundatioBuilder)builder).Services, connectionString); + return builder.UseRuntimeStore(sp => + { + var options = new RedisJobRuntimeStoreOptions { ConnectionMultiplexer = sp.GetRequiredService() }; + configure?.Invoke(options); + return new RedisJobRuntimeStore(options); + }); + } + + /// + /// Runs messaging (queues + pub/sub) over Redis Streams. Uses an already + /// registered in DI, otherwise connects using or the "Redis" connection string + /// from configuration (falling back to localhost). + /// + public static FoundatioBuilder UseRedis(this FoundatioBuilder.MessagingBuilder builder, Action? configure = null, string? connectionString = null) + { + EnsureConnection(((IFoundatioBuilder)builder).Services, connectionString); + return builder.UseTransport(sp => + { + var options = new RedisStreamsMessageTransportOptions { ConnectionMultiplexer = sp.GetRequiredService() }; + configure?.Invoke(options); + return new RedisStreamsMessageTransport(options); + }); + } + + // Register a single shared multiplexer if the app hasn't already, so messaging and jobs reuse one connection. + private static void EnsureConnection(IServiceCollection services, string? connectionString) + { + services.TryAddSingleton(sp => ConnectionMultiplexer.Connect( + connectionString + ?? sp.GetService()?.GetConnectionString("Redis") + ?? "localhost:6379")); + } +} diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 787ee32a0..cad410a05 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -1,5 +1,7 @@ using System; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Extensions; using Foundatio.Jobs; @@ -11,6 +13,7 @@ using Foundatio.Serializer; using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -339,6 +342,82 @@ public FoundatioBuilder UseTransport(Func f return _builder; } + /// + /// Registers a handler that processes messages of type from its queue as + /// competing consumers — exactly one running instance handles each message. The handler is resolved from DI in + /// its own scope per message (so it can inject scoped dependencies); throwing triggers the retry/dead-letter + /// policy. A hosted service starts and stops it automatically. + /// + public FoundatioBuilder AddQueueHandler(QueueConsumerOptions? options = null) + where TMessage : class where THandler : class, IMessageHandler + { + _services.TryAddScoped(); + return AddHandler($"queue:{typeof(TMessage).Name} -> {typeof(THandler).Name}", async (sp, ct) => + await sp.GetRequiredService().StartConsumerAsync( + (message, c) => DispatchAsync(sp, message, c), options, ct).ConfigureAwait(false)); + } + + /// + /// Registers a delegate handler for messages of type from its queue as competing + /// consumers. A hosted service starts and stops it automatically. + /// + public FoundatioBuilder AddQueueHandler(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null) + where TMessage : class + { + ArgumentNullException.ThrowIfNull(handler); + return AddHandler($"queue:{typeof(TMessage).Name}", async (sp, ct) => + await sp.GetRequiredService().StartConsumerAsync(handler, options, ct).ConfigureAwait(false)); + } + + /// + /// Registers a handler that receives every message of type published to its + /// topic — a per-instance subscription means every running instance gets its own copy (fan-out). Pass an explicit + /// to share a named subscription (load-balanced) instead. Resolved from DI per + /// message; a hosted service starts and stops it automatically. + /// + public FoundatioBuilder AddBroadcastHandler(string? subscription = null) + where TMessage : class where THandler : class, IMessageHandler + { + _services.TryAddScoped(); + string name = subscription ?? UniqueSubscriptionName(); + return AddHandler($"broadcast:{typeof(TMessage).Name} -> {typeof(THandler).Name}", async (sp, ct) => + await sp.GetRequiredService().SubscribeAsync( + (message, c) => DispatchAsync(sp, message, c), + new PubSubSubscriptionOptions { Subscription = name }, ct).ConfigureAwait(false)); + } + + /// + /// Registers a delegate handler that receives every message of type published to + /// its topic (fan-out via a per-instance subscription). A hosted service starts and stops it automatically. + /// + public FoundatioBuilder AddBroadcastHandler(Func, CancellationToken, Task> handler, string? subscription = null) + where TMessage : class + { + ArgumentNullException.ThrowIfNull(handler); + string name = subscription ?? UniqueSubscriptionName(); + return AddHandler($"broadcast:{typeof(TMessage).Name}", async (sp, ct) => + await sp.GetRequiredService().SubscribeAsync(handler, + new PubSubSubscriptionOptions { Subscription = name }, ct).ConfigureAwait(false)); + } + + private static async Task DispatchAsync(IServiceProvider serviceProvider, IReceivedMessage message, CancellationToken cancellationToken) + where TMessage : class where THandler : class, IMessageHandler + { + await using var scope = serviceProvider.CreateAsyncScope(); + var handler = scope.ServiceProvider.GetRequiredService(); + await handler.HandleAsync(message, cancellationToken).ConfigureAwait(false); + } + + private static string UniqueSubscriptionName() => $"{Environment.MachineName}-{Guid.NewGuid():N}"; + + private FoundatioBuilder AddHandler(string description, Func> start) + { + _services.AddSingleton(new MessageHandlerRegistration { Description = description, StartAsync = start }); + if (!_services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(MessageHandlerHostedService))) + _services.AddSingleton(); + return _builder; + } + private void RegisterMessagingRuntime(Func factory) { _services.ReplaceSingleton(factory); @@ -461,6 +540,37 @@ public FoundatioBuilder Register(string name) where TJob : IJob return _builder; } + /// + /// Registers a recurring (CRON) job. The schedule is materialized once into the shared runtime store per + /// occurrence, so decides fan-out (Global = one instance per tick, + /// PerNode = every instance per tick). Scheduled automatically when the runtime pump starts — no manual + /// call needed. Requires a runtime store ( + /// / ). + /// + public FoundatioBuilder AddCronJob(string cronSchedule, Action? configure = null) where TJob : IJob + { + ArgumentException.ThrowIfNullOrEmpty(cronSchedule); + + var options = new CronJobOptions(); + configure?.Invoke(options); + string name = options.Name ?? typeof(TJob).Name; + + _services.AddSingleton(new JobTypeRegistration(name, typeof(TJob))); + _services.AddSingleton(new ScheduledJobDefinition + { + Name = name, + Cron = cronSchedule, + JobType = typeof(TJob), + Scope = options.Scope, + Overlap = options.Overlap, + MisfireWindow = options.MisfireWindow, + MaxRetries = options.MaxRetries, + Enabled = options.Enabled, + TimeZone = options.TimeZone + }); + return _builder; + } + /// /// Tunes the auto-registered runtime pump (cadence, batch size, or /// to opt out of automatic pumping and take manual control). diff --git a/src/Foundatio/Jobs/JobRuntimePumpService.cs b/src/Foundatio/Jobs/JobRuntimePumpService.cs index 2ef580d22..c4e16cb8c 100644 --- a/src/Foundatio/Jobs/JobRuntimePumpService.cs +++ b/src/Foundatio/Jobs/JobRuntimePumpService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Foundatio.Utility; @@ -43,14 +44,18 @@ public class JobRuntimePumpService : BackgroundService private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly JobRuntimePumpOptions _options; + private readonly IJobScheduler? _scheduler; + private readonly IEnumerable _scheduledJobs; - public JobRuntimePumpService(JobScheduleProcessor processor, IJobWorker worker, TimeProvider? timeProvider = null, ILoggerFactory? loggerFactory = null, JobRuntimePumpOptions? options = null) + public JobRuntimePumpService(JobScheduleProcessor processor, IJobWorker worker, TimeProvider? timeProvider = null, ILoggerFactory? loggerFactory = null, JobRuntimePumpOptions? options = null, IJobScheduler? scheduler = null, IEnumerable? scheduledJobs = null) { _processor = processor ?? throw new ArgumentNullException(nameof(processor)); _worker = worker ?? throw new ArgumentNullException(nameof(worker)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _options = options ?? new JobRuntimePumpOptions(); + _scheduler = scheduler; + _scheduledJobs = scheduledJobs ?? Array.Empty(); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -63,6 +68,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); + // Schedule CRON jobs registered declaratively via AddFoundatio().Jobs.AddCronJob() so users don't have to + // call IJobScheduler.ScheduleAsync themselves. Idempotent (schedule keyed by name), so every node registering + // the same schedules is fine. + if (_scheduler is not null) + { + foreach (var definition in _scheduledJobs) + { + try + { + await _scheduler.ScheduleAsync(definition, stoppingToken).AnyContext(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to schedule CRON job {JobName}: {Message}", definition.Name, ex.Message); + } + } + } + while (!stoppingToken.IsCancellationRequested) { try diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 395dc4e64..dfabeffbc 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -41,6 +41,34 @@ public sealed record ScheduledJobDefinition public bool Enabled { get; init; } = true; } +/// +/// Options for a declaratively-registered CRON job — AddFoundatio().Jobs.AddCronJob<TJob>(cron, o => ...). +/// The registered definitions are scheduled automatically when the runtime pump starts. +/// +public sealed class CronJobOptions +{ + /// Schedule name (must be unique across scheduled jobs). Defaults to the job type name. + public string? Name { get; set; } + + /// Global (one instance per tick, the default) or PerNode (every instance runs it per tick). + public ScheduledJobScope Scope { get; set; } = ScheduledJobScope.Global; + + /// Whether a new occurrence is skipped while a prior one is still running. Default SkipIfRunning. + public OverlapPolicy Overlap { get; set; } = OverlapPolicy.SkipIfRunning; + + /// How late a missed occurrence may still fire. Null uses the scheduler default. + public TimeSpan? MisfireWindow { get; set; } + + /// Maximum retry attempts for a failed occurrence. Default 3. + public int MaxRetries { get; set; } = 3; + + /// Whether the schedule is active. Default true. + public bool Enabled { get; set; } = true; + + /// Time zone the CRON expression is evaluated in. Null uses the scheduler default (UTC). + public TimeZoneInfo? TimeZone { get; set; } +} + public interface IJobScheduler { Task ScheduleAsync(ScheduledJobDefinition definition, CancellationToken cancellationToken = default); diff --git a/src/Foundatio/Messaging/IMessageHandler.cs b/src/Foundatio/Messaging/IMessageHandler.cs new file mode 100644 index 000000000..f82fa77da --- /dev/null +++ b/src/Foundatio/Messaging/IMessageHandler.cs @@ -0,0 +1,16 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Messaging; + +/// +/// Handles messages of type received from a queue or a pub/sub subscription. Register a +/// handler with AddFoundatio().Messaging.AddQueueHandler<T, THandler>() (competing consumers) or +/// AddBroadcastHandler<T, THandler>() (fan-out); a hosted service then starts and dispatches to it. +/// Handlers are resolved from DI in their own scope per message, so they can inject scoped dependencies. Throwing from +/// triggers the core's retry/dead-letter policy. +/// +public interface IMessageHandler where T : class +{ + Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken); +} diff --git a/src/Foundatio/Messaging/MessageHandlerHostedService.cs b/src/Foundatio/Messaging/MessageHandlerHostedService.cs new file mode 100644 index 000000000..052f20b14 --- /dev/null +++ b/src/Foundatio/Messaging/MessageHandlerHostedService.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Foundatio.Messaging; + +/// +/// One declarative message-handler registration: a description for logging and a factory that starts the underlying +/// queue consumer or pub/sub subscription and returns it for disposal on shutdown. Built by the +/// AddQueueHandler/AddBroadcastHandler builder methods, which bind the message type at compile time. +/// +internal sealed class MessageHandlerRegistration +{ + public required string Description { get; init; } + public required Func> StartAsync { get; init; } +} + +/// +/// Hosts every declaratively-registered message handler for the app's lifetime: on start it launches each handler's +/// consumer/subscription; on stop it disposes them. Auto-registered when the first handler is added, so users register +/// handlers in configuration and never hand-write a hosted service. Programmatic +/// / remain available for dynamic use. +/// +internal sealed class MessageHandlerHostedService : IHostedService +{ + private readonly IServiceProvider _serviceProvider; + private readonly IEnumerable _registrations; + private readonly ILogger _logger; + private readonly List _started = new(); + + public MessageHandlerHostedService(IServiceProvider serviceProvider, IEnumerable registrations, ILoggerFactory? loggerFactory = null) + { + _serviceProvider = serviceProvider; + _registrations = registrations; + _logger = loggerFactory?.CreateLogger() ?? NullLogger.Instance; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + foreach (var registration in _registrations) + { + var disposable = await registration.StartAsync(_serviceProvider, cancellationToken).AnyContext(); + _started.Add(disposable); + _logger.LogInformation("Started message handler {Handler}", registration.Description); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + foreach (var disposable in _started) + await disposable.DisposeAsync().AnyContext(); + + _started.Clear(); + } +} diff --git a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs new file mode 100644 index 000000000..277c12871 --- /dev/null +++ b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Foundatio.Tests; + +public class DeclarativeRegistrationTests +{ + [Fact] + public async Task AddHandlers_HostAndDispatchQueueAndBroadcastMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var probe = new HandlerProbe(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(probe); + services.AddFoundatio() + .Messaging.UseInMemory() + .Messaging.AddQueueHandler() // class handler, competing + .Messaging.AddQueueHandler((message, _) => { probe.Record($"task:{message.Message.Id}"); return Task.CompletedTask; }) // delegate handler + .Messaging.AddBroadcastHandler(); // class handler, fan-out + + await using var provider = services.BuildServiceProvider(); + var hosted = provider.GetServices().ToList(); + Assert.Single(hosted); // exactly one auto-registered hosted service drives every handler + + foreach (var service in hosted) + await service.StartAsync(cancellationToken); + + try + { + await provider.GetRequiredService().EnqueueAsync(new HandledOrder { Id = "o1" }, cancellationToken: cancellationToken); + await provider.GetRequiredService().EnqueueAsync(new HandledTask { Id = "t1" }, cancellationToken: cancellationToken); + await provider.GetRequiredService().PublishAsync(new HandledEvent { Id = "e1" }, cancellationToken: cancellationToken); + + Assert.True(await probe.WaitForAsync(3, TimeSpan.FromSeconds(10)), $"handled: {string.Join(",", probe.Events)}"); + Assert.Contains("order:o1", probe.Events); + Assert.Contains("task:t1", probe.Events); + Assert.Contains("event:e1", probe.Events); + } + finally + { + foreach (var service in hosted) + await service.StopAsync(cancellationToken); + } + } + + [Fact] + public async Task AddCronJob_RegistersDefinitionAndSchedulesWhenPumpStartsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddFoundatio() + .Jobs.UseInMemoryRuntime() + .Jobs.AddCronJob("* * * * *", o => o.Scope = ScheduledJobScope.PerNode); + + await using var provider = services.BuildServiceProvider(); + + // The builder records the schedule as a DI singleton with the requested scope and a type-derived name. + var definition = Assert.Single(provider.GetServices()); + Assert.Equal(typeof(CronProbeJob), definition.JobType); + Assert.Equal(ScheduledJobScope.PerNode, definition.Scope); + Assert.Equal(nameof(CronProbeJob), definition.Name); + + // Starting the runtime pump schedules registered CRON jobs into the scheduler — no manual ScheduleAsync call. + var hosted = provider.GetServices().ToList(); + foreach (var service in hosted) + await service.StartAsync(cancellationToken); + + try + { + var scheduler = provider.GetRequiredService(); + ScheduledJobDefinition? scheduled = null; + long deadline = Environment.TickCount64 + 10_000; + while (Environment.TickCount64 < deadline) + { + scheduled = (await scheduler.GetSchedulesAsync(cancellationToken)).FirstOrDefault(s => s.Name == nameof(CronProbeJob)); + if (scheduled is not null) + break; + await Task.Delay(25, cancellationToken); + } + + Assert.NotNull(scheduled); + Assert.Equal(ScheduledJobScope.PerNode, scheduled!.Scope); + } + finally + { + foreach (var service in hosted) + await service.StopAsync(cancellationToken); + } + } + + private sealed class HandlerProbe + { + private readonly ConcurrentBag _events = new(); + public IReadOnlyCollection Events => _events; + public void Record(string value) => _events.Add(value); + + public async Task WaitForAsync(int count, TimeSpan timeout) + { + long deadline = Environment.TickCount64 + (long)timeout.TotalMilliseconds; + while (Environment.TickCount64 < deadline) + { + if (_events.Count >= count) + return true; + await Task.Delay(25); + } + return _events.Count >= count; + } + } + + [MessageRoute("declarative-orders")] + public class HandledOrder { public string Id { get; set; } = ""; } + + [MessageRoute("declarative-tasks")] + public class HandledTask { public string Id { get; set; } = ""; } + + [MessageRoute("declarative-events")] + public class HandledEvent { public string Id { get; set; } = ""; } + + private sealed class OrderHandler(HandlerProbe probe) : IMessageHandler + { + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + probe.Record($"order:{message.Message.Id}"); + return Task.CompletedTask; + } + } + + private sealed class EventHandler(HandlerProbe probe) : IMessageHandler + { + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + probe.Record($"event:{message.Message.Id}"); + return Task.CompletedTask; + } + } + + private sealed class CronProbeJob : IJob + { + public Task RunAsync(JobExecutionContext context) => Task.FromResult(JobResult.Success); + } +} From 05132ea4aa8aab5ee038c5b468213be3943ead09 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 14:06:29 -0500 Subject: [PATCH 37/57] Address review findings on the jobs/messaging API changes - MessageHandlerHostedService: dispose every started consumer even if one DisposeAsync throws (a broker dropping mid-shutdown no longer leaks the rest), and roll back already-started consumers if StartAsync fails partway (a hosted service whose StartAsync throws is never sent StopAsync). - JobRuntimePumpService: schedule declaratively-registered CRON jobs before the Enabled short-circuit, so AddCronJob's "scheduled automatically" contract holds even when the pump is disabled for manual control. - WorkItemJobTests: drop the now-dead `using Foundatio.Jobs;` so the legacy test imports only Foundatio.Jobs.Legacy, removing a latent CS0104 ambiguity (IJob/ JobResult now exist in both namespaces). - UseRedis: document that messaging and jobs share one connection, so the connection string from the first UseRedis call wins. Found by an adversarial review of the two prior commits (all findings low/medium). Full solution builds (net8.0 + net10.0, warnings-as-errors); declarative + jobs suites green. Co-Authored-By: Claude Opus 4.8 --- .../RedisFoundatioBuilderExtensions.cs | 6 ++- src/Foundatio/Jobs/JobRuntimePumpService.cs | 21 +++++---- .../Messaging/MessageHandlerHostedService.cs | 46 +++++++++++++++---- .../Foundatio.Tests/Jobs/WorkItemJobTests.cs | 1 - 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs index 71d7b040e..db7dab9f4 100644 --- a/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs +++ b/src/Foundatio.Redis/RedisFoundatioBuilderExtensions.cs @@ -13,7 +13,8 @@ public static class RedisFoundatioBuilderExtensions /// /// Backs the durable job runtime with Redis. Uses an already registered in DI, /// otherwise connects using or the "Redis" connection string from configuration - /// (falling back to localhost). The connection is shared with . + /// (falling back to localhost). When both messaging and jobs use Redis a single connection is shared, so the + /// connection string from the first UseRedis call wins (a differing string on the second call is ignored). /// public static FoundatioBuilder UseRedis(this FoundatioBuilder.JobsBuilder builder, Action? configure = null, string? connectionString = null) { @@ -29,7 +30,8 @@ public static FoundatioBuilder UseRedis(this FoundatioBuilder.JobsBuilder builde /// /// Runs messaging (queues + pub/sub) over Redis Streams. Uses an already /// registered in DI, otherwise connects using or the "Redis" connection string - /// from configuration (falling back to localhost). + /// from configuration (falling back to localhost). When both messaging and jobs use Redis a single connection is + /// shared, so the connection string from the first UseRedis call wins (a differing string on the second is ignored). /// public static FoundatioBuilder UseRedis(this FoundatioBuilder.MessagingBuilder builder, Action? configure = null, string? connectionString = null) { diff --git a/src/Foundatio/Jobs/JobRuntimePumpService.cs b/src/Foundatio/Jobs/JobRuntimePumpService.cs index c4e16cb8c..76cda7099 100644 --- a/src/Foundatio/Jobs/JobRuntimePumpService.cs +++ b/src/Foundatio/Jobs/JobRuntimePumpService.cs @@ -60,17 +60,10 @@ public JobRuntimePumpService(JobScheduleProcessor processor, IJobWorker worker, protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - if (!_options.Enabled) - { - _logger.LogInformation("Job runtime pump disabled (JobRuntimePumpOptions.Enabled = false); not pumping the runtime store"); - return; - } - - _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); - // Schedule CRON jobs registered declaratively via AddFoundatio().Jobs.AddCronJob() so users don't have to - // call IJobScheduler.ScheduleAsync themselves. Idempotent (schedule keyed by name), so every node registering - // the same schedules is fine. + // call IJobScheduler.ScheduleAsync themselves. Done before the Enabled check so the "scheduled automatically" + // contract holds even when this node's pump is disabled for manual control. Idempotent (schedule keyed by name), + // so every node registering the same schedules is fine. if (_scheduler is not null) { foreach (var definition in _scheduledJobs) @@ -86,6 +79,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + if (!_options.Enabled) + { + _logger.LogInformation("Job runtime pump disabled (JobRuntimePumpOptions.Enabled = false); not pumping the runtime store"); + return; + } + + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); + while (!stoppingToken.IsCancellationRequested) { try diff --git a/src/Foundatio/Messaging/MessageHandlerHostedService.cs b/src/Foundatio/Messaging/MessageHandlerHostedService.cs index 052f20b14..5503774d7 100644 --- a/src/Foundatio/Messaging/MessageHandlerHostedService.cs +++ b/src/Foundatio/Messaging/MessageHandlerHostedService.cs @@ -42,19 +42,47 @@ public MessageHandlerHostedService(IServiceProvider serviceProvider, IEnumerable public async Task StartAsync(CancellationToken cancellationToken) { - foreach (var registration in _registrations) + try { - var disposable = await registration.StartAsync(_serviceProvider, cancellationToken).AnyContext(); - _started.Add(disposable); - _logger.LogInformation("Started message handler {Handler}", registration.Description); + foreach (var registration in _registrations) + { + var disposable = await registration.StartAsync(_serviceProvider, cancellationToken).AnyContext(); + _started.Add(disposable); + _logger.LogInformation("Started message handler {Handler}", registration.Description); + } + } + catch + { + // A hosted service whose StartAsync throws is not sent StopAsync, so dispose whatever we already started + // rather than leaking those consumers' background receive loops. + await DisposeStartedAsync().AnyContext(); + throw; } } - public async Task StopAsync(CancellationToken cancellationToken) - { - foreach (var disposable in _started) - await disposable.DisposeAsync().AnyContext(); + public Task StopAsync(CancellationToken cancellationToken) => DisposeStartedAsync(); - _started.Clear(); + private async Task DisposeStartedAsync() + { + try + { + // Dispose every started consumer even if one throws (e.g. a broker connection dropped mid-shutdown), so a + // single failure can't leak the rest. + foreach (var disposable in _started) + { + try + { + await disposable.DisposeAsync().AnyContext(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error disposing message handler consumer: {Message}", ex.Message); + } + } + } + finally + { + _started.Clear(); + } } } diff --git a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs index 75906b43e..3bb219ea1 100644 --- a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs +++ b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Exceptionless; using Foundatio.AsyncEx; -using Foundatio.Jobs; using Foundatio.Jobs.Legacy; using Foundatio.Messaging; using Foundatio.Messaging.Legacy; From 321ae96583f7ae07a077d5ad2d6862f0fe00e6d1 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 16:42:05 -0500 Subject: [PATCH 38/57] One IMessageBus: the caller's verb decides delivery, handlers are topology-free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The primary messaging client is now a single IMessageBus with two verbs: SendAsync (a command / unit of work — exactly one handler instance across the fleet processes it) and PublishAsync (an event — each subscribing service receives one copy, its instances competing). Handler registration carries no topology decision: AddHandler(o => ...) replaces AddQueueHandler/ AddBroadcastHandler, wiring both a queue consumer (Send target) and a service-identity subscription (Publish target) for the type. PerInstance opts a handler into per-replica fan-out for published messages (cache invalidation, config reload); Subscription forms an independent named subscriber group. The underlying IQueue/IPubSub clients remain for advanced scenarios (pull receive, programmatic consumers); MessageBus is a facade over them, and DI registers it as the primary client. The interface takes the IMessageBus name from the legacy bus, so the remaining dual-namespace files flip to pure Foundatio.Messaging.Legacy (the builder's legacy compat methods use an alias) — same pattern as the IJob fork. Producer options are renamed to match the verbs: QueueMessageOptions -> MessageSendOptions, PubSubMessageOptions -> MessagePublishOptions. Because one type can now be both sent and published, transports segregate queue and topic namespaces: Redis Streams prefixes stream keys by role ("q:"/"t:") and the in-memory transport keys destinations the same way (subscriptions are queue-shaped destinations a topic fans into, mirroring SNS->SQS). Without this a same-named queue and topic shared one stream and cross-delivered. Sample rewritten to the new surface: endpoints inject IMessageBus and the verb reads as intent; the announcement handler demonstrates PerInstance. Tests: DeclarativeRegistrationTests now proves send/publish isolation on one type and once-per-service vs PerInstance across two simulated instances; a live Redis test proves the same isolation on Streams. Full solution builds (net8 + net10, warnings-as-errors); in-memory 2005 green; live Redis 28 green. Co-Authored-By: Claude Opus 4.8 --- samples/Foundatio.MessagingSample/Handlers.cs | 9 +- samples/Foundatio.MessagingSample/Messages.cs | 9 +- samples/Foundatio.MessagingSample/Program.cs | 22 +-- samples/Foundatio.MessagingSample/README.md | 50 ++++--- .../Jobs/ScheduledJobInstance.cs | 1 - .../Jobs/ScheduledJobService.cs | 1 - .../Messaging/RedisStreamsMessageTransport.cs | 22 +-- .../Caching/HybridCacheClientTestBase.cs | 1 - .../Messaging/MessageBusTestBase.cs | 11 +- src/Foundatio/Caching/HybridCacheClient.cs | 1 - src/Foundatio/FoundatioServicesExtensions.cs | 127 ++++++++++-------- src/Foundatio/Lock/CacheLockProvider.cs | 1 - src/Foundatio/Messaging/IMessageHandler.cs | 43 +++++- .../Messaging/InMemoryMessageTransport.cs | 118 ++++++++-------- src/Foundatio/Messaging/MessageBus.cs | 80 +++++++++++ .../Messaging/MessageHandlerHostedService.cs | 4 +- src/Foundatio/Messaging/MessageQueue.cs | 22 +-- src/Foundatio/Messaging/PubSub.cs | 22 +-- .../RedisJobStoreIntegrationTests.cs | 4 +- .../RedisStreamsTransportIntegrationTests.cs | 55 ++++++++ .../Caching/InMemoryHybridCacheClientTests.cs | 1 - .../DeclarativeRegistrationTests.cs | 106 +++++++++++++-- .../Locks/InMemoryLockTests.cs | 1 - .../Messaging/InMemoryMessageBusTests.cs | 1 - .../Foundatio.Tests/Messaging/PubSubTests.cs | 8 +- .../Queue/MessageQueueTests.cs | 14 +- 26 files changed, 500 insertions(+), 234 deletions(-) create mode 100644 src/Foundatio/Messaging/MessageBus.cs diff --git a/samples/Foundatio.MessagingSample/Handlers.cs b/samples/Foundatio.MessagingSample/Handlers.cs index 3aec7d30e..94cef3619 100644 --- a/samples/Foundatio.MessagingSample/Handlers.cs +++ b/samples/Foundatio.MessagingSample/Handlers.cs @@ -6,8 +6,9 @@ namespace Foundatio.MessagingSample; public sealed record InstanceInfo(string Id); /// -/// Handles orders off the queue — registered with AddQueueHandler, so exactly one running instance processes -/// each order (competing consumers). Resolved from DI per message; throwing would trigger retry/dead-letter. +/// Handles orders. Registration carries no topology — orders arrive here because the endpoint calls +/// bus.SendAsync, so exactly one running instance processes each order (competing consumers). Resolved from DI +/// per message; throwing would trigger retry/dead-letter. /// public sealed class ProcessOrderHandler(InstanceInfo instance, ILogger logger) : IMessageHandler { @@ -19,8 +20,8 @@ public Task HandleAsync(IReceivedMessage message, CancellationToke } /// -/// Handles announcements — registered with AddBroadcastHandler, so every running instance receives its own copy -/// (fan-out via a per-instance subscription). +/// Handles announcements published via bus.PublishAsync. Registered with PerInstance = true, so every +/// running replica receives its own copy — without it, the default is once per service (replicas compete). /// public sealed class AnnouncementHandler(InstanceInfo instance, ILogger logger) : IMessageHandler { diff --git a/samples/Foundatio.MessagingSample/Messages.cs b/samples/Foundatio.MessagingSample/Messages.cs index b37fefe65..4196e4daf 100644 --- a/samples/Foundatio.MessagingSample/Messages.cs +++ b/samples/Foundatio.MessagingSample/Messages.cs @@ -3,8 +3,9 @@ namespace Foundatio.MessagingSample; /// -/// A unit of work processed off a queue. The names the destination ("orders"); -/// with competing consumers, each order is handled by exactly one running instance. +/// A command / unit of work, delivered with bus.SendAsync — exactly one running instance handles each one. +/// The names the destination ("orders"); without it the kebab-cased type name +/// ("process-order") is used. /// [MessageRoute("orders")] public class ProcessOrder @@ -14,8 +15,8 @@ public class ProcessOrder } /// -/// A broadcast event published to a topic ("announcements"). With a per-instance subscription, every running instance -/// receives its own copy. +/// An event, delivered with bus.PublishAsync — each subscribing service receives one copy (and this sample's +/// handler opts into PerInstance, so every replica gets its own). /// [MessageRoute("announcements")] public class Announcement diff --git a/samples/Foundatio.MessagingSample/Program.cs b/samples/Foundatio.MessagingSample/Program.cs index 9f93f8522..aa6e6e6d2 100644 --- a/samples/Foundatio.MessagingSample/Program.cs +++ b/samples/Foundatio.MessagingSample/Program.cs @@ -9,11 +9,12 @@ builder.Services.AddSingleton(new InstanceInfo(Guid.NewGuid().ToString("N")[..6])); builder.Services.AddFoundatio() - // Messaging on AWS (SQS/SNS). Handlers are registered declaratively; Foundatio hosts them and dispatches to them — - // no hand-written IHostedService. Swap UseAws() for UseRedis() to run messaging on Redis Streams instead. + // Messaging on AWS (SQS/SNS). Handlers carry no topology decision — the caller's verb decides delivery + // (bus.SendAsync = one instance across the fleet, bus.PublishAsync = once per subscribing service). Swap UseAws() + // for UseRedis() to run messaging on Redis Streams without touching any handler. .Messaging.UseAws() - .Messaging.AddQueueHandler() // competing consumers: one instance per order - .Messaging.AddBroadcastHandler() // fan-out: every instance gets each announcement + .Messaging.AddHandler() + .Messaging.AddHandler(o => o.PerInstance = true) // every replica shows the announcement // Durable jobs on Redis so any instance can claim them. The pump (auto-registered) runs submitted jobs and // materializes the CRON schedules below — no manual scheduling call. .Jobs.UseRedis() @@ -26,14 +27,15 @@ app.MapGet("/", (InstanceInfo instance) => Results.Ok(new { service = "Foundatio messaging sample", instance = instance.Id })); -// QUEUE — competing consumers: exactly one instance processes each order (handled by ProcessOrderHandler). -app.MapPost("/orders", async (ProcessOrder order, IQueue queue) => - Results.Accepted(value: new { queued = await queue.EnqueueAsync(order) })); +// SEND — a command / unit of work: exactly one instance processes each order (handled by ProcessOrderHandler). +app.MapPost("/orders", async (ProcessOrder order, IMessageBus bus) => + Results.Accepted(value: new { queued = await bus.SendAsync(order) })); -// PUB/SUB — fan-out: every instance receives each announcement (handled by AnnouncementHandler). -app.MapPost("/announcements", async (Announcement announcement, IPubSub pubSub) => +// PUBLISH — an event: subscribers receive it per their registration (AnnouncementHandler opts into PerInstance, so +// every running replica logs each announcement). +app.MapPost("/announcements", async (Announcement announcement, IMessageBus bus) => { - await pubSub.PublishAsync(announcement); + await bus.PublishAsync(announcement); return Results.Accepted(value: new { published = announcement.Text }); }); diff --git a/samples/Foundatio.MessagingSample/README.md b/samples/Foundatio.MessagingSample/README.md index 51b229116..e9cd127b6 100644 --- a/samples/Foundatio.MessagingSample/README.md +++ b/samples/Foundatio.MessagingSample/README.md @@ -1,22 +1,31 @@ # Foundatio.MessagingSample -A minimal ASP.NET app that shows the redesigned Foundatio **messaging** (queues + pub/sub) and **durable jobs** in a -real, scaled-out setup. It runs under Aspire with **3 replicas**, so you can watch the distributed behavior: +A minimal ASP.NET app showing the redesigned Foundatio **messaging** (one bus, two verbs) and **durable jobs** in a +real, scaled-out setup. It runs under Aspire with **3 replicas**, so you can watch the distributed behavior. -- **Queue (competing consumers)** — `POST /orders` enqueues work; exactly **one** replica processes each order. Scale +The core idea: **handlers are registered with no topology decision — the caller's verb decides delivery.** + +- `bus.SendAsync(msg)` — a command / unit of work: exactly **one** instance across the fleet processes it. +- `bus.PublishAsync(msg)` — an event: each subscribing **service** receives one copy (a scaled service's replicas + compete for it), or **every replica** when the handler opts in with `PerInstance = true`. + +What the sample demonstrates: + +- **Send (worker queue)** — `POST /orders` calls `bus.SendAsync`; exactly **one** replica processes each order. Scale up and the work spreads out. -- **Pub/Sub (fan-out)** — `POST /announcements` publishes to a topic; **every** replica receives its own copy (each - uses a per-instance subscription). -- **Durable job** — `POST /reports` submits a job; whichever replica's runtime pump claims it runs it. Poll - `GET /reports/{id}` to watch its status/progress. -- **CRON jobs** — scheduled recurring work, deduped through the shared runtime store so **scope** decides fan-out: - - `heartbeat` — Global, every minute → runs on **one** replica per tick (leader/singleton). - - `refresh-cache` — PerNode, every minute → runs on **every** replica per tick (per-instance maintenance). - - `sweep-stale-orders` — Global, every 2 minutes → a periodic maintenance sweep on one replica. - -Messaging runs on **AWS SQS/SNS** (via a LocalStack container) and durable jobs on **Redis** — both transports wired -from one clean `AddFoundatio()` chain in [`Program.cs`](Program.cs). The transport is selected by `Messaging:Provider` -(`Aws` or `Redis`), so you can flip it without touching any queue/pub-sub code. +- **Publish (events)** — `POST /announcements` calls `bus.PublishAsync`; the announcement handler registers with + `PerInstance = true`, so **every** replica logs each announcement. +- **Durable job** — `POST /reports` submits a job via `IJobClient`; whichever replica's runtime pump claims it runs it. + Poll `GET /reports/{id}` to watch its status/progress. +- **CRON jobs** — declared with `.Jobs.AddCronJob(cron)` and scheduled automatically; occurrences are deduped + through the shared runtime store so **scope** decides fan-out: + - `HeartbeatJob` — Global, every minute → runs on **one** replica per tick (leader/singleton). + - `RefreshCacheJob` — PerNode, every minute → runs on **every** replica per tick (per-instance maintenance). + - `SweepStaleOrdersJob` — Global, every 2 minutes → a periodic maintenance sweep on one replica. + +Messaging runs on **AWS SQS/SNS** (via a LocalStack container) and durable jobs on **Redis** — all wired from one +clean `AddFoundatio()` chain in [`Program.cs`](Program.cs). Swap `UseAws()` for `UseRedis()` to run messaging on +Redis Streams without touching a single handler. ## Run it (Aspire) @@ -30,21 +39,20 @@ The Aspire dashboard launches Redis + LocalStack and 3 replicas of the service. # fire several orders — watch them load-balance across the 3 replicas' logs for i in $(seq 1 6); do curl -sX POST /orders -H 'content-type: application/json' -d "{\"product\":\"widget\",\"quantity\":$i}"; done -# publish an announcement — every replica logs it +# publish an announcement — every replica logs it (the handler is PerInstance) curl -sX POST /announcements -H 'content-type: application/json' -d '{"text":"hello all"}' # submit a durable job, then poll it job=$(curl -sX POST /reports | jq -r .jobId); curl -s /reports/$job ``` -The per-instance id in each log line (`[abc123] processed order: ...`) makes the distribution obvious. To run messaging -on Redis Streams instead of AWS, set `Messaging__Provider=Redis` on the service in the AppHost. +The per-instance id in each log line (`[abc123] processed order: ...`) makes the distribution obvious. ## Run it standalone (no Aspire) -Point it at a Redis instance (defaults to `localhost:6399`) and use the Redis transport: +Swap `UseAws()` for `UseRedis()` in `Program.cs` (or run LocalStack for the AWS transport), point at a Redis +instance, and run: ```sh -Messaging__Provider=Redis ConnectionStrings__Redis=localhost:6399 \ - dotnet run --project samples/Foundatio.MessagingSample +ConnectionStrings__Redis=localhost:6399 dotnet run --project samples/Foundatio.MessagingSample ``` diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs index b30a1a7cb..de9b59697 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobInstance.cs @@ -8,7 +8,6 @@ using Foundatio.Cronos; using Foundatio.Jobs.Legacy; using Foundatio.Lock; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs index cec478544..b557099c2 100644 --- a/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs +++ b/src/Foundatio.Extensions.Hosting/Jobs/ScheduledJobService.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Extensions.Hosting.Startup; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 927a85ea7..3e17f74b5 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -65,9 +65,9 @@ public async Task SendAsync(string destination, IReadOnlyList(messages.Count); foreach (var message in messages) { @@ -222,7 +222,7 @@ public async Task> ReceiveDeadLetteredAsync(string ArgumentException.ThrowIfNullOrEmpty(destination); ArgumentNullException.ThrowIfNull(request); - RedisKey deadKey = DeadKey(StreamKey(destination)); + RedisKey deadKey = DeadKey(Resolve(destination).StreamKey); var entries = await _db.StreamRangeAsync(deadKey, count: Math.Max(1, request.MaxMessages)).ConfigureAwait(false); if (entries.Length == 0) return []; @@ -255,12 +255,12 @@ public async Task EnsureAsync(IReadOnlyList declarations case DestinationRole.Subscription: case DestinationRole.Binding: string topic = declaration.Source ?? declaration.Name; - var sub = new ResolvedSource(StreamKey(topic), declaration.Name, "$"); + var sub = new ResolvedSource(TopicStreamKey(topic), declaration.Name, "$"); _sources[declaration.Name] = sub; await EnsureGroupAsync(sub).ConfigureAwait(false); break; default: - var queue = new ResolvedSource(StreamKey(declaration.Name), _options.DefaultConsumerGroup, "0"); + var queue = new ResolvedSource(QueueStreamKey(declaration.Name), _options.DefaultConsumerGroup, "0"); _sources[declaration.Name] = queue; await EnsureGroupAsync(queue).ConfigureAwait(false); break; @@ -354,8 +354,8 @@ private ResolvedSource Resolve(string source) // PubSub facade sources are "topic/subscription" (a consumer group on the topic stream); a bare name is a queue // on the default group. Parse via the shared convention rather than re-deriving the split. return SubscriptionAddress.TryParse(source, out string topic, out string subscription) - ? new ResolvedSource(StreamKey(topic), subscription, "$") - : new ResolvedSource(StreamKey(source), _options.DefaultConsumerGroup, "0"); + ? new ResolvedSource(TopicStreamKey(topic), subscription, "$") + : new ResolvedSource(QueueStreamKey(source), _options.DefaultConsumerGroup, "0"); } private TransportEntry ToEntry(string destination, ResolvedSource? resolved, StreamEntry entry, int deliveries, string token) @@ -442,7 +442,11 @@ private static string ParseToken(RedisValue meta) return bar >= 0 ? s[..bar] : s; } - private RedisKey StreamKey(string name) => $"{_prefix}{name}"; + // Streams are namespaced by role ("q:" queue, "t:" topic) because an XADD lands on whichever stream the key names: + // without the split, a message type both sent and published would share one stream and cross-deliver (a publish + // consumed as queue work and vice versa). Subscriptions are consumer groups on the topic stream. + private RedisKey QueueStreamKey(string name) => $"{_prefix}q:{name}"; + private RedisKey TopicStreamKey(string name) => $"{_prefix}t:{name}"; private static RedisKey DeadKey(RedisKey streamKey) => streamKey.ToString() + ":dead"; private static RedisKey LockKey(ResolvedSource r) => $"{r.StreamKey}:lock:{r.Group}"; private static RedisKey MetaKey(ResolvedSource r) => $"{r.StreamKey}:meta:{r.Group}"; diff --git a/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs b/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs index 01cebc5ca..c2c77c1cf 100644 --- a/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs +++ b/src/Foundatio.TestHarness/Caching/HybridCacheClientTestBase.cs @@ -3,7 +3,6 @@ using System.Threading.Tasks; using Foundatio.AsyncEx; using Foundatio.Caching; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Microsoft.Extensions.Logging; diff --git a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs index 38abf2689..54d848e21 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Exceptionless; using Foundatio.AsyncEx; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Foundatio.Tests.Serializer; @@ -862,7 +861,7 @@ public virtual async Task PublishAsync_AfterDispose_ThrowsMessageBusExceptionAsy await messageBus.DisposeAsync(); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.PublishAsync(new SimpleMessageA { Data = "Hello" })); } finally @@ -873,7 +872,7 @@ await Assert.ThrowsAsync(async () => /// /// Verifies that cancellation is surfaced as OperationCanceledException, not swallowed - /// or wrapped in MessageBusException. This ensures callers can distinguish between + /// or wrapped in Foundatio.Messaging.MessageBusException. This ensures callers can distinguish between /// cancellation and actual publish failures. /// public virtual async Task PublishAsync_WithCancellation_ThrowsOperationCanceledExceptionAsync() @@ -946,7 +945,7 @@ public virtual async Task PublishAsync_WithSerializationFailure_ThrowsSerializer await messageBus.SubscribeAsync(_ => { }); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.PublishAsync(new SimpleMessageA { Data = "test" }, cancellationToken: TestCancellationToken)); } finally @@ -967,7 +966,7 @@ public virtual async Task SubscribeAsync_AfterDispose_ThrowsMessageBusExceptionA await messageBus.DisposeAsync(); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.SubscribeAsync(_ => { })); } finally @@ -1023,7 +1022,7 @@ await messageBus.SubscribeAsync(msg => /// /// Verifies that cancellation is surfaced as OperationCanceledException, not swallowed - /// or wrapped in MessageBusException. This ensures callers can distinguish between + /// or wrapped in Foundatio.Messaging.MessageBusException. This ensures callers can distinguish between /// cancellation and actual subscribe failures. /// public virtual async Task SubscribeAsync_WithCancellation_ThrowsOperationCanceledExceptionAsync() diff --git a/src/Foundatio/Caching/HybridCacheClient.cs b/src/Foundatio/Caching/HybridCacheClient.cs index 596aef5d0..ce417e6c2 100644 --- a/src/Foundatio/Caching/HybridCacheClient.cs +++ b/src/Foundatio/Caching/HybridCacheClient.cs @@ -5,7 +5,6 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.AsyncEx; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index cad410a05..2d60ac4b7 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -7,7 +7,7 @@ using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Messaging; -using Foundatio.Messaging.Legacy; +using Legacy = Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Resilience; using Foundatio.Serializer; @@ -262,19 +262,19 @@ internal MessagingBuilder(IFoundatioBuilder builder) IServiceCollection IFoundatioBuilder.Services => _services; FoundatioBuilder IFoundatioBuilder.Builder => _builder; - public FoundatioBuilder Use(IMessageBus messageBus) + public FoundatioBuilder Use(Legacy.IMessageBus messageBus) { _services.ReplaceSingleton(_ => messageBus); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); return _builder; } - public FoundatioBuilder Use(Func factory) + public FoundatioBuilder Use(Func factory) { _services.ReplaceSingleton(factory); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); return _builder; } @@ -311,20 +311,20 @@ public MessagingBuilder RegisterMessageType(string name) where T : class return this; } - public FoundatioBuilder UseInMemory(InMemoryMessageBusOptions? options = null) + public FoundatioBuilder UseInMemory(Legacy.InMemoryMessageBusOptions? options = null) { - _services.ReplaceSingleton(sp => new InMemoryMessageBus(options.UseServices(sp))); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => new Legacy.InMemoryMessageBus(options.UseServices(sp))); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); RegisterMessagingRuntime(sp => new InMemoryMessageTransport(sp.GetService())); return _builder; } - public FoundatioBuilder UseInMemory(Builder config) + public FoundatioBuilder UseInMemory(Builder config) { - _services.ReplaceSingleton(sp => new InMemoryMessageBus(b => b.Configure(config).UseServices(sp))); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => new Legacy.InMemoryMessageBus(b => b.Configure(config).UseServices(sp))); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); + _services.ReplaceSingleton(sp => sp.GetRequiredService()); RegisterMessagingRuntime(sp => new InMemoryMessageTransport(sp.GetService())); return _builder; } @@ -343,61 +343,70 @@ public FoundatioBuilder UseTransport(Func f } /// - /// Registers a handler that processes messages of type from its queue as - /// competing consumers — exactly one running instance handles each message. The handler is resolved from DI in - /// its own scope per message (so it can inject scoped dependencies); throwing triggers the retry/dead-letter - /// policy. A hosted service starts and stops it automatically. + /// Registers a handler for messages of type . Registration carries no topology + /// decision — the caller's verb on decides delivery: a SendAsync is processed + /// by exactly one handler instance across the fleet (competing consumers), and a PublishAsync is received + /// once per subscribing service (a scaled service's instances compete), or by every instance when + /// is set. The handler is resolved from DI in its own scope per + /// message (so it can inject scoped dependencies); throwing triggers the retry/dead-letter policy. A single + /// hosted service starts and stops all registered handlers. /// - public FoundatioBuilder AddQueueHandler(QueueConsumerOptions? options = null) + public FoundatioBuilder AddHandler(Action? configure = null) where TMessage : class where THandler : class, IMessageHandler { _services.TryAddScoped(); - return AddHandler($"queue:{typeof(TMessage).Name} -> {typeof(THandler).Name}", async (sp, ct) => - await sp.GetRequiredService().StartConsumerAsync( - (message, c) => DispatchAsync(sp, message, c), options, ct).ConfigureAwait(false)); + return AddHandlerListeners(typeof(THandler).Name, static (sp, message, ct) => DispatchAsync(sp, message, ct), configure); } /// - /// Registers a delegate handler for messages of type from its queue as competing - /// consumers. A hosted service starts and stops it automatically. + /// Registers a delegate handler for messages of type ; see + /// for the delivery semantics. /// - public FoundatioBuilder AddQueueHandler(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null) + public FoundatioBuilder AddHandler(Func, CancellationToken, Task> handler, Action? configure = null) where TMessage : class { ArgumentNullException.ThrowIfNull(handler); - return AddHandler($"queue:{typeof(TMessage).Name}", async (sp, ct) => - await sp.GetRequiredService().StartConsumerAsync(handler, options, ct).ConfigureAwait(false)); + return AddHandlerListeners(null, (_, message, ct) => handler(message, ct), configure); } - /// - /// Registers a handler that receives every message of type published to its - /// topic — a per-instance subscription means every running instance gets its own copy (fan-out). Pass an explicit - /// to share a named subscription (load-balanced) instead. Resolved from DI per - /// message; a hosted service starts and stops it automatically. - /// - public FoundatioBuilder AddBroadcastHandler(string? subscription = null) - where TMessage : class where THandler : class, IMessageHandler - { - _services.TryAddScoped(); - string name = subscription ?? UniqueSubscriptionName(); - return AddHandler($"broadcast:{typeof(TMessage).Name} -> {typeof(THandler).Name}", async (sp, ct) => - await sp.GetRequiredService().SubscribeAsync( - (message, c) => DispatchAsync(sp, message, c), - new PubSubSubscriptionOptions { Subscription = name }, ct).ConfigureAwait(false)); - } - - /// - /// Registers a delegate handler that receives every message of type published to - /// its topic (fan-out via a per-instance subscription). A hosted service starts and stops it automatically. - /// - public FoundatioBuilder AddBroadcastHandler(Func, CancellationToken, Task> handler, string? subscription = null) + private FoundatioBuilder AddHandlerListeners(string? handlerName, Func, CancellationToken, Task> dispatch, Action? configure) where TMessage : class { - ArgumentNullException.ThrowIfNull(handler); - string name = subscription ?? UniqueSubscriptionName(); - return AddHandler($"broadcast:{typeof(TMessage).Name}", async (sp, ct) => - await sp.GetRequiredService().SubscribeAsync(handler, - new PubSubSubscriptionOptions { Subscription = name }, ct).ConfigureAwait(false)); + var options = new MessageHandlerOptions(); + configure?.Invoke(options); + + if (options.PerInstance && !String.IsNullOrEmpty(options.Subscription)) + throw new ArgumentException("PerInstance and Subscription are mutually exclusive: PerInstance derives a unique per-instance subscription.", nameof(configure)); + + string suffix = handlerName is null ? String.Empty : $" -> {handlerName}"; + + // Send target: competing consumers on the message type's queue destination — one handler instance across + // the fleet processes each SendAsync. + AddHandlerRegistration($"send:{typeof(TMessage).Name}{suffix}", async (sp, ct) => + await sp.GetRequiredService().StartConsumerAsync((message, c) => dispatch(sp, message, c), new QueueConsumerOptions + { + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff + }, ct).ConfigureAwait(false)); + + // Publish target: this service's subscription on the message type's topic. The default subscription + // identity is the service identity, so scaled instances share one subscription and compete — each service + // handles a published message once. PerInstance instead takes a unique per-instance subscription so every + // instance receives its own copy. + string? subscription = options.PerInstance ? UniqueSubscriptionName() : options.Subscription; + AddHandlerRegistration($"publish:{typeof(TMessage).Name}{suffix}", async (sp, ct) => + await sp.GetRequiredService().SubscribeAsync((message, c) => dispatch(sp, message, c), new PubSubSubscriptionOptions + { + Subscription = subscription, + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff + }, ct).ConfigureAwait(false)); + + return _builder; } private static async Task DispatchAsync(IServiceProvider serviceProvider, IReceivedMessage message, CancellationToken cancellationToken) @@ -410,12 +419,11 @@ private static async Task DispatchAsync(IServiceProvider ser private static string UniqueSubscriptionName() => $"{Environment.MachineName}-{Guid.NewGuid():N}"; - private FoundatioBuilder AddHandler(string description, Func> start) + private void AddHandlerRegistration(string description, Func> start) { _services.AddSingleton(new MessageHandlerRegistration { Description = description, StartAsync = start }); if (!_services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(MessageHandlerHostedService))) _services.AddSingleton(); - return _builder; } private void RegisterMessagingRuntime(Func factory) @@ -462,6 +470,9 @@ private void RegisterMessageClients() _services.ReplaceSingleton(sp => new MessageTypeRegistry(sp.GetServices())); _services.ReplaceSingleton(sp => new MessageQueue(sp.GetRequiredService(), CreateQueueOptions(sp))); _services.ReplaceSingleton(sp => new PubSub(sp.GetRequiredService(), CreatePubSubOptions(sp))); + // The primary client: one bus, two verbs. The underlying queue/pub-sub clients stay resolvable for + // advanced scenarios (pull receive, programmatic consumers). + _services.ReplaceSingleton(sp => new MessageBus(sp.GetRequiredService(), sp.GetRequiredService())); } private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) @@ -679,7 +690,7 @@ public FoundatioBuilder UseCache() // gets all services from the ICacheClient instance _services.ReplaceSingleton(sp => new CacheLockProvider( sp.GetRequiredService(), - sp.GetService(), // optional for more efficient lock release notifications + sp.GetService(), // optional for more efficient lock release notifications sp.GetService(), sp.GetService(), sp.GetService() diff --git a/src/Foundatio/Lock/CacheLockProvider.cs b/src/Foundatio/Lock/CacheLockProvider.cs index 5edcf2fb2..d48b7bb14 100644 --- a/src/Foundatio/Lock/CacheLockProvider.cs +++ b/src/Foundatio/Lock/CacheLockProvider.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Foundatio.AsyncEx; using Foundatio.Caching; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; diff --git a/src/Foundatio/Messaging/IMessageHandler.cs b/src/Foundatio/Messaging/IMessageHandler.cs index f82fa77da..b68ae68fb 100644 --- a/src/Foundatio/Messaging/IMessageHandler.cs +++ b/src/Foundatio/Messaging/IMessageHandler.cs @@ -1,16 +1,51 @@ +using System; using System.Threading; using System.Threading.Tasks; namespace Foundatio.Messaging; /// -/// Handles messages of type received from a queue or a pub/sub subscription. Register a -/// handler with AddFoundatio().Messaging.AddQueueHandler<T, THandler>() (competing consumers) or -/// AddBroadcastHandler<T, THandler>() (fan-out); a hosted service then starts and dispatches to it. -/// Handlers are resolved from DI in their own scope per message, so they can inject scoped dependencies. Throwing from +/// Handles messages of type . Register with +/// AddFoundatio().Messaging.AddHandler<T, THandler>() — registration carries no topology decision; the +/// caller's verb on decides delivery (SendAsync = one handler instance across the +/// fleet, PublishAsync = once per subscribing service). A hosted service starts and dispatches to it. Handlers +/// are resolved from DI in their own scope per message, so they can inject scoped dependencies. Throwing from /// triggers the core's retry/dead-letter policy. /// public interface IMessageHandler where T : class { Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken); } + +/// +/// Options for a declaratively-registered message handler (AddFoundatio().Messaging.AddHandler<T, THandler>(o => ...)). +/// +public sealed class MessageHandlerOptions +{ + /// + /// When true, published messages are received by EVERY running instance (each instance takes a unique + /// subscription), instead of once per service. For per-instance local state — cache invalidation, config reload. + /// Mutually exclusive with . Does not affect sent messages, which always go to exactly + /// one instance. + /// + public bool PerInstance { get; set; } + + /// + /// The subscriber-group identity used for published messages. Defaults to the service identity, so all instances + /// of a service share one subscription and compete (each published message is handled once per service). Set an + /// explicit name to form an independent named subscriber group. + /// + public string? Subscription { get; set; } + + /// Maximum messages this handler processes concurrently per instance. Default 1. + public int MaxConcurrency { get; set; } = 1; + + /// Maximum delivery attempts before dead-lettering. Null uses the default . + public int? MaxAttempts { get; set; } + + /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's timing. + public Func? RedeliveryBackoff { get; set; } + + /// Whether messages auto-complete when the handler returns (default) or are settled manually. + public AckMode AckMode { get; set; } = AckMode.Auto; +} diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 43964a409..a2afdfd36 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -52,14 +52,13 @@ public Task SendAsync(string destination, IReadOnlyList _timeProvider.GetUtcNow()) throw new NotSupportedException($"Transport \"{GetType().Name}\" does not support native delayed delivery. Use the runtime-store scheduled dispatch fallback."); + // The caller-stated role picks the physical namespace, so a queue and a topic can share a route name (a message + // type that is both sent and published) without colliding or cross-delivering. + string key = options.DestinationRole == DestinationRole.Topic ? TopicKey(destination) : SourceKey(destination); + var results = new SendItemResult[messages.Count]; for (int index = 0; index < messages.Count; index++) { @@ -67,8 +66,8 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, Re ArgumentException.ThrowIfNullOrEmpty(source); int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; - var state = GetOrAddDestination(source, DestinationRole.Queue); + var state = GetOrAddDestination(SourceKey(source)); var entries = new List(maxMessages); DateTimeOffset? waitUntil = request.MaxWaitTime is { } waitTime && waitTime > TimeSpan.Zero ? _timeProvider.GetUtcNow().Add(waitTime) @@ -241,7 +240,7 @@ public Task> ReceiveDeadLetteredAsync(string desti ArgumentException.ThrowIfNullOrEmpty(destination); ArgumentNullException.ThrowIfNull(request); - if (!_destinations.TryGetValue(destination, out var state)) + if (!_destinations.TryGetValue(SourceKey(destination), out var state)) return Task.FromResult>([]); int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; @@ -282,7 +281,7 @@ public Task GetStatsAsync(string destination, Cancellat ct.ThrowIfCancellationRequested(); ArgumentException.ThrowIfNullOrEmpty(destination); - if (!_destinations.TryGetValue(destination, out var state)) + if (!_destinations.TryGetValue(SourceKey(destination), out var state)) return Task.FromResult(new MessageDestinationStats()); return Task.FromResult(new MessageDestinationStats @@ -310,14 +309,14 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc switch (declaration.Role) { case DestinationRole.Queue: - GetOrAddDestination(declaration.Name, DestinationRole.Queue); + GetOrAddDestination(QueueKey(declaration.Name)); break; case DestinationRole.Topic: - SetRole(declaration.Name, DestinationRole.Topic); - _topicSubscriptions.GetOrAdd(declaration.Name, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + _roles.TryAdd(TopicKey(declaration.Name), DestinationRole.Topic); + _topicSubscriptions.GetOrAdd(TopicKey(declaration.Name), static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); break; case DestinationRole.Subscription: - GetOrAddDestination(declaration.Name, DestinationRole.Subscription); + GetOrAddDestination(QueueKey(declaration.Name)); if (!String.IsNullOrEmpty(declaration.Source)) AddTopicSubscription(declaration.Source, declaration.Name); break; @@ -325,7 +324,7 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc if (String.IsNullOrEmpty(declaration.Source)) throw new ArgumentException("A binding declaration must specify a source topic.", nameof(declarations)); - GetOrAddDestination(declaration.Name, DestinationRole.Subscription); + GetOrAddDestination(QueueKey(declaration.Name)); AddTopicSubscription(declaration.Source, declaration.Name); break; default: @@ -342,13 +341,16 @@ public Task DeleteAsync(string name, CancellationToken ct) ct.ThrowIfCancellationRequested(); ArgumentException.ThrowIfNullOrEmpty(name); - _roles.TryRemove(name, out _); - if (_destinations.TryRemove(name, out var removed)) - removed.Complete(); - _topicSubscriptions.TryRemove(name, out _); + foreach (string key in (string[])[QueueKey(name), TopicKey(name)]) + { + _roles.TryRemove(key, out _); + if (_destinations.TryRemove(key, out var removed)) + removed.Complete(); + _topicSubscriptions.TryRemove(key, out _); - foreach (var subscriptions in _topicSubscriptions.Values) - subscriptions.TryRemove(name, out _); + foreach (var subscriptions in _topicSubscriptions.Values) + subscriptions.TryRemove(key, out _); + } return Task.CompletedTask; } @@ -359,7 +361,7 @@ public Task ExistsAsync(string name, CancellationToken ct) ct.ThrowIfCancellationRequested(); ArgumentException.ThrowIfNullOrEmpty(name); - return Task.FromResult(_roles.ContainsKey(name)); + return Task.FromResult(_roles.ContainsKey(QueueKey(name)) || _roles.ContainsKey(TopicKey(name))); } public ValueTask DisposeAsync() @@ -435,12 +437,13 @@ private async Task RunPushSubscriptionAsync(string source, Func new DestinationState()); - } + // Internal state is keyed by role-qualified names: "t:" for topics, "q:" for every receivable destination (queues + // AND subscriptions — a subscription is a queue-shaped destination a topic fans into, exactly like an SNS-bound SQS + // queue). This gives a queue/subscription and a topic sharing a route name distinct namespaces, as real brokers do. + private static string QueueKey(string name) => "q:" + name; + private static string TopicKey(string name) => "t:" + name; + private static string SourceKey(string name) => QueueKey(name); - private DestinationState GetExistingDestination(string name) - { - if (_destinations.TryGetValue(name, out var destination)) - return destination; + private static DestinationRole RoleForKey(string key) => key[0] == 't' ? DestinationRole.Topic : DestinationRole.Queue; - throw new ReceiptExpiredException($"The destination \"{name}\" no longer exists."); + private DestinationState GetOrAddDestination(string key) + { + _roles.TryAdd(key, RoleForKey(key)); + return _destinations.GetOrAdd(key, static _ => new DestinationState()); } - private void SetRole(string name, DestinationRole role) + private DestinationState GetExistingDestination(string key) { - _roles.AddOrUpdate(name, role, (_, existing) => - { - if (existing == role) - return existing; - - if (existing == DestinationRole.Queue && role == DestinationRole.Subscription) - return role; - - if (existing == DestinationRole.Subscription && role == DestinationRole.Queue) - return existing; + if (_destinations.TryGetValue(key, out var destination)) + return destination; - throw new InvalidOperationException($"Destination \"{name}\" is already declared as {existing}."); - }); + throw new ReceiptExpiredException($"The destination \"{key}\" no longer exists."); } private void AddTopicSubscription(string topic, string subscription) { - SetRole(topic, DestinationRole.Topic); - GetOrAddDestination(subscription, DestinationRole.Subscription); - var subscriptions = _topicSubscriptions.GetOrAdd(topic, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); - subscriptions[subscription] = 0; + string topicKey = TopicKey(topic); + _roles.TryAdd(topicKey, DestinationRole.Topic); + GetOrAddDestination(QueueKey(subscription)); + var subscriptions = _topicSubscriptions.GetOrAdd(topicKey, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + subscriptions[QueueKey(subscription)] = 0; } private static MessagePriority NormalizePriority(MessagePriority priority) diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs new file mode 100644 index 000000000..e9f43afd2 --- /dev/null +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Utility; + +namespace Foundatio.Messaging; + +/// +/// The primary messaging client. The verb carries the delivery semantic, so handlers are registered without any +/// topology decision (AddFoundatio().Messaging.AddHandler<T, THandler>()): +/// +/// — a command / unit of work: exactly one handler instance across the fleet processes +/// it (competing consumers on the message type's queue destination). +/// — an event: every subscribing service receives one copy on its own subscription, +/// and a scaled service's instances compete for that copy (so side effects happen once per service, not once per +/// replica). A handler registered with PerInstance = true instead receives a copy on every instance. +/// +/// Retry and dead-lettering are core-owned and identical for both verbs: a handler that throws triggers redelivery and, +/// once attempts are exhausted, the dead-letter policy. +/// +public interface IMessageBus : IAsyncDisposable +{ + /// Sends a command / unit of work; exactly one handler instance across the fleet processes it. + Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default); + + /// Publishes an event; each subscribing service receives one copy (its instances compete). + Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default); +} + +/// +/// Facade unifying the queue (send) and pub/sub (publish) clients behind the two delivery verbs. The underlying +/// clients remain available for advanced scenarios (pull receive, programmatic consumers/subscriptions). +/// Disposing the bus disposes the underlying clients only when ownsClients is true — default false, since DI +/// singleton clients are disposed exactly once by the container. +/// +public sealed class MessageBus : IMessageBus +{ + private readonly IQueue _queue; + private readonly IPubSub _pubSub; + private readonly bool _ownsClients; + + public MessageBus(IQueue queue, IPubSub pubSub, bool ownsClients = false) + { + _queue = queue ?? throw new ArgumentNullException(nameof(queue)); + _pubSub = pubSub ?? throw new ArgumentNullException(nameof(pubSub)); + _ownsClients = ownsClients; + } + + public Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class + => _queue.EnqueueAsync(message, options, cancellationToken); + + public Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class + => _queue.EnqueueBatchAsync(messages, options, cancellationToken); + + public Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) + => _queue.EnqueueBatchAsync(messages, options, cancellationToken); + + public Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + => _pubSub.PublishAsync(message, options, cancellationToken); + + public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class + => _pubSub.PublishBatchAsync(messages, options, cancellationToken); + + public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) + => _pubSub.PublishBatchAsync(messages, options, cancellationToken); + + public async ValueTask DisposeAsync() + { + if (!_ownsClients) + return; + + await _queue.DisposeAsync().AnyContext(); + await _pubSub.DisposeAsync().AnyContext(); + } +} diff --git a/src/Foundatio/Messaging/MessageHandlerHostedService.cs b/src/Foundatio/Messaging/MessageHandlerHostedService.cs index 5503774d7..5d17428bb 100644 --- a/src/Foundatio/Messaging/MessageHandlerHostedService.cs +++ b/src/Foundatio/Messaging/MessageHandlerHostedService.cs @@ -11,8 +11,8 @@ namespace Foundatio.Messaging; /// /// One declarative message-handler registration: a description for logging and a factory that starts the underlying -/// queue consumer or pub/sub subscription and returns it for disposal on shutdown. Built by the -/// AddQueueHandler/AddBroadcastHandler builder methods, which bind the message type at compile time. +/// queue consumer or pub/sub subscription and returns it for disposal on shutdown. Built by the AddHandler +/// builder methods, which bind the message type at compile time (one registration per delivery verb). /// internal sealed class MessageHandlerRegistration { diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs index d710a0429..a00933e21 100644 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ b/src/Foundatio/Messaging/MessageQueue.cs @@ -17,7 +17,7 @@ public enum AckMode Manual } -public sealed record QueueMessageOptions +public sealed record MessageSendOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public TimeSpan? Delay { get; init; } @@ -96,9 +96,9 @@ public sealed record QueueOptions public interface IQueue : IAsyncDisposable { - Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default); + Task EnqueueAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default); Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default); Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default); @@ -189,24 +189,24 @@ public MessageQueue(IMessageTransport transport, QueueOptions? options = null) static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } - public Task EnqueueAsync(T message, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task EnqueueAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); - options ??= new QueueMessageOptions(); + options ??= new MessageSendOptions(); return _core.SendAsync(ScheduledDispatchKind.QueueMessage, typeof(T), message, ToEnvelope(options), GetDestination(typeof(T), options.Destination), ensureDestination: null, cancellationToken); } - public Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); - options ??= new QueueMessageOptions(); + options ??= new MessageSendOptions(); return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); } - public Task EnqueueBatchAsync(IEnumerable messages, QueueMessageOptions? options = null, CancellationToken cancellationToken = default) + public Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(messages); - options ??= new QueueMessageOptions(); + options ??= new MessageSendOptions(); return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages, null, ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); } @@ -278,7 +278,7 @@ private string GetDestination(Type messageType, string? destination) }); } - private static MessageEnvelopeOptions ToEnvelope(QueueMessageOptions options) + private static MessageEnvelopeOptions ToEnvelope(MessageSendOptions options) { return new MessageEnvelopeOptions { diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs index 6d2e663e2..abfc38bee 100644 --- a/src/Foundatio/Messaging/PubSub.cs +++ b/src/Foundatio/Messaging/PubSub.cs @@ -11,7 +11,7 @@ namespace Foundatio.Messaging; -public sealed record PubSubMessageOptions +public sealed record MessagePublishOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public TimeSpan? Delay { get; init; } @@ -56,9 +56,9 @@ public sealed record PubSubOptions public interface IPubSub : IAsyncDisposable { - Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default); + Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default); Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); @@ -96,24 +96,24 @@ public PubSub(IMessageTransport transport, PubSubOptions? options = null) static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } - public Task PublishAsync(T message, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(message); - options ??= new PubSubMessageOptions(); + options ??= new MessagePublishOptions(); return _core.SendAsync(ScheduledDispatchKind.PubSubMessage, typeof(T), message, ToEnvelope(options), GetTopic(typeof(T), options.Topic), EnsureTopicAsync, cancellationToken); } - public Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(messages); - options ??= new PubSubMessageOptions(); + options ??= new MessagePublishOptions(); return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } - public Task PublishBatchAsync(IEnumerable messages, PubSubMessageOptions? options = null, CancellationToken cancellationToken = default) + public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(messages); - options ??= new PubSubMessageOptions(); + options ??= new MessagePublishOptions(); return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } @@ -205,7 +205,7 @@ private string GetSubscription(Type messageType, string topic, string? subscript }); } - private static MessageEnvelopeOptions ToEnvelope(PubSubMessageOptions options) + private static MessageEnvelopeOptions ToEnvelope(MessagePublishOptions options) { return new MessageEnvelopeOptions { diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index 25e528d90..edb6edfad 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -38,7 +38,7 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); var nativeProcessor = CreateProcessor(nativeStore, nativeTransport).Processor; - await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); Assert.Equal(1, nativeTransport.SendCount); Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); Assert.Equal(0, await nativeProcessor.RunDueOccurrencesAsync(now.AddYears(1), cancellationToken: cancellationToken)); @@ -49,7 +49,7 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); var fallbackProcessor = CreateProcessor(fallbackStore, fallbackTransport).Processor; - await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); Assert.Equal(0, fallbackTransport.SendCount); // Durably parked in Redis and time-gated: a drain before the due time claims nothing; only when due does the diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs index aaa801c46..22e72e3ba 100644 --- a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs @@ -156,9 +156,64 @@ public async Task PubSub_PublishToTopic_FansOutToEverySubscriptionAsync() Assert.Equal("broadcast", await receivedByB.Task); } + [Fact] + public async Task MessageBus_SendAndPublishSameType_StayIsolatedAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var ct = TestContext.Current.CancellationToken; + await using var transport = CreateTransport(connection, NewPrefix()); + + // The unified bus: Send targets the queue-role stream, Publish the topic-role stream. The same route name must + // never cross-deliver — a publish must not be consumed as queue work and vice versa. + await using var queue = new MessageQueue(transport, new QueueOptions { OwnsTransport = false }); + await using var pubSub = new PubSub(transport, new PubSubOptions { OwnsTransport = false }); + await using var bus = new MessageBus(queue, pubSub); + + var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var published = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int sendCount = 0, publishCount = 0; + + await using var consumer = await queue.StartConsumerAsync((message, _) => + { + Interlocked.Increment(ref sendCount); + sent.TrySetResult(message.Message.Data ?? ""); + return Task.CompletedTask; + }, cancellationToken: ct); + + await using var subscription = await pubSub.SubscribeAsync((message, _) => + { + Interlocked.Increment(ref publishCount); + published.TrySetResult(message.Message.Data ?? ""); + return Task.CompletedTask; + }, cancellationToken: ct); + + await bus.SendAsync(new DualItem { Data = "for-one" }, cancellationToken: ct); + await bus.PublishAsync(new DualItem { Data = "for-all" }, cancellationToken: ct); + + await Task.WhenAll(sent.Task, published.Task).WaitAsync(TimeSpan.FromSeconds(30), ct); + Assert.Equal("for-one", await sent.Task); + Assert.Equal("for-all", await published.Task); + + // Give any cross-delivery a moment to surface, then assert exactly one delivery per verb. + await Task.Delay(500, ct); + Assert.Equal(1, Volatile.Read(ref sendCount)); + Assert.Equal(1, Volatile.Read(ref publishCount)); + } + private static TransportMessage Message(string body) => new() { Body = System.Text.Encoding.UTF8.GetBytes(body) }; + [MessageRoute("streams-dual")] + private sealed class DualItem + { + public string? Data { get; set; } + } + [MessageRoute("streams-retry")] private sealed class RetryItem { diff --git a/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs b/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs index 5a41bc927..53e0a4b5c 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryHybridCacheClientTests.cs @@ -1,6 +1,5 @@ using System.Threading.Tasks; using Foundatio.Caching; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Utility; using Microsoft.Extensions.Logging; diff --git a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs index 277c12871..363d3cfb0 100644 --- a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs +++ b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs @@ -15,7 +15,7 @@ namespace Foundatio.Tests; public class DeclarativeRegistrationTests { [Fact] - public async Task AddHandlers_HostAndDispatchQueueAndBroadcastMessagesAsync() + public async Task AddHandler_SendGoesToOneHandlerAndPublishReachesSubscriptionAsync() { var cancellationToken = TestContext.Current.CancellationToken; var probe = new HandlerProbe(); @@ -25,27 +25,33 @@ public async Task AddHandlers_HostAndDispatchQueueAndBroadcastMessagesAsync() services.AddSingleton(probe); services.AddFoundatio() .Messaging.UseInMemory() - .Messaging.AddQueueHandler() // class handler, competing - .Messaging.AddQueueHandler((message, _) => { probe.Record($"task:{message.Message.Id}"); return Task.CompletedTask; }) // delegate handler - .Messaging.AddBroadcastHandler(); // class handler, fan-out + .Messaging.AddHandler() // class handler + .Messaging.AddHandler((message, _) => { probe.Record($"task:{message.Message.Id}"); return Task.CompletedTask; }); // delegate handler await using var provider = services.BuildServiceProvider(); var hosted = provider.GetServices().ToList(); - Assert.Single(hosted); // exactly one auto-registered hosted service drives every handler + Assert.Single(hosted); // one auto-registered hosted service drives every handler foreach (var service in hosted) await service.StartAsync(cancellationToken); try { - await provider.GetRequiredService().EnqueueAsync(new HandledOrder { Id = "o1" }, cancellationToken: cancellationToken); - await provider.GetRequiredService().EnqueueAsync(new HandledTask { Id = "t1" }, cancellationToken: cancellationToken); - await provider.GetRequiredService().PublishAsync(new HandledEvent { Id = "e1" }, cancellationToken: cancellationToken); + var bus = provider.GetRequiredService(); - Assert.True(await probe.WaitForAsync(3, TimeSpan.FromSeconds(10)), $"handled: {string.Join(",", probe.Events)}"); - Assert.Contains("order:o1", probe.Events); + // The caller's verb decides delivery; the same registration serves both. + await bus.SendAsync(new HandledOrder { Id = "sent" }, cancellationToken: cancellationToken); + await bus.PublishAsync(new HandledOrder { Id = "published" }, cancellationToken: cancellationToken); + await bus.SendAsync(new HandledTask { Id = "t1" }, cancellationToken: cancellationToken); + + Assert.True(await probe.WaitForAsync(3, TimeSpan.FromSeconds(10)), $"handled: {String.Join(",", probe.Events)}"); + Assert.Contains("order:sent", probe.Events); + Assert.Contains("order:published", probe.Events); Assert.Contains("task:t1", probe.Events); - Assert.Contains("event:e1", probe.Events); + + // Same type sent AND published: exactly one delivery per verb — the queue and topic namespaces are + // segregated, so a send is never fanned out and a publish is never consumed as queue work. + Assert.Equal(3, probe.Events.Count); } finally { @@ -54,6 +60,46 @@ public async Task AddHandlers_HostAndDispatchQueueAndBroadcastMessagesAsync() } } + [Fact] + public async Task AddHandler_PublishIsOncePerServiceUnlessPerInstanceAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + + // Two service providers sharing one transport simulate two scaled instances of the same service. + var sharedProbe = new HandlerProbe(); + var instanceA = BuildInstance(transport, sharedProbe); + var instanceB = BuildInstance(transport, sharedProbe); + + await using (instanceA.Provider) + await using (instanceB.Provider) + { + await StartAsync(instanceA, cancellationToken); + await StartAsync(instanceB, cancellationToken); + + try + { + var bus = instanceA.Provider.GetRequiredService(); + + // Default subscription = service identity, shared by both instances => they compete: one copy total. + await bus.PublishAsync(new HandledEvent { Id = "shared" }, cancellationToken: cancellationToken); + Assert.True(await sharedProbe.WaitForAsync(1, TimeSpan.FromSeconds(10)), $"handled: {String.Join(",", sharedProbe.Events)}"); + await Task.Delay(250, cancellationToken); + Assert.Single(sharedProbe.Events, e => e.StartsWith("event:", StringComparison.Ordinal)); + + // PerInstance handlers each take a unique subscription => every instance receives its own copy. + await bus.PublishAsync(new HandledBroadcast { Id = "all" }, cancellationToken: cancellationToken); + Assert.True(await sharedProbe.WaitForAsync(3, TimeSpan.FromSeconds(10)), $"handled: {String.Join(",", sharedProbe.Events)}"); + Assert.Equal(2, sharedProbe.Events.Count(e => e.StartsWith("broadcast:", StringComparison.Ordinal))); + } + finally + { + await StopAsync(instanceA, cancellationToken); + await StopAsync(instanceB, cancellationToken); + } + } + } + [Fact] public async Task AddCronJob_RegistersDefinitionAndSchedulesWhenPumpStartsAsync() { @@ -101,6 +147,32 @@ public async Task AddCronJob_RegistersDefinitionAndSchedulesWhenPumpStartsAsync( } } + private static (ServiceProvider Provider, List Hosted) BuildInstance(InMemoryMessageTransport transport, HandlerProbe probe) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(probe); + services.AddFoundatio() + .Messaging.UseTransport(transport) + .Messaging.AddHandler() + .Messaging.AddHandler(o => o.PerInstance = true); + + var provider = services.BuildServiceProvider(); + return (provider, provider.GetServices().ToList()); + } + + private static async Task StartAsync((ServiceProvider Provider, List Hosted) instance, CancellationToken cancellationToken) + { + foreach (var service in instance.Hosted) + await service.StartAsync(cancellationToken); + } + + private static async Task StopAsync((ServiceProvider Provider, List Hosted) instance, CancellationToken cancellationToken) + { + foreach (var service in instance.Hosted) + await service.StopAsync(cancellationToken); + } + private sealed class HandlerProbe { private readonly ConcurrentBag _events = new(); @@ -129,6 +201,9 @@ public class HandledTask { public string Id { get; set; } = ""; } [MessageRoute("declarative-events")] public class HandledEvent { public string Id { get; set; } = ""; } + [MessageRoute("declarative-broadcasts")] + public class HandledBroadcast { public string Id { get; set; } = ""; } + private sealed class OrderHandler(HandlerProbe probe) : IMessageHandler { public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) @@ -147,6 +222,15 @@ public Task HandleAsync(IReceivedMessage message, CancellationToke } } + private sealed class BroadcastHandler(HandlerProbe probe) : IMessageHandler + { + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + probe.Record($"broadcast:{message.Message.Id}"); + return Task.CompletedTask; + } + } + private sealed class CronProbeJob : IJob { public Task RunAsync(JobExecutionContext context) => Task.FromResult(JobResult.Success); diff --git a/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs b/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs index 645d9c847..24610e3b6 100644 --- a/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs +++ b/tests/Foundatio.Tests/Locks/InMemoryLockTests.cs @@ -2,7 +2,6 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Lock; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Xunit; diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs index 7f2010848..05d383f78 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageBusTests.cs @@ -1,7 +1,6 @@ using System; using System.Threading.Tasks; using Foundatio.AsyncEx; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Tests.Extensions; using Xunit; diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 4203b85aa..0cdfc2f6a 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -131,8 +131,8 @@ public async Task SubscribeAsync_SameSubscriptionOnTwoTopics_IsolatesPerTopicAsy // Publish one message to each topic. Each subscriber must receive only its own topic's message — proving both // subscribers are live (not an always-broken one passing a negative-only assertion) and that they are isolated. - await pubSub.PublishAsync(new PreviewEvent { Data = "to-orders" }, new PubSubMessageOptions { Topic = "orders" }, cancellationToken); - await pubSub.PublishAsync(new PreviewEvent { Data = "to-payments" }, new PubSubMessageOptions { Topic = "payments" }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "to-orders" }, new MessagePublishOptions { Topic = "orders" }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "to-payments" }, new MessagePublishOptions { Topic = "payments" }, cancellationToken); await ordersSignal.WaitAsync(TimeSpan.FromSeconds(2)); await paymentsSignal.WaitAsync(TimeSpan.FromSeconds(2)); @@ -186,7 +186,7 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() return Task.CompletedTask; }, new PubSubSubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); - await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new PubSubMessageOptions + await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new MessagePublishOptions { CorrelationId = "corr-456", Priority = MessagePriority.High, @@ -225,7 +225,7 @@ public async Task PublishAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() return Task.CompletedTask; }, new PubSubSubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); - await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new PubSubMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); + await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new MessagePublishOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); await Assert.ThrowsAsync(async () => await received.WaitAsync(TimeSpan.FromMilliseconds(50))); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 7b0866699..560c56e4b 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -22,7 +22,7 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() await using var transport = new InMemoryMessageTransport(); await using var queue = new MessageQueue(transport); - string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new QueueMessageOptions + string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new MessageSendOptions { CorrelationId = "corr-123", Priority = MessagePriority.High, @@ -59,7 +59,7 @@ public async Task EnqueueBatchAsync_UsesDestinationOverrideAsync() await queue.EnqueueBatchAsync([ new PreviewWorkItem { Data = "one" }, new PreviewWorkItem { Data = "two" } - ], new QueueMessageOptions { Destination = "custom-work" }, cancellationToken); + ], new MessageSendOptions { Destination = "custom-work" }, cancellationToken); var first = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); var second = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); @@ -232,7 +232,7 @@ public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); var immediate = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); @@ -252,7 +252,7 @@ public async Task EnqueueAsync_WithDelayAndNoRuntimeStore_ThrowsAsync() await using var queue = new MessageQueue(new InMemoryMessageTransport()); await Assert.ThrowsAsync(async () => - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); } [Fact] @@ -266,7 +266,7 @@ public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync( await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); var nativeProcessor = CreateDispatchProcessor(nativeStore, nativeTransport); - await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new QueueMessageOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); Assert.Equal(1, nativeTransport.SendCount); Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); @@ -278,7 +278,7 @@ public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync( await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); var fallbackProcessor = CreateDispatchProcessor(fallbackStore, fallbackTransport); - await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new QueueMessageOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); Assert.Equal(0, fallbackTransport.SendCount); Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddHours(2), cancellationToken: cancellationToken)); @@ -389,7 +389,7 @@ public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync await using var transport = new InMemoryMessageTransport(); await using var queue = new MessageQueue(transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new QueueMessageOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); + await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new MessageSendOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(received); From adf1e3ef1039509ef75f93d117129818e1d4c528 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 17:19:02 -0500 Subject: [PATCH 39/57] IMessageBus is the one messaging abstraction: IQueue and IPubSub are gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MessageBus is now a first-class client over a single MessageClientCore, not a facade: MessageQueue, PubSub, IQueue, IPubSub, and their option records are deleted. The surface is one interface — SendAsync/PublishAsync (+batches), ReceiveAsync (pull), and SubscribeAsync, where a subscription is one logical attachment listening on the message type's two delivery channels (its send destination and this subscriber's identity on its topic). One options type, MessageSubscriptionOptions, serves declarative AddHandler and programmatic SubscribeAsync alike. Semantics hardened per adversarial review of the previous commit: - Multiple handlers can attach to one message type (previously crashed at startup on a consumer-key collision): default consumer keys are unique per subscription, so handlers compete round-robin for sent messages while an explicit Key still opts into one shared group. - Each handler class is its own subscriber group ("{service}.{handler}" via SubscriptionQualifier), so every handler registered for an event receives its own copy of each published message — instead of handler classes nondeterministically competing for one service copy. - Redis Streams: completing/dead-lettering no longer XDELs entries on topic streams (one group settling first must not delete a publish for slower groups); ExistsAsync/DeleteAsync are role-aware (topology validation works, topic cleanup no longer leaks or hits the wrong stream, subscription delete destroys only its group); GetStatsAsync no longer creates phantom streams/groups when probing. - PerInstance subscription names are derived at subscription start, per provider, fixing the shared-"unique"-name flaw for multiple providers. - Legacy gets its own MessageBusException (Foundatio.Messaging.Legacy), so a pure-legacy consumer can catch the legacy bus's exceptions without importing the new namespace; the remaining seven dual-namespace imports are flipped to pure-.Legacy. Tests: multiple-handlers semantics (each gets published events, a send reaches exactly one), send/publish same-type isolation via one subscription, idempotent same-key registration (behavioral, not reference equality), and a live-Redis regression test that a publish completed by one group is still delivered to a slower group. Full solution builds (net8 + net10, warnings-as-errors); in-memory 2006 green; live Redis 29 green. Co-Authored-By: Claude Opus 4.8 --- .../Messaging/RedisStreamsMessageTransport.cs | 66 ++- .../Jobs/WithLockingJob.cs | 1 - .../Messaging/MessageBusTestBase.cs | 10 +- .../Queue/QueueTestBase.cs | 1 - .../Logging/TestLoggerBase.cs | 2 +- .../Logging/TestWithLoggingBase.cs | 2 +- src/Foundatio.Xunit/Logging/TestLoggerBase.cs | 2 +- .../Logging/TestWithLoggingBase.cs | 2 +- .../Caching/HybridAwareCacheClient.cs | 1 - src/Foundatio/FoundatioServicesExtensions.cs | 120 ++--- src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs | 1 - src/Foundatio/Messaging/IMessageHandler.cs | 39 +- src/Foundatio/Messaging/IReceivedMessage.cs | 97 ++++ .../Messaging/LegacyMessageBusException.cs | 17 + src/Foundatio/Messaging/MessageBus.cs | 428 ++++++++++++++++-- src/Foundatio/Messaging/MessageClientCore.cs | 13 +- .../Messaging/MessageHandlerHostedService.cs | 2 +- src/Foundatio/Messaging/MessageQueue.cs | 294 ------------ .../Messaging/MessageQueueException.cs | 17 - src/Foundatio/Messaging/PubSub.cs | 221 --------- .../RedisJobStoreIntegrationTests.cs | 12 +- .../RedisStreamsTransportIntegrationTests.cs | 88 ++-- .../DeclarativeRegistrationTests.cs | 51 +++ .../Foundatio.Tests/Jobs/WorkItemJobTests.cs | 1 - .../Foundatio.Tests/Messaging/MessageTests.cs | 1 - .../Foundatio.Tests/Messaging/PubSubTests.cs | 71 +-- .../Queue/MessageQueueTests.cs | 220 ++++----- .../Utility/ResiliencePolicyTests.cs | 1 - 28 files changed, 898 insertions(+), 883 deletions(-) create mode 100644 src/Foundatio/Messaging/IReceivedMessage.cs create mode 100644 src/Foundatio/Messaging/LegacyMessageBusException.cs delete mode 100644 src/Foundatio/Messaging/MessageQueue.cs delete mode 100644 src/Foundatio/Messaging/MessageQueueException.cs delete mode 100644 src/Foundatio/Messaging/PubSub.cs diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 3e17f74b5..1ac2c4153 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -170,7 +170,11 @@ public async Task CompleteAsync(TransportEntry entry, CancellationToken ct = def var r = await ValidateReceiptAsync(entry).ConfigureAwait(false); long acked = await _db.StreamAcknowledgeAsync(r.StreamKey, r.Group, r.EntryId).ConfigureAwait(false); - await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); + // Only a queue stream (single consumer group) may delete on complete. A topic stream is shared by every + // subscription group, so one group completing must not delete the entry before the others read it; topic + // entries are retained (bound by MaxStreamLength when configured). + if (!IsTopicStream(r.StreamKey)) + await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); await ClearTrackingAsync(r).ConfigureAwait(false); if (acked == 0) @@ -212,7 +216,9 @@ await _db.StreamAddAsync(DeadKey(r.StreamKey), BuildFields(entry.Id, entry.Body, maxLength: _options.MaxStreamLength, useApproximateMaxLength: true).ConfigureAwait(false); await _db.StreamAcknowledgeAsync(r.StreamKey, r.Group, r.EntryId).ConfigureAwait(false); - await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); + // Same rule as CompleteAsync: other subscription groups on a topic stream may not have read this entry yet. + if (!IsTopicStream(r.StreamKey)) + await _db.StreamDeleteAsync(r.StreamKey, [r.EntryId]).ConfigureAwait(false); await ClearTrackingAsync(r).ConfigureAwait(false); } @@ -250,7 +256,10 @@ public async Task EnsureAsync(IReadOnlyList declarations switch (declaration.Role) { case DestinationRole.Topic: - // Topics are read through subscription groups; nothing to create until a subscription appears. + // Topics are read through subscription groups; nothing to create until a subscription appears. The + // name must NOT be registered in _sources: a queue can share the route name, and receive-side + // resolution of the bare name must keep meaning the queue stream. Exists/delete are role-aware by + // probing both namespaces instead. break; case DestinationRole.Subscription: case DestinationRole.Binding: @@ -273,23 +282,63 @@ public async Task DeleteAsync(string name, CancellationToken ct) ThrowIfDisposed(); ArgumentException.ThrowIfNullOrEmpty(name); - var resolved = Resolve(name); - await _db.KeyDeleteAsync([resolved.StreamKey, DeadKey(resolved.StreamKey), LockKey(resolved), MetaKey(resolved)]).ConfigureAwait(false); + // A subscription address deletes only that group's state (never the shared topic stream); a bare name deletes + // the name in both role namespaces, mirroring the in-memory transport. + if (SubscriptionAddress.TryParse(name, out string topic, out string subscription)) + { + var sub = new ResolvedSource(TopicStreamKey(topic), subscription, "$"); + await _db.StreamDeleteConsumerGroupAsync(sub.StreamKey, sub.Group).ConfigureAwait(false); + await _db.KeyDeleteAsync([LockKey(sub), MetaKey(sub)]).ConfigureAwait(false); + _sources.TryRemove(name, out _); + _ensuredGroups.TryRemove(GroupKey(sub), out _); + return; + } + + foreach (var resolved in (ResolvedSource[]) + [ + new ResolvedSource(QueueStreamKey(name), _options.DefaultConsumerGroup, "0"), + new ResolvedSource(TopicStreamKey(name), _options.DefaultConsumerGroup, "$") + ]) + { + // Drop each consumer group's lease/meta state before the stream itself (topic streams can carry several). + if (await _db.KeyExistsAsync(resolved.StreamKey).ConfigureAwait(false)) + { + foreach (var group in await _db.StreamGroupInfoAsync(resolved.StreamKey).ConfigureAwait(false)) + { + var groupSource = resolved with { Group = group.Name }; + await _db.KeyDeleteAsync([LockKey(groupSource), MetaKey(groupSource)]).ConfigureAwait(false); + _ensuredGroups.TryRemove(GroupKey(groupSource), out _); + } + } + + await _db.KeyDeleteAsync([resolved.StreamKey, DeadKey(resolved.StreamKey)]).ConfigureAwait(false); + _ensuredGroups.TryRemove(GroupKey(resolved), out _); + } + _sources.TryRemove(name, out _); - _ensuredGroups.TryRemove(GroupKey(resolved), out _); } - public Task ExistsAsync(string name, CancellationToken ct) + public async Task ExistsAsync(string name, CancellationToken ct) { ThrowIfDisposed(); ArgumentException.ThrowIfNullOrEmpty(name); - return _db.KeyExistsAsync(Resolve(name).StreamKey); + + if (SubscriptionAddress.TryParse(name, out string topic, out _)) + return await _db.KeyExistsAsync(TopicStreamKey(topic)).ConfigureAwait(false); + + return await _db.KeyExistsAsync(QueueStreamKey(name)).ConfigureAwait(false) + || await _db.KeyExistsAsync(TopicStreamKey(name)).ConfigureAwait(false); } public async Task GetStatsAsync(string destination, CancellationToken ct) { ThrowIfDisposed(); var resolved = Resolve(destination); + + // Probing stats must not create phantom streams/groups; a destination that doesn't exist yet is simply empty. + if (!await _db.KeyExistsAsync(resolved.StreamKey).ConfigureAwait(false)) + return new MessageDestinationStats(); + await EnsureGroupAsync(resolved).ConfigureAwait(false); long length = await _db.StreamLengthAsync(resolved.StreamKey).ConfigureAwait(false); @@ -447,6 +496,7 @@ private static string ParseToken(RedisValue meta) // consumed as queue work and vice versa). Subscriptions are consumer groups on the topic stream. private RedisKey QueueStreamKey(string name) => $"{_prefix}q:{name}"; private RedisKey TopicStreamKey(string name) => $"{_prefix}t:{name}"; + private bool IsTopicStream(string streamKey) => streamKey.StartsWith($"{_prefix}t:", StringComparison.Ordinal); private static RedisKey DeadKey(RedisKey streamKey) => streamKey.ToString() + ":dead"; private static RedisKey LockKey(ResolvedSource r) => $"{r.StreamKey}:lock:{r.Group}"; private static RedisKey MetaKey(ResolvedSource r) => $"{r.StreamKey}:meta:{r.Group}"; diff --git a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs index 6e9435108..3c31e39ab 100644 --- a/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs +++ b/src/Foundatio.TestHarness/Jobs/WithLockingJob.cs @@ -4,7 +4,6 @@ using Foundatio.Caching; using Foundatio.Jobs.Legacy; using Foundatio.Lock; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Microsoft.Extensions.Logging; using Xunit; diff --git a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs index 54d848e21..dc492a1f5 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageBusTestBase.cs @@ -861,7 +861,7 @@ public virtual async Task PublishAsync_AfterDispose_ThrowsMessageBusExceptionAsy await messageBus.DisposeAsync(); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.PublishAsync(new SimpleMessageA { Data = "Hello" })); } finally @@ -872,7 +872,7 @@ public virtual async Task PublishAsync_AfterDispose_ThrowsMessageBusExceptionAsy /// /// Verifies that cancellation is surfaced as OperationCanceledException, not swallowed - /// or wrapped in Foundatio.Messaging.MessageBusException. This ensures callers can distinguish between + /// or wrapped in MessageBusException. This ensures callers can distinguish between /// cancellation and actual publish failures. /// public virtual async Task PublishAsync_WithCancellation_ThrowsOperationCanceledExceptionAsync() @@ -945,7 +945,7 @@ public virtual async Task PublishAsync_WithSerializationFailure_ThrowsSerializer await messageBus.SubscribeAsync(_ => { }); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.PublishAsync(new SimpleMessageA { Data = "test" }, cancellationToken: TestCancellationToken)); } finally @@ -966,7 +966,7 @@ public virtual async Task SubscribeAsync_AfterDispose_ThrowsMessageBusExceptionA await messageBus.DisposeAsync(); // Act & Assert - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => await messageBus.SubscribeAsync(_ => { })); } finally @@ -1022,7 +1022,7 @@ await messageBus.SubscribeAsync(msg => /// /// Verifies that cancellation is surfaced as OperationCanceledException, not swallowed - /// or wrapped in Foundatio.Messaging.MessageBusException. This ensures callers can distinguish between + /// or wrapped in MessageBusException. This ensures callers can distinguish between /// cancellation and actual subscribe failures. /// public virtual async Task SubscribeAsync_WithCancellation_ThrowsOperationCanceledExceptionAsync() diff --git a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs index 977f0db21..01d2f63ca 100644 --- a/src/Foundatio.TestHarness/Queue/QueueTestBase.cs +++ b/src/Foundatio.TestHarness/Queue/QueueTestBase.cs @@ -9,7 +9,6 @@ using Foundatio.Caching; using Foundatio.Jobs.Legacy; using Foundatio.Lock; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Serializer; diff --git a/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs b/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs index 7e0fc9fe2..b675ce80b 100644 --- a/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs +++ b/src/Foundatio.Xunit.v3/Logging/TestLoggerBase.cs @@ -27,7 +27,7 @@ protected TestLoggerBase(ITestOutputHelper output, TestLoggerFixture fixture) /// /// Gets a cancellation token that is cancelled when the current test completes or /// when the test run is aborted/timed out. Pass this token to - /// + /// /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs b/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs index e4ffd1b7d..e96c6a8f4 100644 --- a/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs +++ b/src/Foundatio.Xunit.v3/Logging/TestWithLoggingBase.cs @@ -22,7 +22,7 @@ protected TestWithLoggingBase(ITestOutputHelper output) /// /// Gets a cancellation token that is cancelled when the current test completes or /// when the test run is aborted/timed out. Pass this token to - /// + /// /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit/Logging/TestLoggerBase.cs b/src/Foundatio.Xunit/Logging/TestLoggerBase.cs index 6fca07cb6..9adf3d4b4 100644 --- a/src/Foundatio.Xunit/Logging/TestLoggerBase.cs +++ b/src/Foundatio.Xunit/Logging/TestLoggerBase.cs @@ -26,7 +26,7 @@ protected TestLoggerBase(ITestOutputHelper output, TestLoggerFixture fixture) /// /// Gets a cancellation token that is cancelled when the current test completes. - /// Pass this token to + /// Pass this token to /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs b/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs index 16fe79d70..c9afc52c9 100644 --- a/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs +++ b/src/Foundatio.Xunit/Logging/TestWithLoggingBase.cs @@ -21,7 +21,7 @@ protected TestWithLoggingBase(ITestOutputHelper output) /// /// Gets a cancellation token that is cancelled when the current test completes. - /// Pass this token to + /// Pass this token to /// and other async operations to ensure automatic cleanup between tests. /// /// diff --git a/src/Foundatio/Caching/HybridAwareCacheClient.cs b/src/Foundatio/Caching/HybridAwareCacheClient.cs index 4cda2aa9d..4b41c9a1f 100644 --- a/src/Foundatio/Caching/HybridAwareCacheClient.cs +++ b/src/Foundatio/Caching/HybridAwareCacheClient.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 2d60ac4b7..2fe3d80c1 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -347,64 +347,53 @@ public FoundatioBuilder UseTransport(Func f /// decision — the caller's verb on decides delivery: a SendAsync is processed /// by exactly one handler instance across the fleet (competing consumers), and a PublishAsync is received /// once per subscribing service (a scaled service's instances compete), or by every instance when - /// is set. The handler is resolved from DI in its own scope per - /// message (so it can inject scoped dependencies); throwing triggers the retry/dead-letter policy. A single + /// is set. The handler is resolved from DI in its own scope + /// per message (so it can inject scoped dependencies); throwing triggers the retry/dead-letter policy. A single /// hosted service starts and stops all registered handlers. /// - public FoundatioBuilder AddHandler(Action? configure = null) + public FoundatioBuilder AddHandler(Action? configure = null) where TMessage : class where THandler : class, IMessageHandler { _services.TryAddScoped(); - return AddHandlerListeners(typeof(THandler).Name, static (sp, message, ct) => DispatchAsync(sp, message, ct), configure); + // Each handler class is its own subscriber group ("{service}.{handler}"), so every handler registered for + // an event type receives its own copy of each published message. + return AddHandlerRegistration(typeof(THandler).Name, static (sp, message, ct) => DispatchAsync(sp, message, ct), + options => + { + configure?.Invoke(options); + options.SubscriptionQualifier ??= typeof(THandler).Name; + }); } /// /// Registers a delegate handler for messages of type ; see /// for the delivery semantics. /// - public FoundatioBuilder AddHandler(Func, CancellationToken, Task> handler, Action? configure = null) + public FoundatioBuilder AddHandler(Func, CancellationToken, Task> handler, Action? configure = null) where TMessage : class { ArgumentNullException.ThrowIfNull(handler); - return AddHandlerListeners(null, (_, message, ct) => handler(message, ct), configure); + return AddHandlerRegistration(null, (_, message, ct) => handler(message, ct), configure); } - private FoundatioBuilder AddHandlerListeners(string? handlerName, Func, CancellationToken, Task> dispatch, Action? configure) + private FoundatioBuilder AddHandlerRegistration(string? handlerName, Func, CancellationToken, Task> dispatch, Action? configure) where TMessage : class { - var options = new MessageHandlerOptions(); - configure?.Invoke(options); - - if (options.PerInstance && !String.IsNullOrEmpty(options.Subscription)) - throw new ArgumentException("PerInstance and Subscription are mutually exclusive: PerInstance derives a unique per-instance subscription.", nameof(configure)); - string suffix = handlerName is null ? String.Empty : $" -> {handlerName}"; - - // Send target: competing consumers on the message type's queue destination — one handler instance across - // the fleet processes each SendAsync. - AddHandlerRegistration($"send:{typeof(TMessage).Name}{suffix}", async (sp, ct) => - await sp.GetRequiredService().StartConsumerAsync((message, c) => dispatch(sp, message, c), new QueueConsumerOptions - { - AckMode = options.AckMode, - MaxConcurrency = options.MaxConcurrency, - MaxAttempts = options.MaxAttempts, - RedeliveryBackoff = options.RedeliveryBackoff - }, ct).ConfigureAwait(false)); - - // Publish target: this service's subscription on the message type's topic. The default subscription - // identity is the service identity, so scaled instances share one subscription and compete — each service - // handles a published message once. PerInstance instead takes a unique per-instance subscription so every - // instance receives its own copy. - string? subscription = options.PerInstance ? UniqueSubscriptionName() : options.Subscription; - AddHandlerRegistration($"publish:{typeof(TMessage).Name}{suffix}", async (sp, ct) => - await sp.GetRequiredService().SubscribeAsync((message, c) => dispatch(sp, message, c), new PubSubSubscriptionOptions + _services.AddSingleton(new MessageHandlerRegistration + { + Description = $"handler:{typeof(TMessage).Name}{suffix}", + StartAsync = async (sp, ct) => { - Subscription = subscription, - AckMode = options.AckMode, - MaxConcurrency = options.MaxConcurrency, - MaxAttempts = options.MaxAttempts, - RedeliveryBackoff = options.RedeliveryBackoff - }, ct).ConfigureAwait(false)); + var options = new MessageSubscriptionOptions(); + configure?.Invoke(options); + return await sp.GetRequiredService() + .SubscribeAsync((message, c) => dispatch(sp, message, c), options, ct).ConfigureAwait(false); + } + }); + + if (!_services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(MessageHandlerHostedService))) + _services.AddSingleton(); return _builder; } @@ -417,15 +406,6 @@ private static async Task DispatchAsync(IServiceProvider ser await handler.HandleAsync(message, cancellationToken).ConfigureAwait(false); } - private static string UniqueSubscriptionName() => $"{Environment.MachineName}-{Guid.NewGuid():N}"; - - private void AddHandlerRegistration(string description, Func> start) - { - _services.AddSingleton(new MessageHandlerRegistration { Description = description, StartAsync = start }); - if (!_services.Any(s => s.ServiceType == typeof(IHostedService) && s.ImplementationType == typeof(MessageHandlerHostedService))) - _services.AddSingleton(); - } - private void RegisterMessagingRuntime(Func factory) { _services.ReplaceSingleton(factory); @@ -468,44 +448,18 @@ private void RegisterMessageClients() { RegisterRoutingServices(); _services.ReplaceSingleton(sp => new MessageTypeRegistry(sp.GetServices())); - _services.ReplaceSingleton(sp => new MessageQueue(sp.GetRequiredService(), CreateQueueOptions(sp))); - _services.ReplaceSingleton(sp => new PubSub(sp.GetRequiredService(), CreatePubSubOptions(sp))); - // The primary client: one bus, two verbs. The underlying queue/pub-sub clients stay resolvable for - // advanced scenarios (pull receive, programmatic consumers). - _services.ReplaceSingleton(sp => new MessageBus(sp.GetRequiredService(), sp.GetRequiredService())); - } - - private static QueueOptions CreateQueueOptions(IServiceProvider serviceProvider) - { - return new QueueOptions - { - Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, - Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, - MessageTypes = serviceProvider.GetService() ?? new MessageTypeRegistry(), - RuntimeStore = serviceProvider.GetService(), - RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), - // The transport is a shared DI singleton owned by the container; the queue must not dispose it (the - // pub/sub client uses the same instance). - OwnsTransport = false, - TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, - LoggerFactory = serviceProvider.GetService() - }; - } - - private static PubSubOptions CreatePubSubOptions(IServiceProvider serviceProvider) - { - return new PubSubOptions + _services.ReplaceSingleton(sp => new MessageBus(sp.GetRequiredService(), new MessageBusOptions { - Serializer = serviceProvider.GetService() ?? DefaultSerializer.Instance, - Router = serviceProvider.GetService() ?? DefaultMessageRouter.Instance, - MessageTypes = serviceProvider.GetService() ?? new MessageTypeRegistry(), - RuntimeStore = serviceProvider.GetService(), - RetryPolicy = serviceProvider.GetService() ?? new RetryPolicy(), - // Shared DI singleton transport; disposed once by the container, not by this client. + Serializer = sp.GetService() ?? DefaultSerializer.Instance, + Router = sp.GetService() ?? DefaultMessageRouter.Instance, + MessageTypes = sp.GetService() ?? new MessageTypeRegistry(), + RuntimeStore = sp.GetService(), + RetryPolicy = sp.GetService() ?? new RetryPolicy(), + // The transport is a shared DI singleton owned by the container; the bus must not dispose it. OwnsTransport = false, - TimeProvider = serviceProvider.GetService() ?? TimeProvider.System, - LoggerFactory = serviceProvider.GetService() - }; + TimeProvider = sp.GetService() ?? TimeProvider.System, + LoggerFactory = sp.GetService() + })); } } diff --git a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs index 129e71d17..12632fdc3 100644 --- a/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs +++ b/src/Foundatio/Jobs/WorkItemJob/WorkItemJob.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.Threading; using System.Threading.Tasks; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Serializer; diff --git a/src/Foundatio/Messaging/IMessageHandler.cs b/src/Foundatio/Messaging/IMessageHandler.cs index b68ae68fb..3117c0699 100644 --- a/src/Foundatio/Messaging/IMessageHandler.cs +++ b/src/Foundatio/Messaging/IMessageHandler.cs @@ -1,4 +1,3 @@ -using System; using System.Threading; using System.Threading.Tasks; @@ -8,44 +7,12 @@ namespace Foundatio.Messaging; /// Handles messages of type . Register with /// AddFoundatio().Messaging.AddHandler<T, THandler>() — registration carries no topology decision; the /// caller's verb on decides delivery (SendAsync = one handler instance across the -/// fleet, PublishAsync = once per subscribing service). A hosted service starts and dispatches to it. Handlers -/// are resolved from DI in their own scope per message, so they can inject scoped dependencies. Throwing from +/// fleet, PublishAsync = once per subscribing service, or every instance with +/// ). A hosted service starts and dispatches to it. Handlers are +/// resolved from DI in their own scope per message, so they can inject scoped dependencies. Throwing from /// triggers the core's retry/dead-letter policy. /// public interface IMessageHandler where T : class { Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken); } - -/// -/// Options for a declaratively-registered message handler (AddFoundatio().Messaging.AddHandler<T, THandler>(o => ...)). -/// -public sealed class MessageHandlerOptions -{ - /// - /// When true, published messages are received by EVERY running instance (each instance takes a unique - /// subscription), instead of once per service. For per-instance local state — cache invalidation, config reload. - /// Mutually exclusive with . Does not affect sent messages, which always go to exactly - /// one instance. - /// - public bool PerInstance { get; set; } - - /// - /// The subscriber-group identity used for published messages. Defaults to the service identity, so all instances - /// of a service share one subscription and compete (each published message is handled once per service). Set an - /// explicit name to form an independent named subscriber group. - /// - public string? Subscription { get; set; } - - /// Maximum messages this handler processes concurrently per instance. Default 1. - public int MaxConcurrency { get; set; } = 1; - - /// Maximum delivery attempts before dead-lettering. Null uses the default . - public int? MaxAttempts { get; set; } - - /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's timing. - public Func? RedeliveryBackoff { get; set; } - - /// Whether messages auto-complete when the handler returns (default) or are settled manually. - public AckMode AckMode { get; set; } = AckMode.Auto; -} diff --git a/src/Foundatio/Messaging/IReceivedMessage.cs b/src/Foundatio/Messaging/IReceivedMessage.cs new file mode 100644 index 000000000..82d453c62 --- /dev/null +++ b/src/Foundatio/Messaging/IReceivedMessage.cs @@ -0,0 +1,97 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Foundatio.Messaging; + +public enum AckMode +{ + Auto, + Manual +} + +/// +/// Core-owned retry and dead-letter policy. Foundatio always owns redelivery and dead-lettering so the behavior is +/// identical across transports; transports stay simple and only provide the underlying primitives (redelivery and an +/// optional dead-letter sink). Configure a default on ; a subscription can override +/// /backoff per subscription. +/// +public sealed record RetryPolicy +{ + /// Maximum delivery attempts for a failing handler before the message is dead-lettered. Default 5. + public int MaxAttempts { get; init; } = 5; + + /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's own redelivery timing. + public Func? Backoff { get; init; } + + /// + /// Destination terminal (dead-lettered) messages are sent to when the transport has no native dead-letter sink. + /// Null drops terminal messages on such transports. Ignored when the transport supports native dead-lettering. + /// + public string? DeadLetterDestination { get; init; } + + /// Maximum attempts for a message whose type has no registered consumer before it is dead-lettered as "no-handler". Default 50. + public int UnmatchedMaxAttempts { get; init; } = 50; + + /// Delay before redelivering an unmatched-type message. Null defers to the transport's own redelivery timing. + public Func? UnmatchedBackoff { get; init; } +} + +/// +/// Thrown by the consumer loop when a message arrives on a shared destination whose type has no registered consumer +/// on this node (for example a newer message type mid rolling-deploy, or a misconfiguration). It is surfaced loudly +/// per message and isolated to that message — the receive loop and the other type handlers keep running. +/// +public sealed class UnhandledMessageTypeException : Exception +{ + public UnhandledMessageTypeException(string? messageType, string source) + : base($"No consumer is registered for message type \"{messageType ?? "(unknown)"}\" received on source \"{source}\".") + { + MessageType = messageType; + SourceName = source; + } + + public string? MessageType { get; } + public string SourceName { get; } +} + +public sealed record RejectOptions +{ + /// + /// When false (default) the message is returned for redelivery (a retry). When true the message is terminal: it + /// is moved to the transport's dead-letter sink where one exists, otherwise dropped. Terminal messages are never + /// redelivered. + /// + public bool Terminal { get; init; } + + /// Reason carried to the dead-letter sink (where the transport supports one) for a terminal reject. + public string? Reason { get; init; } + + /// + /// An explicit delay before the message is redelivered. Honored only for a non-terminal reject, served natively + /// when the transport supports redelivery delay within its advertised maximum, otherwise through the runtime store. + /// When null the transport's own redelivery timing applies. + /// + public TimeSpan? RedeliveryDelay { get; init; } +} + +public interface IReceivedMessage +{ + string Id { get; } + ReadOnlyMemory Body { get; } + MessageHeaders Headers { get; } + string? CorrelationId { get; } + string? MessageType { get; } + MessagePriority Priority { get; } + int Attempts { get; } + bool IsHandled { get; } + CancellationToken CancellationToken { get; } + Task CompleteAsync(CancellationToken cancellationToken = default); + Task RejectAsync(RejectOptions? options = null, CancellationToken cancellationToken = default); + Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); +} + +public interface IReceivedMessage : IReceivedMessage where T : class +{ + T Message { get; } +} diff --git a/src/Foundatio/Messaging/LegacyMessageBusException.cs b/src/Foundatio/Messaging/LegacyMessageBusException.cs new file mode 100644 index 000000000..aef314810 --- /dev/null +++ b/src/Foundatio/Messaging/LegacyMessageBusException.cs @@ -0,0 +1,17 @@ +using System; + +namespace Foundatio.Messaging.Legacy; + +/// +/// Exception thrown when a legacy message bus operation fails. +/// +public class MessageBusException : Exception +{ + public MessageBusException(string message) : base(message) + { + } + + public MessageBusException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index e9f43afd2..efcb71156 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -1,23 +1,143 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Foundatio.Jobs; +using Foundatio.Serializer; using Foundatio.Utility; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace Foundatio.Messaging; +public sealed record MessageSendOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public TimeSpan? Delay { get; init; } + public DateTimeOffset? DeliverAt { get; init; } + public TimeSpan? TimeToLive { get; init; } + public string? CorrelationId { get; init; } + public string? DeduplicationId { get; init; } + /// Overrides the routed destination for this send. + public string? Destination { get; init; } + public MessageHeaders? Headers { get; init; } +} + +public sealed record MessagePublishOptions +{ + public MessagePriority Priority { get; init; } = MessagePriority.Normal; + public TimeSpan? Delay { get; init; } + public DateTimeOffset? DeliverAt { get; init; } + public TimeSpan? TimeToLive { get; init; } + public string? CorrelationId { get; init; } + public string? DeduplicationId { get; init; } + /// Overrides the routed topic for this publish. + public string? Topic { get; init; } + public MessageHeaders? Headers { get; init; } +} + +public sealed record MessageReceiveOptions +{ + /// Overrides the source to pull from; defaults to the message type's send destination. + public string? Source { get; init; } + public Type? RouteType { get; init; } + public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); +} + /// -/// The primary messaging client. The verb carries the delivery semantic, so handlers are registered without any -/// topology decision (AddFoundatio().Messaging.AddHandler<T, THandler>()): +/// Options for attaching a handler to a message type — via AddFoundatio().Messaging.AddHandler<T, THandler>(o => ...) +/// or programmatically via . A subscription listens on the type's two +/// delivery channels: sent messages (one handler instance across the fleet processes each) and published messages +/// (delivered per the subscription identity below). +/// +public sealed class MessageSubscriptionOptions +{ + /// + /// When true, published messages are received by EVERY running instance (each takes a unique subscription), + /// instead of once per service. For per-instance local state — cache invalidation, config reload. Mutually + /// exclusive with . Does not affect sent messages, which always go to exactly one instance. + /// + public bool PerInstance { get; set; } + + /// + /// The subscriber-group identity for published messages. Defaults to the service identity (plus the + /// when set), so all instances of a service share one subscription and compete + /// (each published message is handled once per service). Set an explicit name to form an independent named + /// subscriber group. + /// + public string? Subscription { get; set; } + + /// + /// Distinguishes this subscriber group from others in the same service when no explicit + /// is set — the default group becomes "{service-identity}.{qualifier}". Set automatically to the handler type name + /// by AddHandler<T, THandler> so each handler class receives its own copy of published messages. + /// Ignored when or is set. + /// + public string? SubscriptionQualifier { get; set; } + + /// Maximum messages this subscription processes concurrently per instance. Default 1. + public int MaxConcurrency { get; set; } = 1; + + /// Maximum delivery attempts before dead-lettering. Null uses the default . + public int? MaxAttempts { get; set; } + + /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's timing. + public Func? RedeliveryBackoff { get; set; } + + /// Whether messages auto-complete when the handler returns (default) or are settled manually. + public AckMode AckMode { get; set; } = AckMode.Auto; + + /// Routes by a different type than the handler's type parameter (grouped/interface consumers). + public Type? RouteType { get; set; } + + /// Overrides the routed send destination this subscription listens on. + public string? Destination { get; set; } + + /// Overrides the routed topic this subscription listens on. + public string? Topic { get; set; } + + /// + /// Consumer identity. Subscriptions sharing a key on the same channel form one consumer group and compete; + /// defaults to a per-channel key derived from the route. + /// + public string? Key { get; set; } +} + +/// A started subscription; disposing detaches the handler from the message type's delivery channels. +public interface IMessageSubscription : IAsyncDisposable +{ + /// Consumer identity; subscriptions sharing a key on a channel form one competing group. + string Key { get; } + + /// The send-channel destination this subscription listens on. + string Destination { get; } + + /// The publish-channel topic this subscription listens on. + string Topic { get; } + + /// The publish-channel subscriber-group identity (service identity unless overridden or per-instance). + string Subscription { get; } + + /// + /// The publish-channel transport source: the topic-qualified subscription address, so the same subscription + /// identity on two topics resolves to two distinct sources. + /// + string Source { get; } +} + +/// +/// The messaging client. Handlers are registered without any topology decision and the caller's verb carries the +/// delivery semantic: /// -/// — a command / unit of work: exactly one handler instance across the fleet processes -/// it (competing consumers on the message type's queue destination). -/// — an event: every subscribing service receives one copy on its own subscription, -/// and a scaled service's instances compete for that copy (so side effects happen once per service, not once per -/// replica). A handler registered with PerInstance = true instead receives a copy on every instance. +/// — a command / unit of work: exactly one handler instance across the fleet +/// processes it (competing consumers). +/// — an event: every subscribing service receives one copy (a scaled service's +/// instances compete for it), or every instance when the subscription opts into +/// . /// -/// Retry and dead-lettering are core-owned and identical for both verbs: a handler that throws triggers redelivery and, -/// once attempts are exhausted, the dead-letter policy. +/// Retry and dead-lettering are core-owned and identical for both verbs: a handler that throws triggers redelivery +/// and, once attempts are exhausted, the dead-letter policy. /// public interface IMessageBus : IAsyncDisposable { @@ -30,51 +150,297 @@ public interface IMessageBus : IAsyncDisposable Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default); + + /// + /// Attaches a handler to the message type's delivery channels (sent and published messages). Prefer declarative + /// registration (AddFoundatio().Messaging.AddHandler<T, THandler>()) for handlers that live for the + /// app's lifetime; use this for dynamic subscriptions. + /// + Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default); + + /// Pulls one sent message of type , or null when none arrives within the wait window. + Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default); +} + +public sealed record MessageBusOptions +{ + public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; + public string ContentType { get; init; } = "application/json"; + public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; + public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); + public IJobRuntimeStore? RuntimeStore { get; init; } + public RetryPolicy RetryPolicy { get; init; } = new(); + + /// + /// Whether disposing this bus also disposes the transport. True (default) for a transport the bus solely uses; set + /// false when the transport is a shared/externally-owned instance (e.g. a DI singleton). + /// + public bool OwnsTransport { get; init; } = true; + public TimeProvider TimeProvider { get; init; } = TimeProvider.System; + public ILoggerFactory? LoggerFactory { get; init; } } /// -/// Facade unifying the queue (send) and pub/sub (publish) clients behind the two delivery verbs. The underlying -/// clients remain available for advanced scenarios (pull receive, programmatic consumers/subscriptions). -/// Disposing the bus disposes the underlying clients only when ownsClients is true — default false, since DI -/// singleton clients are disposed exactly once by the container. +/// The one messaging client over the transport. Routing, serialization, settlement, scheduling, and the consumer loop +/// live in ; this type maps the two delivery verbs and subscriptions onto that core. /// public sealed class MessageBus : IMessageBus { - private readonly IQueue _queue; - private readonly IPubSub _pubSub; - private readonly bool _ownsClients; + private readonly MessageClientCore _core; - public MessageBus(IQueue queue, IPubSub pubSub, bool ownsClients = false) + public MessageBus(IMessageTransport transport, MessageBusOptions? options = null) { - _queue = queue ?? throw new ArgumentNullException(nameof(queue)); - _pubSub = pubSub ?? throw new ArgumentNullException(nameof(pubSub)); - _ownsClients = ownsClients; + ArgumentNullException.ThrowIfNull(transport); + options ??= new MessageBusOptions(); + var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } public Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class - => _queue.EnqueueAsync(message, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(message); + options ??= new MessageSendOptions(); + return _core.SendAsync(ScheduledDispatchKind.QueueMessage, typeof(T), message, ToEnvelope(options), GetDestination(typeof(T), options.Destination), ensureDestination: null, cancellationToken); + } public Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class - => _queue.EnqueueBatchAsync(messages, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(messages); + options ??= new MessageSendOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); + } public Task SendBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) - => _queue.EnqueueBatchAsync(messages, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(messages); + options ??= new MessageSendOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages, null, ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); + } public Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class - => _pubSub.PublishAsync(message, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(message); + options ??= new MessagePublishOptions(); + return _core.SendAsync(ScheduledDispatchKind.PubSubMessage, typeof(T), message, ToEnvelope(options), GetTopic(typeof(T), options.Topic), EnsureTopicAsync, cancellationToken); + } public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class - => _pubSub.PublishBatchAsync(messages, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(messages); + options ??= new MessagePublishOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); + } public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) - => _pubSub.PublishBatchAsync(messages, options, cancellationToken); + { + ArgumentNullException.ThrowIfNull(messages); + options ??= new MessagePublishOptions(); + return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); + } - public async ValueTask DisposeAsync() + public async Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class { - if (!_ownsClients) - return; + ArgumentNullException.ThrowIfNull(handler); + var channels = BuildChannels(options, typeof(T)); + var sent = await _core.StartListenerAsync(channels.Send, handler, cancellationToken).AnyContext(); + try + { + await EnsureSubscriptionAsync(channels.Publish, cancellationToken).AnyContext(); + var published = await _core.StartListenerAsync(channels.Publish, handler, cancellationToken).AnyContext(); + return new MessageSubscription(sent, published); + } + catch + { + await sent.DisposeAsync().AnyContext(); + throw; + } + } + + public async Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + var channels = BuildChannels(options, typeof(object)); + var sent = await _core.StartListenerAsync(channels.Send, handler, cancellationToken).AnyContext(); + try + { + await EnsureSubscriptionAsync(channels.Publish, cancellationToken).AnyContext(); + var published = await _core.StartListenerAsync(channels.Publish, handler, cancellationToken).AnyContext(); + return new MessageSubscription(sent, published); + } + catch + { + await sent.DisposeAsync().AnyContext(); + throw; + } + } + + public Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class + { + options ??= new MessageReceiveOptions(); + return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(T), options.Source), options.MaxWaitTime, cancellationToken); + } + + public Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) + { + options ??= new MessageReceiveOptions(); + return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(object), options.Source), options.MaxWaitTime, cancellationToken); + } + + public ValueTask DisposeAsync() + { + return _core.DisposeAsync(); + } + + // A subscription is one logical attachment listening on the type's two delivery channels: the send (queue-role) + // destination and this subscriber's identity on the publish (topic-role) route. The publish channel is provisioned + // before listening so a publish can reach it from the first message. + private (ListenerConfig Send, ListenerConfig Publish) BuildChannels(MessageSubscriptionOptions? options, Type fallbackType) + { + options ??= new MessageSubscriptionOptions(); + if (options.PerInstance && !String.IsNullOrEmpty(options.Subscription)) + throw new ArgumentException("PerInstance and Subscription are mutually exclusive: PerInstance derives a unique per-instance subscription.", nameof(options)); + + var routeType = options.RouteType ?? fallbackType; + + // The default consumer key is unique per subscription so multiple handlers can attach to the same type: they + // compete round-robin for sent messages (a command still reaches exactly one handler instance) and each keeps + // its own subscriber group for published ones. An explicit Key opts subscriptions into one shared group. + string uniqueKey = Guid.NewGuid().ToString("N"); + + string destination = GetDestination(routeType, options.Destination); + var send = new ListenerConfig + { + Source = destination, + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{destination}:{uniqueKey}", + MessageType = routeType, + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff + }; + + string topic = GetTopic(routeType, options.Topic); + string subscription = options.PerInstance + ? $"{Environment.MachineName}-{Guid.NewGuid():N}" + : options.Subscription ?? QualifySubscription(GetSubscription(routeType, topic, null), options.SubscriptionQualifier); + var publish = new ListenerConfig + { + Topic = topic, + Subscription = subscription, + // The transport source is the topic-qualified subscription destination, not the bare subscription name, so + // the same subscription identity used on two topics resolves to two distinct sources (and isolates). + Source = SubscriptionAddress.Format(topic, subscription), + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{uniqueKey}", + MessageType = routeType, + AckMode = options.AckMode, + MaxConcurrency = options.MaxConcurrency, + MaxAttempts = options.MaxAttempts, + RedeliveryBackoff = options.RedeliveryBackoff + }; + + return (send, publish); + } + + private static string QualifySubscription(string identity, string? qualifier) + { + return String.IsNullOrEmpty(qualifier) ? identity : $"{identity}.{MessageRoutingConventions.ToKebabCase(qualifier)}"; + } + + private Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) + { + return _core.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken); + } + + private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken cancellationToken) + { + return _core.EnsureAsync([ + new DestinationDeclaration { Name = config.Topic, Role = DestinationRole.Topic }, + new DestinationDeclaration { Name = config.Source, Role = DestinationRole.Subscription, Source = config.Topic } + ], cancellationToken); + } + + private string GetDestination(Type messageType, string? destination) + { + return _core.Router.ResolveRoute(new MessageRouteContext + { + MessageType = messageType, + Role = MessageRouteRole.QueueDestination, + OperationOverride = destination + }); + } + + private string GetTopic(Type messageType, string? topic) + { + return _core.Router.ResolveRoute(new MessageRouteContext + { + MessageType = messageType, + Role = MessageRouteRole.PubSubTopic, + OperationOverride = topic + }); + } + + private string GetSubscription(Type messageType, string topic, string? subscription) + { + return _core.Router.ResolveSubscription(new MessageSubscriptionContext + { + MessageType = messageType, + Topic = topic, + OperationOverride = subscription + }); + } + + private static MessageEnvelopeOptions ToEnvelope(MessageSendOptions options) + { + return new MessageEnvelopeOptions + { + Priority = options.Priority, + Delay = options.Delay, + DeliverAt = options.DeliverAt, + TimeToLive = options.TimeToLive, + CorrelationId = options.CorrelationId, + DeduplicationId = options.DeduplicationId, + Headers = options.Headers + }; + } + + private static MessageEnvelopeOptions ToEnvelope(MessagePublishOptions options) + { + return new MessageEnvelopeOptions + { + Priority = options.Priority, + Delay = options.Delay, + DeliverAt = options.DeliverAt, + TimeToLive = options.TimeToLive, + CorrelationId = options.CorrelationId, + DeduplicationId = options.DeduplicationId, + Headers = options.Headers + }; + } + + private sealed class MessageSubscription : IMessageSubscription + { + private readonly MessageListenerHandle _sent; + private readonly MessageListenerHandle _published; + + public MessageSubscription(MessageListenerHandle sent, MessageListenerHandle published) + { + _sent = sent; + _published = published; + } + + public string Key => _sent.Key; + public string Destination => _sent.Source; + public string Topic => _published.Topic; + public string Subscription => _published.Subscription; + public string Source => _published.Source; - await _queue.DisposeAsync().AnyContext(); - await _pubSub.DisposeAsync().AnyContext(); + public async ValueTask DisposeAsync() + { + await _sent.DisposeAsync().AnyContext(); + await _published.DisposeAsync().AnyContext(); + } } } diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 6756ad993..faba42fad 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -16,7 +16,7 @@ namespace Foundatio.Messaging; /// /// Core-owned messaging instruments. Counters and histograms are transport-agnostic and shared by every -/// and instance so that send/receive/settlement volume and handler +/// instance so that send/receive/settlement volume and handler /// latency are observable regardless of which transport is plugged in. /// internal static class MessagingInstruments @@ -62,7 +62,7 @@ internal sealed record ListenerConfig } /// -/// Shared implementation behind and : serialization, header/trace +/// Shared implementation behind : serialization, header/trace /// construction, routing-agnostic send (with batch chunking and runtime-store scheduled dispatch), received-message /// creation with poison handling, auto/manual ack settlement, and the resilient consumer/subscription loop. /// @@ -975,7 +975,7 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c } if (_runtimeStore is null) - throw new MessageQueueException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); + throw new MessageBusException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); // Advance from the reconciled attempt count, not the raw transport DeliveryCount: the re-send produces a new // transport message whose native DeliveryCount resets to 1, so basing the next attempt on DeliveryCount would @@ -1085,11 +1085,10 @@ public static string ToKebabCase(string value) } /// -/// A started listener handle. A single type backs both the queue consumer and pub/sub subscription surfaces; queue -/// callers observe it as (Source/Key), pub/sub callers as -/// (Topic/Subscription/Key). +/// A started listener handle for one channel (a send destination or a topic subscription); the bus composes one per +/// channel into the it returns. /// -internal sealed class MessageListenerHandle : IMessageConsumer, IMessageSubscription +internal sealed class MessageListenerHandle : IAsyncDisposable { private readonly Func _dispose; private int _isDisposed; diff --git a/src/Foundatio/Messaging/MessageHandlerHostedService.cs b/src/Foundatio/Messaging/MessageHandlerHostedService.cs index 5d17428bb..b6cb795f1 100644 --- a/src/Foundatio/Messaging/MessageHandlerHostedService.cs +++ b/src/Foundatio/Messaging/MessageHandlerHostedService.cs @@ -24,7 +24,7 @@ internal sealed class MessageHandlerRegistration /// Hosts every declaratively-registered message handler for the app's lifetime: on start it launches each handler's /// consumer/subscription; on stop it disposes them. Auto-registered when the first handler is added, so users register /// handlers in configuration and never hand-write a hosted service. Programmatic -/// / remain available for dynamic use. +/// remain available for dynamic use. /// internal sealed class MessageHandlerHostedService : IHostedService { diff --git a/src/Foundatio/Messaging/MessageQueue.cs b/src/Foundatio/Messaging/MessageQueue.cs deleted file mode 100644 index a00933e21..000000000 --- a/src/Foundatio/Messaging/MessageQueue.cs +++ /dev/null @@ -1,294 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Jobs; -using Foundatio.Serializer; -using Foundatio.Utility; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Foundatio.Messaging; - -public enum AckMode -{ - Auto, - Manual -} - -public sealed record MessageSendOptions -{ - public MessagePriority Priority { get; init; } = MessagePriority.Normal; - public TimeSpan? Delay { get; init; } - public DateTimeOffset? DeliverAt { get; init; } - public TimeSpan? TimeToLive { get; init; } - public string? CorrelationId { get; init; } - public string? DeduplicationId { get; init; } - public string? Destination { get; init; } - public MessageHeaders? Headers { get; init; } -} - -public sealed record QueueReceiveOptions -{ - public string? Source { get; init; } - public Type? RouteType { get; init; } - public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); -} - -/// -/// Core-owned retry and dead-letter policy. Foundatio always owns redelivery and dead-lettering so the behavior is -/// identical across transports; transports stay simple and only provide the underlying primitives (redelivery and an -/// optional dead-letter sink). Configure a default on /; a -/// consumer can override /backoff per consumer. -/// -public sealed record RetryPolicy -{ - /// Maximum delivery attempts for a failing handler before the message is dead-lettered. Default 5. - public int MaxAttempts { get; init; } = 5; - - /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's own redelivery timing. - public Func? Backoff { get; init; } - - /// - /// Destination terminal (dead-lettered) messages are sent to when the transport has no native dead-letter sink. - /// Null drops terminal messages on such transports. Ignored when the transport supports native dead-lettering. - /// - public string? DeadLetterDestination { get; init; } - - /// Maximum attempts for a message whose type has no registered consumer before it is dead-lettered as "no-handler". Default 50. - public int UnmatchedMaxAttempts { get; init; } = 50; - - /// Delay before redelivering an unmatched-type message. Null defers to the transport's own redelivery timing. - public Func? UnmatchedBackoff { get; init; } -} - -public sealed record QueueConsumerOptions -{ - public AckMode AckMode { get; init; } = AckMode.Auto; - public string? Source { get; init; } - public Type? RouteType { get; init; } - public string? Key { get; init; } - public int MaxConcurrency { get; init; } = 1; - // Null falls back to the queue's default RetryPolicy. - public int? MaxAttempts { get; init; } - public Func? RedeliveryBackoff { get; init; } -} - -public sealed record QueueOptions -{ - public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; - public string ContentType { get; init; } = "application/json"; - public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; - public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); - public IJobRuntimeStore? RuntimeStore { get; init; } - public RetryPolicy RetryPolicy { get; init; } = new(); - - /// - /// Whether disposing this queue also disposes the transport. True (default) for a transport this queue created or - /// solely uses; set false when the transport is a shared/externally-owned instance (e.g. a DI singleton also used - /// by a pub/sub client) so it is disposed exactly once by its owner. - /// - public bool OwnsTransport { get; init; } = true; - public TimeProvider TimeProvider { get; init; } = TimeProvider.System; - public ILoggerFactory? LoggerFactory { get; init; } -} - -public interface IQueue : IAsyncDisposable -{ - Task EnqueueAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default); - Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default); - Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default); - Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task RunConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default); - Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class; -} - -public interface IMessageConsumer : IAsyncDisposable -{ - string Source { get; } - string Key { get; } -} - -/// -/// Thrown by the consumer loop when a message arrives on a shared destination whose type has no registered consumer -/// on this node (for example a newer message type mid rolling-deploy, or a misconfiguration). It is surfaced loudly -/// per message and isolated to that message — the receive loop and the other type handlers keep running. -/// -public sealed class UnhandledMessageTypeException : Exception -{ - public UnhandledMessageTypeException(string? messageType, string source) - : base($"No consumer is registered for message type \"{messageType ?? "(unknown)"}\" received on source \"{source}\".") - { - MessageType = messageType; - SourceName = source; - } - - public string? MessageType { get; } - public string SourceName { get; } -} - -public sealed record RejectOptions -{ - /// - /// When false (default) the message is returned for redelivery (a retry). When true the message is terminal: it - /// is moved to the transport's dead-letter sink where one exists, otherwise dropped. Terminal messages are never - /// redelivered. - /// - public bool Terminal { get; init; } - - /// Reason carried to the dead-letter sink (where the transport supports one) for a terminal reject. - public string? Reason { get; init; } - - /// - /// An explicit delay before the message is redelivered. Honored only for a non-terminal reject, served natively - /// when the transport supports redelivery delay within its advertised maximum, otherwise through the runtime store. - /// When null the transport's own redelivery timing applies. - /// - public TimeSpan? RedeliveryDelay { get; init; } -} - -public interface IReceivedMessage -{ - string Id { get; } - ReadOnlyMemory Body { get; } - MessageHeaders Headers { get; } - string? CorrelationId { get; } - string? MessageType { get; } - MessagePriority Priority { get; } - int Attempts { get; } - bool IsHandled { get; } - CancellationToken CancellationToken { get; } - Task CompleteAsync(CancellationToken cancellationToken = default); - Task RejectAsync(RejectOptions? options = null, CancellationToken cancellationToken = default); - Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); -} - -public interface IReceivedMessage : IReceivedMessage where T : class -{ - T Message { get; } -} - -/// -/// App-facing durable competing-consumer queue. Routing, serialization, settlement, scheduling, and the consumer loop -/// live in ; this type maps queue-shaped options onto that shared core. -/// -public sealed class MessageQueue : IQueue -{ - private readonly MessageClientCore _core; - - public MessageQueue(IMessageTransport transport, QueueOptions? options = null) - { - ArgumentNullException.ThrowIfNull(transport); - options ??= new QueueOptions(); - var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); - _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageQueueException(message) : new MessageQueueException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); - } - - public Task EnqueueAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(message); - options ??= new MessageSendOptions(); - return _core.SendAsync(ScheduledDispatchKind.QueueMessage, typeof(T), message, ToEnvelope(options), GetDestination(typeof(T), options.Destination), ensureDestination: null, cancellationToken); - } - - public Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(messages); - options ??= new MessageSendOptions(); - return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); - } - - public Task EnqueueBatchAsync(IEnumerable messages, MessageSendOptions? options = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(messages); - options ??= new MessageSendOptions(); - return _core.SendBatchAsync(ScheduledDispatchKind.QueueMessage, messages, null, ToEnvelope(options), type => GetDestination(type, options.Destination), ensureDestination: null, cancellationToken); - } - - public Task ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) - { - options ??= new QueueReceiveOptions(); - return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(object), options.Source), options.MaxWaitTime, cancellationToken); - } - - public Task?> ReceiveAsync(QueueReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - options ??= new QueueReceiveOptions(); - return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(T), options.Source), options.MaxWaitTime, cancellationToken); - } - - public async Task StartConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(handler); - options ??= new QueueConsumerOptions(); - return await _core.StartListenerAsync(BuildConfig(options.RouteType ?? typeof(object), options), handler, cancellationToken).AnyContext(); - } - - public async Task StartConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(handler); - options ??= new QueueConsumerOptions(); - return await _core.StartListenerAsync(BuildConfig(options.RouteType ?? typeof(T), options), handler, cancellationToken).AnyContext(); - } - - public async Task RunConsumerAsync(Func handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) - { - await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public async Task RunConsumerAsync(Func, CancellationToken, Task> handler, QueueConsumerOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - await using var consumer = await StartConsumerAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public ValueTask DisposeAsync() - { - return _core.DisposeAsync(); - } - - private ListenerConfig BuildConfig(Type routeType, QueueConsumerOptions options) - { - string source = GetDestination(routeType, options.Source); - return new ListenerConfig - { - Source = source, - Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{source}:{routeType.FullName ?? routeType.Name}", - MessageType = routeType, - AckMode = options.AckMode, - MaxConcurrency = options.MaxConcurrency, - MaxAttempts = options.MaxAttempts, - RedeliveryBackoff = options.RedeliveryBackoff - }; - } - - private string GetDestination(Type messageType, string? destination) - { - return _core.Router.ResolveRoute(new MessageRouteContext - { - MessageType = messageType, - Role = MessageRouteRole.QueueDestination, - OperationOverride = destination - }); - } - - private static MessageEnvelopeOptions ToEnvelope(MessageSendOptions options) - { - return new MessageEnvelopeOptions - { - Priority = options.Priority, - Delay = options.Delay, - DeliverAt = options.DeliverAt, - TimeToLive = options.TimeToLive, - CorrelationId = options.CorrelationId, - DeduplicationId = options.DeduplicationId, - Headers = options.Headers - }; - } -} diff --git a/src/Foundatio/Messaging/MessageQueueException.cs b/src/Foundatio/Messaging/MessageQueueException.cs deleted file mode 100644 index f9935d4dd..000000000 --- a/src/Foundatio/Messaging/MessageQueueException.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; - -namespace Foundatio.Messaging; - -/// -/// Exception thrown when a message queue operation fails. -/// -public class MessageQueueException : MessageBusException -{ - public MessageQueueException(string message) : base(message) - { - } - - public MessageQueueException(string message, Exception innerException) : base(message, innerException) - { - } -} diff --git a/src/Foundatio/Messaging/PubSub.cs b/src/Foundatio/Messaging/PubSub.cs deleted file mode 100644 index abfc38bee..000000000 --- a/src/Foundatio/Messaging/PubSub.cs +++ /dev/null @@ -1,221 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Foundatio.Jobs; -using Foundatio.Serializer; -using Foundatio.Utility; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Foundatio.Messaging; - -public sealed record MessagePublishOptions -{ - public MessagePriority Priority { get; init; } = MessagePriority.Normal; - public TimeSpan? Delay { get; init; } - public DateTimeOffset? DeliverAt { get; init; } - public TimeSpan? TimeToLive { get; init; } - public string? CorrelationId { get; init; } - public string? DeduplicationId { get; init; } - public string? Topic { get; init; } - public MessageHeaders? Headers { get; init; } -} - -public sealed record PubSubSubscriptionOptions -{ - public string? Topic { get; init; } - public Type? RouteType { get; init; } - public string? Subscription { get; init; } - public string? Key { get; init; } - public AckMode AckMode { get; init; } = AckMode.Auto; - public int MaxConcurrency { get; init; } = 1; - // Null falls back to the pub/sub default RetryPolicy. - public int? MaxAttempts { get; init; } - public Func? RedeliveryBackoff { get; init; } -} - -public sealed record PubSubOptions -{ - public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; - public string ContentType { get; init; } = "application/json"; - public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; - public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); - public IJobRuntimeStore? RuntimeStore { get; init; } - public RetryPolicy RetryPolicy { get; init; } = new(); - - /// - /// Whether disposing this pub/sub client also disposes the transport. True (default) for a transport it solely - /// uses; set false when the transport is shared/externally owned (e.g. a DI singleton also used by a queue client). - /// - public bool OwnsTransport { get; init; } = true; - public TimeProvider TimeProvider { get; init; } = TimeProvider.System; - public ILoggerFactory? LoggerFactory { get; init; } -} - -public interface IPubSub : IAsyncDisposable -{ - Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default); - Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); - Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default); - Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; -} - -public interface IMessageSubscription : IAsyncDisposable -{ - string Topic { get; } - string Subscription { get; } - string Key { get; } - - /// - /// The transport destination this subscription receives from. It encodes both the topic and the subscription - /// identity (so the same subscription name on two different topics maps to two distinct sources), rather than the - /// bare subscription name. - /// - string Source { get; } -} - -/// -/// App-facing fan-out pub/sub. Routing, serialization, settlement, scheduling, and the subscription loop live in -/// ; this type maps topic/subscription-shaped options onto that shared core. -/// -public sealed class PubSub : IPubSub -{ - private readonly MessageClientCore _core; - - public PubSub(IMessageTransport transport, PubSubOptions? options = null) - { - ArgumentNullException.ThrowIfNull(transport); - options ??= new PubSubOptions(); - var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); - _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, - static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); - } - - public Task PublishAsync(T message, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(message); - options ??= new MessagePublishOptions(); - return _core.SendAsync(ScheduledDispatchKind.PubSubMessage, typeof(T), message, ToEnvelope(options), GetTopic(typeof(T), options.Topic), EnsureTopicAsync, cancellationToken); - } - - public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(messages); - options ??= new MessagePublishOptions(); - return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages.Cast(), typeof(T), ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); - } - - public Task PublishBatchAsync(IEnumerable messages, MessagePublishOptions? options = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(messages); - options ??= new MessagePublishOptions(); - return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); - } - - public async Task SubscribeAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(handler); - options ??= new PubSubSubscriptionOptions(); - var config = BuildConfig(options.RouteType ?? typeof(object), options); - await EnsureSubscriptionAsync(config, cancellationToken).AnyContext(); - return await _core.StartListenerAsync(config, handler, cancellationToken).AnyContext(); - } - - public async Task SubscribeAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - ArgumentNullException.ThrowIfNull(handler); - options ??= new PubSubSubscriptionOptions(); - var config = BuildConfig(options.RouteType ?? typeof(T), options); - await EnsureSubscriptionAsync(config, cancellationToken).AnyContext(); - return await _core.StartListenerAsync(config, handler, cancellationToken).AnyContext(); - } - - public async Task RunSubscriptionAsync(Func handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) - { - await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public async Task RunSubscriptionAsync(Func, CancellationToken, Task> handler, PubSubSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - await using var subscription = await SubscribeAsync(handler, options, cancellationToken).AnyContext(); - await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); - } - - public ValueTask DisposeAsync() - { - return _core.DisposeAsync(); - } - - private ListenerConfig BuildConfig(Type routeType, PubSubSubscriptionOptions options) - { - string topic = GetTopic(routeType, options.Topic); - string subscription = GetSubscription(routeType, topic, options.Subscription); - return new ListenerConfig - { - Topic = topic, - Subscription = subscription, - // The transport source is the topic-qualified subscription destination, not the bare subscription name, so - // the same subscription identity used on two topics resolves to two distinct sources (and isolates). - Source = SubscriptionAddress.Format(topic, subscription), - Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{routeType.FullName ?? routeType.Name}", - MessageType = routeType, - AckMode = options.AckMode, - MaxConcurrency = options.MaxConcurrency, - MaxAttempts = options.MaxAttempts, - RedeliveryBackoff = options.RedeliveryBackoff - }; - } - - private Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) - { - return _core.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken); - } - - private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken cancellationToken) - { - return _core.EnsureAsync([ - new DestinationDeclaration { Name = config.Topic, Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = config.Source, Role = DestinationRole.Subscription, Source = config.Topic } - ], cancellationToken); - } - - private string GetTopic(Type messageType, string? topic) - { - return _core.Router.ResolveRoute(new MessageRouteContext - { - MessageType = messageType, - Role = MessageRouteRole.PubSubTopic, - OperationOverride = topic - }); - } - - private string GetSubscription(Type messageType, string topic, string? subscription) - { - return _core.Router.ResolveSubscription(new MessageSubscriptionContext - { - MessageType = messageType, - Topic = topic, - OperationOverride = subscription - }); - } - - private static MessageEnvelopeOptions ToEnvelope(MessagePublishOptions options) - { - return new MessageEnvelopeOptions - { - Priority = options.Priority, - Delay = options.Delay, - DeliverAt = options.DeliverAt, - TimeToLive = options.TimeToLive, - CorrelationId = options.CorrelationId, - DeduplicationId = options.DeduplicationId, - Headers = options.Headers - }; - } -} diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index edb6edfad..831275682 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -35,10 +35,10 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh // Within the transport's advertised maximum: delivered natively, nothing is parked in Redis. var nativeStore = RedisTestConnection.CreateStore(connection); await using var nativeTransport = new CappedDelayTransport(TimeSpan.FromMinutes(15)); - await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); + await using var nativeQueue = new MessageBus(nativeTransport, new MessageBusOptions { RuntimeStore = nativeStore }); var nativeProcessor = CreateProcessor(nativeStore, nativeTransport).Processor; - await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + await nativeQueue.SendAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); Assert.Equal(1, nativeTransport.SendCount); Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); Assert.Equal(0, await nativeProcessor.RunDueOccurrencesAsync(now.AddYears(1), cancellationToken: cancellationToken)); @@ -46,10 +46,10 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh // Beyond the transport's maximum: routed into the Redis store rather than truncated to the broker ceiling. var fallbackStore = RedisTestConnection.CreateStore(connection); await using var fallbackTransport = new CappedDelayTransport(TimeSpan.FromMinutes(15)); - await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); + await using var fallbackQueue = new MessageBus(fallbackTransport, new MessageBusOptions { RuntimeStore = fallbackStore }); var fallbackProcessor = CreateProcessor(fallbackStore, fallbackTransport).Processor; - await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + await fallbackQueue.SendAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); Assert.Equal(0, fallbackTransport.SendCount); // Durably parked in Redis and time-gated: a drain before the due time claims nothing; only when due does the @@ -60,7 +60,7 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(now.AddHours(2), cancellationToken: cancellationToken)); Assert.Equal(1, fallbackTransport.SendCount); - var delivered = await fallbackQueue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var delivered = await fallbackQueue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); Assert.NotNull(delivered); Assert.Equal("later", delivered.Message.Data); await delivered.CompleteAsync(cancellationToken); @@ -235,7 +235,7 @@ private sealed class PreviewWorkItem } // Minimal pull transport with a configurable native delayed-delivery ceiling, so a delay beyond the cap is forced - // through the runtime store (mirrors the fixture used by the in-memory MessageQueue tests). + // through the runtime store (mirrors the fixture used by the in-memory MessageBus tests). private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ISupportsDelayedDelivery { private readonly Queue _entries = new(); diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs index 22e72e3ba..d3e7f39cc 100644 --- a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs @@ -9,7 +9,7 @@ namespace Foundatio.Redis.Tests; /// /// End-to-end tests for the Redis Streams transport that the cross-transport conformance suite can't express: at-least-once /// recovery across two consumer instances, the core's retry/dead-letter machinery driving the transport, and topic -/// fan-out through the facade. Gated on FOUNDATIO_REDIS_CONNECTION_STRING; unique key prefix +/// fan-out through the facade. Gated on FOUNDATIO_REDIS_CONNECTION_STRING; unique key prefix /// per test. /// public class RedisStreamsTransportIntegrationTests @@ -74,13 +74,13 @@ public async Task Core_RetriesFailedHandler_ThenDeadLettersAfterMaxAttemptsAsync var ct = TestContext.Current.CancellationToken; var transport = CreateTransport(connection, NewPrefix()); - await using var queue = new MessageQueue(transport, new QueueOptions()); + await using var queue = new MessageBus(transport, new MessageBusOptions()); // (a) A handler that throws once is redelivered (via the transport) and succeeds on the second attempt — the // core's retry machinery works unchanged over Streams. int retryAttempts = 0; var succeeded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - await using var retryConsumer = await queue.StartConsumerAsync((message, _) => + await using var retryConsumer = await queue.SubscribeAsync((message, _) => { int attempt = Interlocked.Increment(ref retryAttempts); if (attempt == 1) @@ -89,15 +89,15 @@ public async Task Core_RetriesFailedHandler_ThenDeadLettersAfterMaxAttemptsAsync Assert.Equal(2, message.Attempts); succeeded.TrySetResult(); return Task.CompletedTask; - }, new QueueConsumerOptions { MaxAttempts = 3, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(200) }, ct); + }, new MessageSubscriptionOptions { MaxAttempts = 3, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(200) }, ct); // (b) A handler that always throws is dead-lettered once its attempt budget is spent. - await using var poisonConsumer = await queue.StartConsumerAsync((_, _) => + await using var poisonConsumer = await queue.SubscribeAsync((_, _) => throw new InvalidOperationException("always fails"), - new QueueConsumerOptions { MaxAttempts = 2, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(100) }, ct); + new MessageSubscriptionOptions { MaxAttempts = 2, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(100) }, ct); - await queue.EnqueueAsync(new RetryItem { Data = "retry" }, cancellationToken: ct); - await queue.EnqueueAsync(new PoisonItem { Data = "poison" }, cancellationToken: ct); + await queue.SendAsync(new RetryItem { Data = "retry" }, cancellationToken: ct); + await queue.SendAsync(new PoisonItem { Data = "poison" }, cancellationToken: ct); await succeeded.Task.WaitAsync(TimeSpan.FromSeconds(30), ct); Assert.Equal(2, Volatile.Read(ref retryAttempts)); @@ -129,7 +129,7 @@ public async Task PubSub_PublishToTopic_FansOutToEverySubscriptionAsync() var ct = TestContext.Current.CancellationToken; var transport = CreateTransport(connection, NewPrefix()); - await using var pubsub = new PubSub(transport, new PubSubOptions()); + await using var pubsub = new MessageBus(transport, new MessageBusOptions()); var receivedByA = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var receivedByB = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -138,13 +138,13 @@ public async Task PubSub_PublishToTopic_FansOutToEverySubscriptionAsync() { receivedByA.TrySetResult(message.Message.Data ?? ""); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "sub-a" }, ct); + }, new MessageSubscriptionOptions { Subscription = "sub-a" }, ct); await using var subB = await pubsub.SubscribeAsync((message, _) => { receivedByB.TrySetResult(message.Message.Data ?? ""); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "sub-b" }, ct); + }, new MessageSubscriptionOptions { Subscription = "sub-b" }, ct); await pubsub.PublishAsync(new FanItem { Data = "broadcast" }, cancellationToken: ct); @@ -166,29 +166,23 @@ public async Task MessageBus_SendAndPublishSameType_StayIsolatedAsync() } var ct = TestContext.Current.CancellationToken; - await using var transport = CreateTransport(connection, NewPrefix()); - - // The unified bus: Send targets the queue-role stream, Publish the topic-role stream. The same route name must - // never cross-deliver — a publish must not be consumed as queue work and vice versa. - await using var queue = new MessageQueue(transport, new QueueOptions { OwnsTransport = false }); - await using var pubSub = new PubSub(transport, new PubSubOptions { OwnsTransport = false }); - await using var bus = new MessageBus(queue, pubSub); + var transport = CreateTransport(connection, NewPrefix()); + await using var bus = new MessageBus(transport, new MessageBusOptions()); + // One subscription listens on both of the type's channels. Send targets the queue-role stream and Publish the + // topic-role stream, so the same route name must never cross-deliver: exactly one delivery per verb. (A shared + // stream would deliver each message through BOTH channels — 4 deliveries instead of 2.) var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var published = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - int sendCount = 0, publishCount = 0; - - await using var consumer = await queue.StartConsumerAsync((message, _) => - { - Interlocked.Increment(ref sendCount); - sent.TrySetResult(message.Message.Data ?? ""); - return Task.CompletedTask; - }, cancellationToken: ct); + int deliveries = 0; - await using var subscription = await pubSub.SubscribeAsync((message, _) => + await using var subscription = await bus.SubscribeAsync((message, _) => { - Interlocked.Increment(ref publishCount); - published.TrySetResult(message.Message.Data ?? ""); + Interlocked.Increment(ref deliveries); + if (message.Message.Data == "for-one") + sent.TrySetResult(message.Message.Data); + else + published.TrySetResult(message.Message.Data ?? ""); return Task.CompletedTask; }, cancellationToken: ct); @@ -201,10 +195,42 @@ public async Task MessageBus_SendAndPublishSameType_StayIsolatedAsync() // Give any cross-delivery a moment to surface, then assert exactly one delivery per verb. await Task.Delay(500, ct); - Assert.Equal(1, Volatile.Read(ref sendCount)); - Assert.Equal(1, Volatile.Read(ref publishCount)); + Assert.Equal(2, Volatile.Read(ref deliveries)); } + [Fact] + public async Task Publish_CompletedByOneGroup_StillDeliveredToSlowerGroupAsync() + { + if (RedisTestConnection.Multiplexer is not { } connection) + { + Assert.Skip("FOUNDATIO_REDIS_CONNECTION_STRING not set."); + return; + } + + var ct = TestCancellation(); + await using var transport = CreateTransport(connection, NewPrefix()); + + await transport.EnsureAsync( + [ + new DestinationDeclaration { Name = "iso-topic", Role = DestinationRole.Topic }, + new DestinationDeclaration { Name = "iso-topic/sub-a", Role = DestinationRole.Subscription, Source = "iso-topic" }, + new DestinationDeclaration { Name = "iso-topic/sub-b", Role = DestinationRole.Subscription, Source = "iso-topic" } + ], ct); + + await transport.SendAsync("iso-topic", [Message("retained")], new TransportSendOptions { DestinationRole = DestinationRole.Topic }, ct); + + // Group A reads and completes FIRST; the entry must remain on the topic stream for group B (completing must + // not delete a shared topic entry other groups haven't read yet). + var byA = Assert.Single(await transport.ReceiveAsync("iso-topic/sub-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); + await transport.CompleteAsync(byA, ct); + + var byB = Assert.Single(await transport.ReceiveAsync("iso-topic/sub-b", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); + Assert.Equal("retained", System.Text.Encoding.UTF8.GetString(byB.Body.Span)); + await transport.CompleteAsync(byB, ct); + } + + private static CancellationToken TestCancellation() => TestContext.Current.CancellationToken; + private static TransportMessage Message(string body) => new() { Body = System.Text.Encoding.UTF8.GetBytes(body) }; diff --git a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs index 363d3cfb0..48e62dd91 100644 --- a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs +++ b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs @@ -100,6 +100,48 @@ public async Task AddHandler_PublishIsOncePerServiceUnlessPerInstanceAsync() } } + [Fact] + public async Task AddHandler_TwoHandlerClassesForOneType_EachGetsPublishedAndSendReachesOneAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var probe = new HandlerProbe(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(probe); + services.AddFoundatio() + .Messaging.UseInMemory() + .Messaging.AddHandler() + .Messaging.AddHandler(); + + await using var provider = services.BuildServiceProvider(); + var hosted = provider.GetServices().ToList(); + foreach (var service in hosted) + await service.StartAsync(cancellationToken); + + try + { + var bus = provider.GetRequiredService(); + + // An event reaches EVERY handler class (each is its own subscriber group). + await bus.PublishAsync(new HandledEvent { Id = "e1" }, cancellationToken: cancellationToken); + Assert.True(await probe.WaitForAsync(2, TimeSpan.FromSeconds(10)), $"handled: {String.Join(",", probe.Events)}"); + Assert.Contains("event:e1", probe.Events); + Assert.Contains("second:e1", probe.Events); + + // A command reaches exactly ONE handler (competing consumers on the type's send channel). + await bus.SendAsync(new HandledEvent { Id = "s1" }, cancellationToken: cancellationToken); + Assert.True(await probe.WaitForAsync(3, TimeSpan.FromSeconds(10)), $"handled: {String.Join(",", probe.Events)}"); + await Task.Delay(250, cancellationToken); + Assert.Equal(1, probe.Events.Count(e => e.EndsWith(":s1", StringComparison.Ordinal))); + } + finally + { + foreach (var service in hosted) + await service.StopAsync(cancellationToken); + } + } + [Fact] public async Task AddCronJob_RegistersDefinitionAndSchedulesWhenPumpStartsAsync() { @@ -222,6 +264,15 @@ public Task HandleAsync(IReceivedMessage message, CancellationToke } } + private sealed class SecondEventHandler(HandlerProbe probe) : IMessageHandler + { + public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + { + probe.Record($"second:{message.Message.Id}"); + return Task.CompletedTask; + } + } + private sealed class BroadcastHandler(HandlerProbe probe) : IMessageHandler { public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) diff --git a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs index 3bb219ea1..0e5f6a9ee 100644 --- a/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs +++ b/tests/Foundatio.Tests/Jobs/WorkItemJobTests.cs @@ -8,7 +8,6 @@ using Exceptionless; using Foundatio.AsyncEx; using Foundatio.Jobs.Legacy; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Queues; using Foundatio.Tests.Extensions; diff --git a/tests/Foundatio.Tests/Messaging/MessageTests.cs b/tests/Foundatio.Tests/Messaging/MessageTests.cs index 67de43086..695d9a586 100644 --- a/tests/Foundatio.Tests/Messaging/MessageTests.cs +++ b/tests/Foundatio.Tests/Messaging/MessageTests.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.InteropServices; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Xunit; diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 0cdfc2f6a..709a9f423 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -19,7 +19,7 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport); + await using var pubSub = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -31,14 +31,14 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() Assert.Equal("published", message.Message.Data); firstReceived.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "subscriber-a" }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "subscriber-a" }, cts.Token); await using var second = await pubSub.SubscribeAsync((message, _) => { Assert.Equal("published", message.Message.Data); secondReceived.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "subscriber-b" }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "subscriber-b" }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "published" }, cancellationToken: cancellationToken); @@ -55,7 +55,7 @@ public async Task SubscribeAsync_WithSameSubscriptionAndDifferentKeys_CompetesOn { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport); + await using var pubSub = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(2); @@ -68,12 +68,12 @@ public async Task SubscribeAsync_WithSameSubscriptionAndDifferentKeys_CompetesOn return Task.CompletedTask; }; - await using var first = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions + await using var first = await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions { Subscription = "billing-service", Key = "node-a" }, cts.Token); - await using var second = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions + await using var second = await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions { Subscription = "billing-service", Key = "node-b" @@ -100,7 +100,7 @@ public async Task SubscribeAsync_SameSubscriptionOnTwoTopics_IsolatesPerTopicAsy { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport); + await using var pubSub = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -116,7 +116,7 @@ public async Task SubscribeAsync_SameSubscriptionOnTwoTopics_IsolatesPerTopicAsy ordersReceived.Add(message.Message.Data); ordersSignal.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Topic = "orders", Subscription = "shared" }, cts.Token); + }, new MessageSubscriptionOptions { Topic = "orders", Subscription = "shared" }, cts.Token); await using var payments = await pubSub.SubscribeAsync((message, _) => { @@ -124,7 +124,7 @@ public async Task SubscribeAsync_SameSubscriptionOnTwoTopics_IsolatesPerTopicAsy paymentsReceived.Add(message.Message.Data); paymentsSignal.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Topic = "payments", Subscription = "shared" }, cts.Token); + }, new MessageSubscriptionOptions { Topic = "payments", Subscription = "shared" }, cts.Token); Assert.Equal(orders.Subscription, payments.Subscription); // same logical subscription identity Assert.NotEqual(orders.Source, payments.Source); // but distinct topic-qualified transport sources @@ -149,7 +149,7 @@ public async Task PublishBatchAsync_DeliversAllMessagesAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport); + await using var pubSub = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(2); @@ -159,7 +159,7 @@ public async Task PublishBatchAsync_DeliversAllMessagesAsync() Assert.StartsWith("batch-", message.Message.Data); received.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "batch-subscription" }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "batch-subscription" }, cts.Token); await pubSub.PublishBatchAsync([ new PreviewEvent { Data = "batch-one" }, @@ -175,7 +175,7 @@ await pubSub.PublishBatchAsync([ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var pubSub = new PubSub(new InMemoryMessageTransport()); + await using var pubSub = new MessageBus(new InMemoryMessageTransport()); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); @@ -184,7 +184,7 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() { received.TrySetResult(message); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "metadata-subscription" }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "metadata" }, new MessagePublishOptions { @@ -213,7 +213,7 @@ public async Task PublishAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() var cancellationToken = TestContext.Current.CancellationToken; var store = new InMemoryJobRuntimeStore(); await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport, new PubSubOptions { RuntimeStore = store }); + await using var pubSub = new MessageBus(transport, new MessageBusOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -223,7 +223,7 @@ public async Task PublishAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() { received.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "delayed-subscription" }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new MessagePublishOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); @@ -238,7 +238,7 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var pubSub = new PubSub(transport); + await using var pubSub = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(2); @@ -254,7 +254,7 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() throw new InvalidOperationException("try again"); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { Subscription = "retry-subscription", MaxAttempts = 2 }, cts.Token); + }, new MessageSubscriptionOptions { Subscription = "retry-subscription", MaxAttempts = 2 }, cts.Token); await pubSub.PublishAsync(new PreviewEvent { Data = "retry" }, cancellationToken: cancellationToken); @@ -266,28 +266,43 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() [Fact] - public async Task SubscribeAsync_WithSameKeyAndSameRegistration_ReturnsExistingSubscriptionAsync() + public async Task SubscribeAsync_WithSameKeyAndSameRegistration_SharesTheUnderlyingConsumerAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var pubSub = new PubSub(new InMemoryMessageTransport()); - Func, CancellationToken, Task> handler = (_, _) => Task.CompletedTask; + await using var pubSub = new MessageBus(new InMemoryMessageTransport()); + int handled = 0; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Func, CancellationToken, Task> handler = (_, _) => + { + Interlocked.Increment(ref handled); + received.TrySetResult(); + return Task.CompletedTask; + }; + + // Registering the same key + handler + options twice is idempotent: both handles refer to the one underlying + // consumer, so a published message is handled exactly once. + await using var first = await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + await using var second = await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); - await using var first = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); - var second = await pubSub.SubscribeAsync(handler, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + Assert.Equal(first.Key, second.Key); + Assert.Equal(first.Source, second.Source); - Assert.Same(first, second); + await pubSub.PublishAsync(new PreviewEvent { Data = "once" }, cancellationToken: cancellationToken); + await received.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + await Task.Delay(250, cancellationToken); + Assert.Equal(1, Volatile.Read(ref handled)); } [Fact] public async Task SubscribeAsync_WithSameKeyAndDifferentHandler_ThrowsAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var pubSub = new PubSub(new InMemoryMessageTransport()); + await using var pubSub = new MessageBus(new InMemoryMessageTransport()); - await using var first = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); + await using var first = await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new MessageSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken); await Assert.ThrowsAsync(async () => - await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new PubSubSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken)); + await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new MessageSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken)); } [Fact] @@ -299,7 +314,7 @@ public async Task SubscribeAsync_WithGroupedTopicAndSubscriptionIdentity_Receive .MapTopic("order-events", typeof(IGroupedEvent)) .UseSubscriptionIdentity("billing-service") .Build(); - await using var pubSub = new PubSub(transport, new PubSubOptions { Router = new DefaultMessageRouter(routing) }); + await using var pubSub = new MessageBus(transport, new MessageBusOptions { Router = new DefaultMessageRouter(routing) }); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var received = new AsyncCountdownEvent(2); @@ -312,7 +327,7 @@ public async Task SubscribeAsync_WithGroupedTopicAndSubscriptionIdentity_Receive received.Signal(); return Task.CompletedTask; - }, new PubSubSubscriptionOptions { RouteType = typeof(IGroupedEvent) }, cts.Token); + }, new MessageSubscriptionOptions { RouteType = typeof(IGroupedEvent) }, cts.Token); await pubSub.PublishBatchAsync(new object[] { diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 560c56e4b..c7b55e9e8 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -20,9 +20,9 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - string id = await queue.EnqueueAsync(new PreviewWorkItem { Data = "hello" }, new MessageSendOptions + string id = await queue.SendAsync(new PreviewWorkItem { Data = "hello" }, new MessageSendOptions { CorrelationId = "corr-123", Priority = MessagePriority.High, @@ -31,7 +31,7 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() ]) }, cancellationToken); - var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(received); Assert.Equal(id, received.Id); @@ -54,15 +54,15 @@ public async Task EnqueueBatchAsync_UsesDestinationOverrideAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - await queue.EnqueueBatchAsync([ + await queue.SendBatchAsync([ new PreviewWorkItem { Data = "one" }, new PreviewWorkItem { Data = "two" } ], new MessageSendOptions { Destination = "custom-work" }, cancellationToken); - var first = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - var second = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var first = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); Assert.NotNull(second); @@ -77,15 +77,15 @@ await queue.EnqueueBatchAsync([ public async Task RejectAsync_NonTerminal_RedeliversAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cancellationToken); + await using var queue = new MessageBus(new InMemoryMessageTransport()); + await queue.SendAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cancellationToken); - var first = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var first = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); await first.RejectAsync(cancellationToken: cancellationToken); - var second = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(second); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.Attempts); @@ -100,10 +100,10 @@ public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() var cancellationToken = TestContext.Current.CancellationToken; // BasicQueueTransport intentionally does not implement ISupportsLockRenewal, so the core must surface the // unsupported capability rather than silently no-op. - await using var queue = new MessageQueue(new BasicQueueTransport()); + await using var queue = new MessageBus(new BasicQueueTransport()); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); await Assert.ThrowsAsync(async () => await message.RenewLockAsync(cancellationToken: cancellationToken)); @@ -114,10 +114,10 @@ public async Task RejectAsync_Terminal_DeadLettersAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); @@ -132,19 +132,19 @@ public async Task StartConsumerAsync_WithAutoAck_CompletesMessageAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var handled = new AsyncCountdownEvent(1); - await using var consumer = await queue.StartConsumerAsync((message, _) => + await using var consumer = await queue.SubscribeAsync((message, _) => { Assert.Equal("work", message.Message.Data); handled.Signal(); return Task.CompletedTask; }, cancellationToken: cts.Token); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "work" }, cancellationToken: cts.Token); + await queue.SendAsync(new PreviewWorkItem { Data = "work" }, cancellationToken: cts.Token); await handled.WaitAsync(TimeSpan.FromSeconds(2)); await WaitForCompletedAsync(transport, "preview-work-item", cancellationToken); } @@ -154,18 +154,18 @@ public async Task StartConsumerAsync_WithManualAck_DoesNotAutoCompleteAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var handled = new AsyncCountdownEvent(1); - await using var consumer = await queue.StartConsumerAsync((message, _) => + await using var consumer = await queue.SubscribeAsync((message, _) => { handled.Signal(); return Task.CompletedTask; // intentionally does NOT settle the message - }, new QueueConsumerOptions { AckMode = AckMode.Manual }, cts.Token); + }, new MessageSubscriptionOptions { AckMode = AckMode.Manual }, cts.Token); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "manual" }, cancellationToken: cts.Token); + await queue.SendAsync(new PreviewWorkItem { Data = "manual" }, cancellationToken: cts.Token); await handled.WaitAsync(TimeSpan.FromSeconds(2)); await Task.Delay(200, cts.Token); @@ -180,12 +180,12 @@ public async Task StartConsumerAsync_WithPoisonMessage_DeadLettersAndKeepsConsum { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); var handled = new AsyncCountdownEvent(1); - await using var consumer = await queue.StartConsumerAsync((message, _) => + await using var consumer = await queue.SubscribeAsync((message, _) => { Assert.Equal("good", message.Message.Data); handled.Signal(); @@ -197,7 +197,7 @@ public async Task StartConsumerAsync_WithPoisonMessage_DeadLettersAndKeepsConsum await transport.SendAsync("preview-work-item", [ new TransportMessage { Body = System.Text.Encoding.UTF8.GetBytes("}{ not json"), Headers = MessageHeaders.Empty } ], new TransportSendOptions(), cts.Token); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "good" }, cancellationToken: cts.Token); + await queue.SendAsync(new PreviewWorkItem { Data = "good" }, cancellationToken: cts.Token); await handled.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(0, handled.CurrentCount); @@ -208,9 +208,9 @@ public async Task EnqueueBatchAsync_RespectsTransportMaxBatchSizeAsync() { var cancellationToken = TestContext.Current.CancellationToken; var transport = new BatchLimitTransport(maxBatchSize: 2); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - await queue.EnqueueBatchAsync(new[] + await queue.SendBatchAsync(new[] { new PreviewWorkItem { Data = "1" }, new PreviewWorkItem { Data = "2" }, @@ -229,17 +229,17 @@ public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() var cancellationToken = TestContext.Current.CancellationToken; var store = new InMemoryJobRuntimeStore(); await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); - var immediate = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var immediate = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); - var delayed = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var delayed = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); @@ -249,10 +249,10 @@ public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() public async Task EnqueueAsync_WithDelayAndNoRuntimeStore_ThrowsAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + await using var queue = new MessageBus(new InMemoryMessageTransport()); - await Assert.ThrowsAsync(async () => - await queue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); + await Assert.ThrowsAsync(async () => + await queue.SendAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken)); } [Fact] @@ -263,10 +263,10 @@ public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync( // Within the transport's advertised maximum: delivered natively, never touches the runtime store. var nativeStore = new InMemoryJobRuntimeStore(); await using var nativeTransport = new CappedDelayTransport(maxDeliveryDelay: TimeSpan.FromMinutes(15)); - await using var nativeQueue = new MessageQueue(nativeTransport, new QueueOptions { RuntimeStore = nativeStore }); + await using var nativeQueue = new MessageBus(nativeTransport, new MessageBusOptions { RuntimeStore = nativeStore }); var nativeProcessor = CreateDispatchProcessor(nativeStore, nativeTransport); - await nativeQueue.EnqueueAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + await nativeQueue.SendAsync(new PreviewWorkItem { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); Assert.Equal(1, nativeTransport.SendCount); Assert.NotNull(nativeTransport.LastSendOptions?.DeliverAt); @@ -275,16 +275,16 @@ public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync( // Beyond the transport's maximum: routed through the runtime store instead of being silently truncated. var fallbackStore = new InMemoryJobRuntimeStore(); await using var fallbackTransport = new CappedDelayTransport(maxDeliveryDelay: TimeSpan.FromMinutes(15)); - await using var fallbackQueue = new MessageQueue(fallbackTransport, new QueueOptions { RuntimeStore = fallbackStore }); + await using var fallbackQueue = new MessageBus(fallbackTransport, new MessageBusOptions { RuntimeStore = fallbackStore }); var fallbackProcessor = CreateDispatchProcessor(fallbackStore, fallbackTransport); - await fallbackQueue.EnqueueAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); + await fallbackQueue.SendAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromHours(1) }, cancellationToken); Assert.Equal(0, fallbackTransport.SendCount); Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddHours(2), cancellationToken: cancellationToken)); Assert.Equal(1, fallbackTransport.SendCount); - var delayed = await fallbackQueue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var delayed = await fallbackQueue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); @@ -299,7 +299,7 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough // never seeds the delivery count from the message.attempts header, proving the core reconciles the attempt // count from the header itself (second attempt must observe Attempts == 2, not a reset-to-1 loop). await using var transport = new BasicQueueTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -307,7 +307,7 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough var secondAttempt = new AsyncCountdownEvent(1); int attempts = 0; - await using var consumer = await queue.StartConsumerAsync((message, _) => + await using var consumer = await queue.SubscribeAsync((message, _) => { attempts++; if (attempts == 1) @@ -321,12 +321,12 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough Assert.Equal("retry", message.Message.Data); secondAttempt.Signal(); return Task.CompletedTask; - }, new QueueConsumerOptions { RedeliveryBackoff = _ => TimeSpan.FromMinutes(1), MaxAttempts = 3 }, cts.Token); + }, new MessageSubscriptionOptions { RedeliveryBackoff = _ => TimeSpan.FromMinutes(1), MaxAttempts = 3 }, cts.Token); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cts.Token); + await queue.SendAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cts.Token); await firstAttempt.WaitAsync(TimeSpan.FromSeconds(2)); - var immediate = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var immediate = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(immediate); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); @@ -341,10 +341,10 @@ public async Task EnqueueAsync_ExceedingTransportMaxMessageBytes_ThrowsAsync() // The transport advertises an 8-byte maximum; the core must enforce it up front with a clear error rather than // let an opaque broker rejection surface mid-send. await using var transport = new BatchLimitTransport(maxBatchSize: 10, maxMessageBytes: 8); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - await Assert.ThrowsAsync(async () => - await queue.EnqueueAsync(new PreviewWorkItem { Data = "a payload well over eight bytes" }, cancellationToken: cancellationToken)); + await Assert.ThrowsAsync(async () => + await queue.SendAsync(new PreviewWorkItem { Data = "a payload well over eight bytes" }, cancellationToken: cancellationToken)); } [Fact] @@ -357,15 +357,15 @@ public async Task RejectAsync_RuntimeStoreRedelivery_AdvancesAttemptCountEachCyc // redeliveries (1 -> 2 -> 3); a regression that bases the next attempt on the reset DeliveryCount would pin it at // 2 and redeliver forever (never reaching MaxAttempts / dead-letter). await using var transport = new BasicQueueTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RuntimeStore = store }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RuntimeStore = store }); var processor = CreateDispatchProcessor(store, transport); var now = DateTimeOffset.UtcNow; - await queue.EnqueueAsync(new PreviewWorkItem { Data = "loop" }, cancellationToken: cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "loop" }, cancellationToken: cancellationToken); for (int expectedAttempt = 1; expectedAttempt <= 3; expectedAttempt++) { - var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(received); Assert.Equal(expectedAttempt, received.Attempts); Assert.Equal("loop", received.Message.Data); @@ -387,11 +387,11 @@ public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "expired" }, new MessageSendOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "expired" }, new MessageSendOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); - var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); Assert.Null(received); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); @@ -403,7 +403,7 @@ public async Task ReceiveAsync_WithPoisonPayload_DeadLettersAndThrowsMessageQueu { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport); + await using var queue = new MessageBus(transport); await transport.SendAsync("preview-work-item", [ new TransportMessage @@ -415,8 +415,8 @@ await transport.SendAsync("preview-work-item", [ } ], new TransportSendOptions(), cancellationToken); - await Assert.ThrowsAsync(async () => - await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); + await Assert.ThrowsAsync(async () => + await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Deadletter); @@ -435,8 +435,7 @@ public async Task AddFoundatio_WithInMemoryMessagingAndJobs_RegistersAppFacingSe await using var provider = services.BuildServiceProvider(); - Assert.NotNull(provider.GetRequiredService()); - Assert.NotNull(provider.GetRequiredService()); + Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); Assert.NotNull(provider.GetRequiredService()); @@ -486,11 +485,11 @@ public async Task AddFoundatio_WithRouting_RegistersRouterAndTopologyAsync() public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + await using var queue = new MessageBus(new InMemoryMessageTransport()); - await queue.EnqueueAsync(new RoutedWorkItem { Data = "route" }, cancellationToken: cancellationToken); + await queue.SendAsync(new RoutedWorkItem { Data = "route" }, cancellationToken: cancellationToken); - var received = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "routed-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "routed-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(received); Assert.Equal("route", received.Message.Data); @@ -498,28 +497,43 @@ public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync } [Fact] - public async Task StartConsumerAsync_WithSameKeyAndSameRegistration_ReturnsExistingConsumerAsync() + public async Task StartConsumerAsync_WithSameKeyAndSameRegistration_SharesTheUnderlyingConsumerAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); - Func, CancellationToken, Task> handler = (_, _) => Task.CompletedTask; + await using var queue = new MessageBus(new InMemoryMessageTransport()); + int handled = 0; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Func, CancellationToken, Task> handler = (_, _) => + { + Interlocked.Increment(ref handled); + received.TrySetResult(); + return Task.CompletedTask; + }; + + // Registering the same key + handler + options twice is idempotent: both handles refer to the one underlying + // consumer, so a sent message is handled exactly once. + await using var first = await queue.SubscribeAsync(handler, new MessageSubscriptionOptions { Key = "shared" }, cancellationToken); + await using var second = await queue.SubscribeAsync(handler, new MessageSubscriptionOptions { Key = "shared" }, cancellationToken); - await using var first = await queue.StartConsumerAsync(handler, new QueueConsumerOptions { Key = "shared" }, cancellationToken); - var second = await queue.StartConsumerAsync(handler, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + Assert.Equal(first.Key, second.Key); + Assert.Equal(first.Destination, second.Destination); - Assert.Same(first, second); + await queue.SendAsync(new PreviewWorkItem { Data = "once" }, cancellationToken: cancellationToken); + await received.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); + await Task.Delay(250, cancellationToken); + Assert.Equal(1, Volatile.Read(ref handled)); } [Fact] public async Task StartConsumerAsync_WithSameKeyAndDifferentHandler_ThrowsAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + await using var queue = new MessageBus(new InMemoryMessageTransport()); - await using var first = await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken); + await using var first = await queue.SubscribeAsync((_, _) => Task.CompletedTask, new MessageSubscriptionOptions { Key = "shared" }, cancellationToken); await Assert.ThrowsAsync(async () => - await queue.StartConsumerAsync((_, _) => Task.CompletedTask, new QueueConsumerOptions { Key = "shared" }, cancellationToken)); + await queue.SubscribeAsync((_, _) => Task.CompletedTask, new MessageSubscriptionOptions { Key = "shared" }, cancellationToken)); } [Fact] @@ -529,16 +543,16 @@ public async Task ReceiveAsync_WithGroupedInterfaceRoute_ReturnsRawMessagesAsync var routing = new MessageRoutingOptionsBuilder() .MapQueue("grouped-work", typeof(IGroupedWorkItem)) .Build(); - await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + await using var queue = new MessageBus(new InMemoryMessageTransport(), new MessageBusOptions { Router = new DefaultMessageRouter(routing) }); - await queue.EnqueueBatchAsync(new object[] + await queue.SendBatchAsync(new object[] { new PreviewWorkItem { Data = "one" }, new OtherWorkItem { Data = "two" } }, cancellationToken: cancellationToken); - var first = await queue.ReceiveAsync(new QueueReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - var second = await queue.ReceiveAsync(new QueueReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var first = await queue.ReceiveAsync(new MessageReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await queue.ReceiveAsync(new MessageReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(first); Assert.NotNull(second); @@ -557,7 +571,7 @@ public async Task StartConsumerAsync_WithGroupedInterfaceRoute_DeserializesConcr var routing = new MessageRoutingOptionsBuilder() .MapQueue("grouped-work", typeof(IGroupedWorkItem)) .Build(); - await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + await using var queue = new MessageBus(new InMemoryMessageTransport(), new MessageBusOptions { Router = new DefaultMessageRouter(routing) }); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -566,7 +580,7 @@ public async Task StartConsumerAsync_WithGroupedInterfaceRoute_DeserializesConcr // An interface-typed consumer receives the concrete payload (assignable to the interface), not raw bytes — // the core resolves the concrete type from the message-type header and deserializes that. - await using var consumer = await queue.StartConsumerAsync((message, _) => + await using var consumer = await queue.SubscribeAsync((message, _) => { string? data = message.Message switch { @@ -579,7 +593,7 @@ public async Task StartConsumerAsync_WithGroupedInterfaceRoute_DeserializesConcr return Task.CompletedTask; }, cancellationToken: cts.Token); - await queue.EnqueueBatchAsync(new object[] + await queue.SendBatchAsync(new object[] { new PreviewWorkItem { Data = "one" }, new OtherWorkItem { Data = "two" } @@ -598,11 +612,11 @@ public async Task ReceiveAsync_WithDefaultQueueRoute_ReturnsRawMessageAsync() var routing = new MessageRoutingOptionsBuilder() .UseDefaultQueue("all-work") .Build(); - await using var queue = new MessageQueue(new InMemoryMessageTransport(), new QueueOptions { Router = new DefaultMessageRouter(routing) }); + await using var queue = new MessageBus(new InMemoryMessageTransport(), new MessageBusOptions { Router = new DefaultMessageRouter(routing) }); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "global" }, cancellationToken: cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "global" }, cancellationToken: cancellationToken); - var received = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(received); Assert.Equal(typeof(PreviewWorkItem).FullName, received.MessageType); @@ -630,7 +644,7 @@ private static async Task WaitForCompletedAsync(InMemoryMessageTransport transpo public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTypeAsync() { var cancellationToken = TestContext.Current.CancellationToken; - await using var queue = new MessageQueue(new InMemoryMessageTransport()); + await using var queue = new MessageBus(new InMemoryMessageTransport()); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); @@ -639,7 +653,7 @@ public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTyp var aSignal = new AsyncCountdownEvent(1); var bSignal = new AsyncCountdownEvent(1); - await using var consumerA = await queue.StartConsumerAsync((message, _) => + await using var consumerA = await queue.SubscribeAsync((message, _) => { lock (aReceived) aReceived.Add(message.Message.Data); @@ -647,7 +661,7 @@ public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTyp return Task.CompletedTask; }, cancellationToken: cts.Token); - await using var consumerB = await queue.StartConsumerAsync((message, _) => + await using var consumerB = await queue.SubscribeAsync((message, _) => { lock (bReceived) bReceived.Add(message.Message.Data); @@ -658,8 +672,8 @@ public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTyp // Both types route to the same destination, so they share one underlying receive loop that dispatches by type. Assert.Equal(consumerA.Source, consumerB.Source); - await queue.EnqueueAsync(new SharedAWorkItem { Data = "a" }, cancellationToken: cts.Token); - await queue.EnqueueAsync(new SharedBWorkItem { Data = "b" }, cancellationToken: cts.Token); + await queue.SendAsync(new SharedAWorkItem { Data = "a" }, cancellationToken: cts.Token); + await queue.SendAsync(new SharedBWorkItem { Data = "b" }, cancellationToken: cts.Token); await aSignal.WaitAsync(TimeSpan.FromSeconds(2)); await bSignal.WaitAsync(TimeSpan.FromSeconds(2)); @@ -673,19 +687,19 @@ public async Task StartConsumerAsync_UnmatchedType_DeadLettersAndKeepsConsumingA { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { UnmatchedMaxAttempts = 3 } }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RetryPolicy = new RetryPolicy { UnmatchedMaxAttempts = 3 } }); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(20)); var aSignal = new AsyncCountdownEvent(1); - await using var consumerA = await queue.StartConsumerAsync((_, _) => + await using var consumerA = await queue.SubscribeAsync((_, _) => { aSignal.Signal(); return Task.CompletedTask; }, cancellationToken: cts.Token); // SharedBWorkItem routes to the same destination but has no registered consumer on this node. - await queue.EnqueueAsync(new SharedBWorkItem { Data = "orphan" }, cancellationToken: cts.Token); + await queue.SendAsync(new SharedBWorkItem { Data = "orphan" }, cancellationToken: cts.Token); // It is retried and finally dead-lettered as "no-handler" once the configured unmatched budget is exhausted. for (int i = 0; i < 400; i++) @@ -698,7 +712,7 @@ public async Task StartConsumerAsync_UnmatchedType_DeadLettersAndKeepsConsumingA Assert.Equal(1, (await transport.GetStatsAsync("shared-demux", cts.Token)).Deadletter); // The loop survived the unmatched message and keeps consuming the type it does handle. - await queue.EnqueueAsync(new SharedAWorkItem { Data = "ok" }, cancellationToken: cts.Token); + await queue.SendAsync(new SharedAWorkItem { Data = "ok" }, cancellationToken: cts.Token); await aSignal.WaitAsync(TimeSpan.FromSeconds(2)); } @@ -707,16 +721,16 @@ public async Task RejectAsync_Terminal_WithoutNativeDeadLetter_SendsToConfigured { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new NoDeadLetterTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { DeadLetterDestination = "preview-dead-letter" } }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RetryPolicy = new RetryPolicy { DeadLetterDestination = "preview-dead-letter" } }); - await queue.EnqueueAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new QueueReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await queue.SendAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); + var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(message); await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); // The transport has no native dead-letter sink, so core routes the terminal message to the configured destination. - var dead = await queue.ReceiveAsync(new QueueReceiveOptions { Source = "preview-dead-letter", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var dead = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "preview-dead-letter", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); Assert.NotNull(dead); Assert.Equal("validation", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterReason)); } @@ -726,18 +740,18 @@ public async Task StartConsumerAsync_UsesDefaultRetryPolicyMaxAttempts_WhenConsu { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageQueue(transport, new QueueOptions { RetryPolicy = new RetryPolicy { MaxAttempts = 2 } }); + await using var queue = new MessageBus(transport, new MessageBusOptions { RetryPolicy = new RetryPolicy { MaxAttempts = 2 } }); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(20)); int attempts = 0; - await using var consumer = await queue.StartConsumerAsync((_, _) => + await using var consumer = await queue.SubscribeAsync((_, _) => { Interlocked.Increment(ref attempts); throw new InvalidOperationException("always fails"); }, cancellationToken: cts.Token); // no per-consumer MaxAttempts -> default RetryPolicy (2) - await queue.EnqueueAsync(new PreviewWorkItem { Data = "x" }, cancellationToken: cts.Token); + await queue.SendAsync(new PreviewWorkItem { Data = "x" }, cancellationToken: cts.Token); for (int i = 0; i < 400; i++) { @@ -757,14 +771,14 @@ public async Task DisposeAsync_RespectsTransportOwnershipAsync() // Non-owning client (shared transport): disposing the client leaves the transport usable. var shared = new InMemoryMessageTransport(); - var nonOwning = new MessageQueue(shared, new QueueOptions { OwnsTransport = false }); + var nonOwning = new MessageBus(shared, new MessageBusOptions { OwnsTransport = false }); await nonOwning.DisposeAsync(); await shared.SendAsync("still-alive", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken); await shared.DisposeAsync(); // Owning client (default): disposing the client disposes the transport. var owned = new InMemoryMessageTransport(); - var owning = new MessageQueue(owned); + var owning = new MessageBus(owned); await owning.DisposeAsync(); await Assert.ThrowsAsync(async () => await owned.SendAsync("dead", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken)); @@ -778,13 +792,11 @@ public async Task DiBuiltClients_ShareTransport_DisposedExactlyOnceAsync() services.AddFoundatio().Messaging.UseTransport(transport); await using var provider = services.BuildServiceProvider(); - // Both clients resolve the same singleton transport. - _ = provider.GetRequiredService(); - _ = provider.GetRequiredService(); + _ = provider.GetRequiredService(); await provider.DisposeAsync(); - // The container owns the shared transport singleton; neither client disposes it, so it is disposed once. + // The container owns the shared transport singleton; the bus does not dispose it, so it is disposed once. Assert.Equal(1, transport.DisposeCount); } diff --git a/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs b/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs index e698317f7..16671bc8e 100644 --- a/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs +++ b/tests/Foundatio.Tests/Utility/ResiliencePolicyTests.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using Foundatio.Caching; using Foundatio.Lock; -using Foundatio.Messaging; using Foundatio.Messaging.Legacy; using Foundatio.Resilience; using Foundatio.Utility; From 8e500bb4ba4ec343ae86c16ae6410360b5518a87 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 20:10:12 -0500 Subject: [PATCH 40/57] Rename IReceivedMessage to IMessageContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handler-facing type is the consumption context wrapping a message — payload plus delivery metadata (Attempts, headers) plus the operations valid during this delivery (Complete/Reject/RenewLock) — so name it that, matching the ecosystem convention (MassTransit ConsumeContext.Message) and rhyming with the jobs side, where a job runs with a JobExecutionContext. Handlers now read context.Message instead of message.Message; internal CreateReceivedMessage/ ReceivedMessage names follow suit, and handler parameters are named context. Mechanical rename; no behavior change. Full solution builds; in-memory 2006 green; live Redis 29 green. Co-Authored-By: Claude Opus 4.8 --- samples/Foundatio.MessagingSample/Handlers.cs | 8 ++-- src/Foundatio/FoundatioServicesExtensions.cs | 6 +-- ...IReceivedMessage.cs => IMessageContext.cs} | 4 +- src/Foundatio/Messaging/IMessageHandler.cs | 2 +- src/Foundatio/Messaging/MessageBus.cs | 16 +++---- src/Foundatio/Messaging/MessageClientCore.cs | 42 +++++++++---------- .../DeclarativeRegistrationTests.cs | 18 ++++---- .../Foundatio.Tests/Messaging/PubSubTests.cs | 6 +-- .../Queue/MessageQueueTests.cs | 2 +- 9 files changed, 52 insertions(+), 52 deletions(-) rename src/Foundatio/Messaging/{IReceivedMessage.cs => IMessageContext.cs} (97%) diff --git a/samples/Foundatio.MessagingSample/Handlers.cs b/samples/Foundatio.MessagingSample/Handlers.cs index 94cef3619..563aa7dbf 100644 --- a/samples/Foundatio.MessagingSample/Handlers.cs +++ b/samples/Foundatio.MessagingSample/Handlers.cs @@ -12,9 +12,9 @@ public sealed record InstanceInfo(string Id); /// public sealed class ProcessOrderHandler(InstanceInfo instance, ILogger logger) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - logger.LogInformation("[{Instance}] processed order: {Quantity} x {Product}", instance.Id, message.Message.Quantity, message.Message.Product); + logger.LogInformation("[{Instance}] processed order: {Quantity} x {Product}", instance.Id, context.Message.Quantity, context.Message.Product); return Task.CompletedTask; } } @@ -25,9 +25,9 @@ public Task HandleAsync(IReceivedMessage message, CancellationToke /// public sealed class AnnouncementHandler(InstanceInfo instance, ILogger logger) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - logger.LogInformation("[{Instance}] announcement: {Text}", instance.Id, message.Message.Text); + logger.LogInformation("[{Instance}] announcement: {Text}", instance.Id, context.Message.Text); return Task.CompletedTask; } } diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 2fe3d80c1..d5362d6c5 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -369,14 +369,14 @@ public FoundatioBuilder AddHandler(Action; see /// for the delivery semantics. /// - public FoundatioBuilder AddHandler(Func, CancellationToken, Task> handler, Action? configure = null) + public FoundatioBuilder AddHandler(Func, CancellationToken, Task> handler, Action? configure = null) where TMessage : class { ArgumentNullException.ThrowIfNull(handler); return AddHandlerRegistration(null, (_, message, ct) => handler(message, ct), configure); } - private FoundatioBuilder AddHandlerRegistration(string? handlerName, Func, CancellationToken, Task> dispatch, Action? configure) + private FoundatioBuilder AddHandlerRegistration(string? handlerName, Func, CancellationToken, Task> dispatch, Action? configure) where TMessage : class { string suffix = handlerName is null ? String.Empty : $" -> {handlerName}"; @@ -398,7 +398,7 @@ private FoundatioBuilder AddHandlerRegistration(string? handlerName, F return _builder; } - private static async Task DispatchAsync(IServiceProvider serviceProvider, IReceivedMessage message, CancellationToken cancellationToken) + private static async Task DispatchAsync(IServiceProvider serviceProvider, IMessageContext message, CancellationToken cancellationToken) where TMessage : class where THandler : class, IMessageHandler { await using var scope = serviceProvider.CreateAsyncScope(); diff --git a/src/Foundatio/Messaging/IReceivedMessage.cs b/src/Foundatio/Messaging/IMessageContext.cs similarity index 97% rename from src/Foundatio/Messaging/IReceivedMessage.cs rename to src/Foundatio/Messaging/IMessageContext.cs index 82d453c62..4652f5550 100644 --- a/src/Foundatio/Messaging/IReceivedMessage.cs +++ b/src/Foundatio/Messaging/IMessageContext.cs @@ -75,7 +75,7 @@ public sealed record RejectOptions public TimeSpan? RedeliveryDelay { get; init; } } -public interface IReceivedMessage +public interface IMessageContext { string Id { get; } ReadOnlyMemory Body { get; } @@ -91,7 +91,7 @@ public interface IReceivedMessage Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancellationToken = default); } -public interface IReceivedMessage : IReceivedMessage where T : class +public interface IMessageContext : IMessageContext where T : class { T Message { get; } } diff --git a/src/Foundatio/Messaging/IMessageHandler.cs b/src/Foundatio/Messaging/IMessageHandler.cs index 3117c0699..196095076 100644 --- a/src/Foundatio/Messaging/IMessageHandler.cs +++ b/src/Foundatio/Messaging/IMessageHandler.cs @@ -14,5 +14,5 @@ namespace Foundatio.Messaging; /// public interface IMessageHandler where T : class { - Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken); + Task HandleAsync(IMessageContext context, CancellationToken cancellationToken); } diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index efcb71156..abde727b2 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -156,12 +156,12 @@ public interface IMessageBus : IAsyncDisposable /// registration (AddFoundatio().Messaging.AddHandler<T, THandler>()) for handlers that live for the /// app's lifetime; use this for dynamic subscriptions. /// - Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default); + Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default); /// Pulls one sent message of type , or null when none arrives within the wait window. - Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default); + Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; + Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default); } public sealed record MessageBusOptions @@ -241,7 +241,7 @@ public Task PublishBatchAsync(IEnumerable messages, MessagePublishOption return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } - public async Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + public async Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); var channels = BuildChannels(options, typeof(T)); @@ -259,7 +259,7 @@ public async Task SubscribeAsync(Func SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) + public async Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(handler); var channels = BuildChannels(options, typeof(object)); @@ -277,13 +277,13 @@ public async Task SubscribeAsync(Func?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class { options ??= new MessageReceiveOptions(); return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(T), options.Source), options.MaxWaitTime, cancellationToken); } - public Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) + public Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) { options ??= new MessageReceiveOptions(); return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(object), options.Source), options.MaxWaitTime, cancellationToken); diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index faba42fad..094a7c85d 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -168,38 +168,38 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable } } - public async Task ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) + public async Task ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) { ThrowIfDisposed(); var pull = RequirePull(); var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); - return entries.Count == 0 ? null : CreateReceivedMessage(entries[0], cancellationToken); + return entries.Count == 0 ? null : CreateMessageContext(entries[0], cancellationToken); } - public async Task?> ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) where T : class + public async Task?> ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) where T : class { ThrowIfDisposed(); var pull = RequirePull(); var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); - return entries.Count == 0 ? null : await CreateReceivedMessageAsync(entries[0], cancellationToken).AnyContext(); + return entries.Count == 0 ? null : await CreateMessageContextAsync(entries[0], cancellationToken).AnyContext(); } - public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) + public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(handler); return RegisterConsumerAsync(config, handler, async (entry, token) => { - var received = CreateReceivedMessage(entry, token); + var received = CreateMessageContext(entry, token); await HandleMessageAsync(received, config, handler, token).AnyContext(); }, cancellationToken); } - public Task StartListenerAsync(ListenerConfig config, Func, CancellationToken, Task> handler, CancellationToken cancellationToken) where T : class + public Task StartListenerAsync(ListenerConfig config, Func, CancellationToken, Task> handler, CancellationToken cancellationToken) where T : class { ArgumentNullException.ThrowIfNull(handler); return RegisterConsumerAsync(config, handler, async (entry, token) => { - var received = await CreateReceivedMessageAsync(entry, token).AnyContext(); + var received = await CreateMessageContextAsync(entry, token).AnyContext(); await HandleMessageAsync(received, config, handler, token).AnyContext(); }, cancellationToken); } @@ -275,7 +275,7 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, string source, Can { MessagingInstruments.Unhandled.Add(1, new KeyValuePair("source", source)); - var message = CreateReceivedMessage(entry, cancellationToken); + var message = CreateMessageContext(entry, cancellationToken); // Retry so a node that does handle this type can pick it up; dead-letter as "no-handler" once the lenient // budget is exhausted so a genuinely orphaned type cannot loop forever. @@ -400,7 +400,7 @@ private async Task SafeProcessAsync(TransportEntry entry, Func(TMessage message, ListenerConfig config, Func handler, CancellationToken cancellationToken) where TMessage : IReceivedMessage + private async Task HandleMessageAsync(TMessage message, ListenerConfig config, Func handler, CancellationToken cancellationToken) where TMessage : IMessageContext { // Re-establish the producer's trace context on the consumer side so a cross-process trace continues here // instead of breaking at the transport boundary. @@ -427,7 +427,7 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig } } - private static Activity? StartProcessActivity(IReceivedMessage message, ListenerConfig config) + private static Activity? StartProcessActivity(IMessageContext message, ListenerConfig config) { string? traceParent = message.Headers.GetValueOrDefault(KnownHeaders.TraceParent); var activity = FoundatioDiagnostics.ActivitySource.StartActivity("ProcessMessage", ActivityKind.Consumer, traceParent); @@ -449,7 +449,7 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig return activity; } - private static Task SettleFailedMessageAsync(IReceivedMessage message, int maxAttempts, Func? backoff, string deadLetterReason, CancellationToken cancellationToken) + private static Task SettleFailedMessageAsync(IMessageContext message, int maxAttempts, Func? backoff, string deadLetterReason, CancellationToken cancellationToken) { if (message.IsHandled) return Task.CompletedTask; @@ -460,13 +460,13 @@ private static Task SettleFailedMessageAsync(IReceivedMessage message, int maxAt return message.RejectAsync(new RejectOptions { RedeliveryDelay = backoff?.Invoke(message.Attempts) }, cancellationToken); } - private ReceivedMessage CreateReceivedMessage(TransportEntry entry, CancellationToken cancellationToken) + private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken) { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); - return new ReceivedMessage(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); + return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); } - private async Task> CreateReceivedMessageAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class + private async Task> CreateMessageContextAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); @@ -504,13 +504,13 @@ private async Task> CreateReceivedMessageAsync(TransportE throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); } - return new ReceivedMessage(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); + return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); } private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) { MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination)); - return ReceivedMessage.DeadLetterOrDropAsync(_transport, entry, reason, _retryPolicy.DeadLetterDestination, cancellationToken); + return MessageContext.DeadLetterOrDropAsync(_transport, entry, reason, _retryPolicy.DeadLetterDestination, cancellationToken); } @@ -899,7 +899,7 @@ public void Remove(ConsumerRegistration registration) } } -internal class ReceivedMessage : IReceivedMessage +internal class MessageContext : IMessageContext { private readonly IMessageTransport _transport; private readonly TransportEntry _entry; @@ -908,7 +908,7 @@ internal class ReceivedMessage : IReceivedMessage private readonly string? _deadLetterDestination; private int _isHandled; - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) + public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) { _transport = transport; _entry = entry; @@ -1044,9 +1044,9 @@ private static int ParseAttemptsHeader(MessageHeaders headers) } } -internal sealed class ReceivedMessage : ReceivedMessage, IReceivedMessage where T : class +internal sealed class MessageContext : MessageContext, IMessageContext where T : class { - public ReceivedMessage(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) + public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination) { Message = message; diff --git a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs index 48e62dd91..ab20b430e 100644 --- a/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs +++ b/tests/Foundatio.Tests/DeclarativeRegistrationTests.cs @@ -26,7 +26,7 @@ public async Task AddHandler_SendGoesToOneHandlerAndPublishReachesSubscriptionAs services.AddFoundatio() .Messaging.UseInMemory() .Messaging.AddHandler() // class handler - .Messaging.AddHandler((message, _) => { probe.Record($"task:{message.Message.Id}"); return Task.CompletedTask; }); // delegate handler + .Messaging.AddHandler((context, _) => { probe.Record($"task:{context.Message.Id}"); return Task.CompletedTask; }); // delegate handler await using var provider = services.BuildServiceProvider(); var hosted = provider.GetServices().ToList(); @@ -248,36 +248,36 @@ public class HandledBroadcast { public string Id { get; set; } = ""; } private sealed class OrderHandler(HandlerProbe probe) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - probe.Record($"order:{message.Message.Id}"); + probe.Record($"order:{context.Message.Id}"); return Task.CompletedTask; } } private sealed class EventHandler(HandlerProbe probe) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - probe.Record($"event:{message.Message.Id}"); + probe.Record($"event:{context.Message.Id}"); return Task.CompletedTask; } } private sealed class SecondEventHandler(HandlerProbe probe) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - probe.Record($"second:{message.Message.Id}"); + probe.Record($"second:{context.Message.Id}"); return Task.CompletedTask; } } private sealed class BroadcastHandler(HandlerProbe probe) : IMessageHandler { - public Task HandleAsync(IReceivedMessage message, CancellationToken cancellationToken) + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) { - probe.Record($"broadcast:{message.Message.Id}"); + probe.Record($"broadcast:{context.Message.Id}"); return Task.CompletedTask; } } diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 709a9f423..ab54d5bd7 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -61,7 +61,7 @@ public async Task SubscribeAsync_WithSameSubscriptionAndDifferentKeys_CompetesOn var received = new AsyncCountdownEvent(2); var deliveriesByMessageId = new ConcurrentDictionary(StringComparer.Ordinal); - Func, CancellationToken, Task> handler = (message, _) => + Func, CancellationToken, Task> handler = (message, _) => { deliveriesByMessageId.AddOrUpdate(message.Id, 1, (_, count) => count + 1); received.Signal(); @@ -178,7 +178,7 @@ public async Task PublishAsync_WithOptions_PropagatesHeadersAsync() await using var pubSub = new MessageBus(new InMemoryMessageTransport()); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(TimeSpan.FromSeconds(10)); - var received = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var received = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); await using var subscription = await pubSub.SubscribeAsync((message, _) => { @@ -272,7 +272,7 @@ public async Task SubscribeAsync_WithSameKeyAndSameRegistration_SharesTheUnderly await using var pubSub = new MessageBus(new InMemoryMessageTransport()); int handled = 0; var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Func, CancellationToken, Task> handler = (_, _) => + Func, CancellationToken, Task> handler = (_, _) => { Interlocked.Increment(ref handled); received.TrySetResult(); diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index c7b55e9e8..b9db9dbd7 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -503,7 +503,7 @@ public async Task StartConsumerAsync_WithSameKeyAndSameRegistration_SharesTheUnd await using var queue = new MessageBus(new InMemoryMessageTransport()); int handled = 0; var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Func, CancellationToken, Task> handler = (_, _) => + Func, CancellationToken, Task> handler = (_, _) => { Interlocked.Increment(ref handled); received.TrySetResult(); From 38421c69b23beb516709146617e697203ec0871d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 1 Jul 2026 23:55:31 -0500 Subject: [PATCH 41/57] Remove pull receive from IMessageBus; harden the pull loop against sync-empty transports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bus contract is now exactly three ideas: Send, Publish, Subscribe. ReceiveAsync/MessageReceiveOptions were queue-residue no bus-level abstraction in the ecosystem carries (MassTransit/NServiceBus/Wolverine/Rebus are all handler-based) — and they were asymmetric, pulling only the send channel. Advanced pull consumption remains where it belongs, on the transport (IMessageTransport / ISupportsPull). Converting the pull-based tests to subscriptions exposed a real core hole: the pull loop trusted transports to honor MaxWaitTime, so a transport whose ReceiveAsync completes synchronously-empty ran the loop inline forever (the subscribe call never returned) or hot-spun. The loop now starts via Task.Run and sleeps out the remainder of the poll window after an early empty poll. Tests move to a MessageCollector helper over SubscribeAsync with manual ack (inspect + settle deliveries explicitly); the pull-only poison-payload test is deleted (the push-path equivalent already covers poison dead-lettering). Full solution builds; in-memory 2005 green; live Redis 29 green. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio/Messaging/MessageBus.cs | 24 --- src/Foundatio/Messaging/MessageClientCore.cs | 33 ++-- .../RedisJobStoreIntegrationTests.cs | 10 +- .../Queue/MessageQueueTests.cs | 173 +++++++++++++----- 4 files changed, 148 insertions(+), 92 deletions(-) diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index abde727b2..456ea36ac 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -37,14 +37,6 @@ public sealed record MessagePublishOptions public MessageHeaders? Headers { get; init; } } -public sealed record MessageReceiveOptions -{ - /// Overrides the source to pull from; defaults to the message type's send destination. - public string? Source { get; init; } - public Type? RouteType { get; init; } - public TimeSpan? MaxWaitTime { get; init; } = TimeSpan.FromSeconds(30); -} - /// /// Options for attaching a handler to a message type — via AddFoundatio().Messaging.AddHandler<T, THandler>(o => ...) /// or programmatically via . A subscription listens on the type's two @@ -158,10 +150,6 @@ public interface IMessageBus : IAsyncDisposable /// Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class; Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default); - - /// Pulls one sent message of type , or null when none arrives within the wait window. - Task?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class; - Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default); } public sealed record MessageBusOptions @@ -277,18 +265,6 @@ public async Task SubscribeAsync(Func?> ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) where T : class - { - options ??= new MessageReceiveOptions(); - return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(T), options.Source), options.MaxWaitTime, cancellationToken); - } - - public Task ReceiveAsync(MessageReceiveOptions? options = null, CancellationToken cancellationToken = default) - { - options ??= new MessageReceiveOptions(); - return _core.ReceiveAsync(GetDestination(options.RouteType ?? typeof(object), options.Source), options.MaxWaitTime, cancellationToken); - } - public ValueTask DisposeAsync() { return _core.DisposeAsync(); diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 094a7c85d..8dc31acbe 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -168,22 +168,6 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable } } - public async Task ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - var pull = RequirePull(); - var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); - return entries.Count == 0 ? null : CreateMessageContext(entries[0], cancellationToken); - } - - public async Task?> ReceiveAsync(string source, TimeSpan? maxWaitTime, CancellationToken cancellationToken) where T : class - { - ThrowIfDisposed(); - var pull = RequirePull(); - var entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = 1, MaxWaitTime = maxWaitTime }, cancellationToken).AnyContext(); - return entries.Count == 0 ? null : await CreateMessageContextAsync(entries[0], cancellationToken).AnyContext(); - } - public Task StartListenerAsync(ListenerConfig config, Func handler, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(handler); @@ -317,13 +301,15 @@ private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func entries; try { entries = await pull.ReceiveAsync(source, new ReceiveRequest { MaxMessages = claimed, - MaxWaitTime = TimeSpan.FromSeconds(1) + MaxWaitTime = pollWindow }, cancellationToken).AnyContext(); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -346,6 +332,15 @@ private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func= 0) + // An empty poll should have blocked for MaxWaitTime; a transport that returns empty early (or + // synchronously) would otherwise hot-spin this loop, so sleep out the remainder of the window. + if (toProcess == 0) + { + var remaining = pollWindow - _timeProvider.GetElapsedTime(pollStart); + if (remaining > TimeSpan.Zero) + await _timeProvider.SafeDelay(remaining, cancellationToken).AnyContext(); + } + for (int index = 0; index < toProcess; index++) { var task = ProcessAndReleaseSlotAsync(entries[index], onMessage, source, slots, cancellationToken); @@ -763,7 +758,9 @@ public async Task StartAsync(CancellationToken cancellationToken) if (_core._transport is not ISupportsPull pull) throw _core._exceptionFactory($"Transport \"{_core._transport.GetType().Name}\" does not support receiving messages.", null); - _loop = _core.RunPullLoopAsync(_source, pull, DispatchAsync, _maxConcurrency, _cancellationTokenSource.Token); + // Task.Run so a transport whose ReceiveAsync completes synchronously can never run the loop inline on the + // caller's thread and block the subscribe call from returning. + _loop = Task.Run(() => _core.RunPullLoopAsync(_source, pull, DispatchAsync, _maxConcurrency, _cancellationTokenSource.Token), CancellationToken.None); } public async ValueTask DisposeAsync() diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index 831275682..6b5166c5d 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -60,8 +60,14 @@ public async Task DelayedQueueSend_BeyondTransportLimit_StoresInRedisAndDrainsWh Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(now.AddHours(2), cancellationToken: cancellationToken)); Assert.Equal(1, fallbackTransport.SendCount); - var delivered = await fallbackQueue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); - Assert.NotNull(delivered); + var deliveredContext = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + await using var subscription = await fallbackQueue.SubscribeAsync((context, _) => + { + deliveredContext.TrySetResult(context); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { AckMode = AckMode.Manual }, cancellationToken); + + var delivered = await deliveredContext.Task.WaitAsync(TimeSpan.FromSeconds(10), cancellationToken); Assert.Equal("later", delivered.Message.Data); await delivered.CompleteAsync(cancellationToken); } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index b9db9dbd7..354b36c63 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Threading.Channels; using System.Threading; using System.Threading.Tasks; using Foundatio; @@ -31,7 +32,8 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() ]) }, cancellationToken); - var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var received = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(received); Assert.Equal(id, received.Id); @@ -61,8 +63,9 @@ await queue.SendBatchAsync([ new PreviewWorkItem { Data = "two" } ], new MessageSendOptions { Destination = "custom-work" }, cancellationToken); - var first = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - var second = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "custom-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, destination: "custom-work", cancellationToken: cancellationToken); + var first = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); + var second = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(first); Assert.NotNull(second); @@ -80,12 +83,13 @@ public async Task RejectAsync_NonTerminal_RedeliversAsync() await using var queue = new MessageBus(new InMemoryMessageTransport()); await queue.SendAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cancellationToken); - var first = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var first = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(first); await first.RejectAsync(cancellationToken: cancellationToken); - var second = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var second = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(second); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.Attempts); @@ -103,7 +107,8 @@ public async Task RenewLockAsync_WhenUnsupported_ThrowsAsync() await using var queue = new MessageBus(new BasicQueueTransport()); await queue.SendAsync(new PreviewWorkItem { Data = "lock" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var message = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(message); await Assert.ThrowsAsync(async () => await message.RenewLockAsync(cancellationToken: cancellationToken)); @@ -117,7 +122,8 @@ public async Task RejectAsync_Terminal_DeadLettersAsync() await using var queue = new MessageBus(transport); await queue.SendAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var message = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(message); await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); @@ -234,12 +240,13 @@ public async Task EnqueueAsync_WithDelay_SchedulesThroughRuntimeStoreAsync() await queue.SendAsync(new PreviewWorkItem { Data = "later" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(1) }, cancellationToken); - var immediate = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); - Assert.Null(immediate); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var immediate = await collector.NextAsync(TimeSpan.FromMilliseconds(250), cancellationToken); + Assert.Null(immediate); // parked in the runtime store, not on the transport Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); - var delayed = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var delayed = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); @@ -284,7 +291,8 @@ public async Task EnqueueAsync_WithDelay_RespectsTransportMaxDeliveryDelayAsync( Assert.Equal(1, await fallbackProcessor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddHours(2), cancellationToken: cancellationToken)); Assert.Equal(1, fallbackTransport.SendCount); - var delayed = await fallbackQueue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(fallbackQueue, cancellationToken: cancellationToken); + var delayed = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(delayed); Assert.Equal("later", delayed.Message.Data); await delayed.CompleteAsync(cancellationToken); @@ -326,8 +334,10 @@ public async Task StartConsumerAsync_WithRedeliveryBackoff_SchedulesRetryThrough await queue.SendAsync(new PreviewWorkItem { Data = "retry" }, cancellationToken: cts.Token); await firstAttempt.WaitAsync(TimeSpan.FromSeconds(2)); - var immediate = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); - Assert.Null(immediate); + // The retry is parked in the runtime store: the still-attached consumer must NOT get a second attempt until + // the dispatch pump drains the store. + await Task.Delay(250, cancellationToken); + Assert.Equal(1, attempts); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(2), cancellationToken: cancellationToken)); await secondAttempt.WaitAsync(TimeSpan.FromSeconds(2)); @@ -362,10 +372,11 @@ public async Task RejectAsync_RuntimeStoreRedelivery_AdvancesAttemptCountEachCyc var now = DateTimeOffset.UtcNow; await queue.SendAsync(new PreviewWorkItem { Data = "loop" }, cancellationToken: cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); for (int expectedAttempt = 1; expectedAttempt <= 3; expectedAttempt++) { - var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + var received = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(received); Assert.Equal(expectedAttempt, received.Attempts); Assert.Equal("loop", received.Message.Data); @@ -383,7 +394,7 @@ public async Task RejectAsync_RuntimeStoreRedelivery_AdvancesAttemptCountEachCyc } [Fact] - public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync() + public async Task SendAsync_WithExpiredMessage_IsDeadLetteredNotDeliveredAsync() { var cancellationToken = TestContext.Current.CancellationToken; await using var transport = new InMemoryMessageTransport(); @@ -391,38 +402,14 @@ public async Task ReceiveAsync_WithExpiredMessage_DeadLettersAndReturnsNullAsync await queue.SendAsync(new PreviewWorkItem { Data = "expired" }, new MessageSendOptions { TimeToLive = TimeSpan.FromMilliseconds(-1) }, cancellationToken); - var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var received = await collector.NextAsync(TimeSpan.FromMilliseconds(500), cancellationToken); Assert.Null(received); var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); Assert.Equal(1, stats.Deadletter); } - [Fact] - public async Task ReceiveAsync_WithPoisonPayload_DeadLettersAndThrowsMessageQueueExceptionAsync() - { - var cancellationToken = TestContext.Current.CancellationToken; - await using var transport = new InMemoryMessageTransport(); - await using var queue = new MessageBus(transport); - - await transport.SendAsync("preview-work-item", [ - new TransportMessage - { - Body = "not-json"u8.ToArray(), - Headers = MessageHeaders.Create([ - new KeyValuePair(KnownHeaders.MessageType, typeof(PreviewWorkItem).FullName!) - ]) - } - ], new TransportSendOptions(), cancellationToken); - - await Assert.ThrowsAsync(async () => - await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken)); - - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); - Assert.Equal(1, stats.Deadletter); - Assert.Equal(0, stats.Working); - } - [Fact] public async Task AddFoundatio_WithInMemoryMessagingAndJobs_RegistersAppFacingServices() @@ -487,10 +474,12 @@ public async Task EnqueueAsync_WithRouteAttribute_UsesAttributedDestinationAsync var cancellationToken = TestContext.Current.CancellationToken; await using var queue = new MessageBus(new InMemoryMessageTransport()); - await queue.SendAsync(new RoutedWorkItem { Data = "route" }, cancellationToken: cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + Assert.Equal("routed-work", collector.Destination); // the [MessageRoute] attribute names the send destination - var received = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "routed-work", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await queue.SendAsync(new RoutedWorkItem { Data = "route" }, cancellationToken: cancellationToken); + var received = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(received); Assert.Equal("route", received.Message.Data); await received.CompleteAsync(cancellationToken); @@ -551,8 +540,9 @@ await queue.SendBatchAsync(new object[] new OtherWorkItem { Data = "two" } }, cancellationToken: cancellationToken); - var first = await queue.ReceiveAsync(new MessageReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); - var second = await queue.ReceiveAsync(new MessageReceiveOptions { RouteType = typeof(IGroupedWorkItem), MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, routeType: typeof(IGroupedWorkItem), cancellationToken: cancellationToken); + var first = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); + var second = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(first); Assert.NotNull(second); @@ -616,7 +606,8 @@ public async Task ReceiveAsync_WithDefaultQueueRoute_ReturnsRawMessageAsync() await queue.SendAsync(new PreviewWorkItem { Data = "global" }, cancellationToken: cancellationToken); - var received = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var received = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(received); Assert.Equal(typeof(PreviewWorkItem).FullName, received.MessageType); @@ -640,6 +631,90 @@ private static async Task WaitForCompletedAsync(InMemoryMessageTransport transpo Assert.Equal(1, finalStats.Completed); } + // Pull-style test helper over the subscription API: collects manually-acked contexts so tests can inspect and + // settle deliveries explicitly, now that the bus surface is subscription-only. + private sealed class MessageCollector : IAsyncDisposable where T : class + { + private readonly Channel> _received = Channel.CreateUnbounded>(); + private IMessageSubscription _subscription = null!; + + public static async Task> StartAsync(IMessageBus bus, string? destination = null, CancellationToken cancellationToken = default) + { + var collector = new MessageCollector(); + collector._subscription = await bus.SubscribeAsync((context, _) => + { + collector._received.Writer.TryWrite(context); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { AckMode = AckMode.Manual, Destination = destination }, cancellationToken); + return collector; + } + + public string Destination => _subscription.Destination; + + public async Task?> NextAsync(TimeSpan maxWait, CancellationToken cancellationToken = default) + { + // WaitToReadAsync + TryRead (not ReadAsync + WaitAsync): a timed-out WaitAsync abandons its ReadAsync, + // which would silently consume the next item. + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(maxWait); + try + { + while (await _received.Reader.WaitToReadAsync(cts.Token)) + { + if (_received.Reader.TryRead(out var context)) + return context; + } + + return null; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return null; + } + } + + public ValueTask DisposeAsync() => _subscription.DisposeAsync(); + } + + private sealed class MessageCollector : IAsyncDisposable + { + private readonly Channel _received = Channel.CreateUnbounded(); + private IMessageSubscription _subscription = null!; + + public static async Task StartAsync(IMessageBus bus, Type? routeType = null, string? destination = null, CancellationToken cancellationToken = default) + { + var collector = new MessageCollector(); + collector._subscription = await bus.SubscribeAsync((context, _) => + { + collector._received.Writer.TryWrite(context); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { AckMode = AckMode.Manual, RouteType = routeType, Destination = destination }, cancellationToken); + return collector; + } + + public async Task NextAsync(TimeSpan maxWait, CancellationToken cancellationToken = default) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(maxWait); + try + { + while (await _received.Reader.WaitToReadAsync(cts.Token)) + { + if (_received.Reader.TryRead(out var context)) + return context; + } + + return null; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return null; + } + } + + public ValueTask DisposeAsync() => _subscription.DisposeAsync(); + } + [Fact] public async Task StartConsumerAsync_MultipleTypesOnOneDestination_DispatchByTypeAsync() { @@ -724,13 +799,15 @@ public async Task RejectAsync_Terminal_WithoutNativeDeadLetter_SendsToConfigured await using var queue = new MessageBus(transport, new MessageBusOptions { RetryPolicy = new RetryPolicy { DeadLetterDestination = "preview-dead-letter" } }); await queue.SendAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); - var message = await queue.ReceiveAsync(new MessageReceiveOptions { MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var message = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(message); await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); // The transport has no native dead-letter sink, so core routes the terminal message to the configured destination. - var dead = await queue.ReceiveAsync(new MessageReceiveOptions { Source = "preview-dead-letter", MaxWaitTime = TimeSpan.FromSeconds(1) }, cancellationToken); + await using var deadCollector = await MessageCollector.StartAsync(queue, destination: "preview-dead-letter", cancellationToken: cancellationToken); + var dead = await deadCollector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); Assert.NotNull(dead); Assert.Equal("validation", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterReason)); } From 4c0726ebcea95790703e6de3147cf9fa8778197e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 2 Jul 2026 00:49:35 -0500 Subject: [PATCH 42/57] Failure-path conventions: DeadLetterOn, proven retry defaults, DLQ forensics, topology logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the messaging-conventions research (the strongest cross-library convergence points), all convention-over-config: better behavior with zero new required setup. Fail-fast exception classification — the escape hatch every mature stack independently grew (NServiceBus AddUnrecoverableException, MassTransit Ignore, Rebus IFailFastException, Wolverine's error DSL) distilled to one knob: MessageSubscriptionOptions.DeadLetterOn() / DeadLetterWhen predicate (per-subscription, overriding a RetryPolicy.DeadLetterWhen default). A matching failure dead-letters on attempt 1 with reason "unrecoverable:{ExceptionType}" instead of burning the whole attempt budget. Deserialization failures already never retried; that stays structural. Production-proven retry defaults: RetryPolicy.Backoff now defaults to DefaultBackoff — immediate first retry, then 10s/20s/30s (capped) with ±20% jitter, the curve mature stacks converged on. Policy-driven delays are best-effort (RejectOptions.BestEffortDelay): a transport that can't honor the delay redelivers immediately instead of failing the settle; explicit caller delays stay strict. MaxConcurrency=1 keeps its value but now documents why it deliberately diverges from libraries that default higher. Dead-letter forensics: terminal messages are stamped with a documented header contract (exception type/message/truncated stack, reconciled attempts, original destination, failed-at) on both native and fallback sinks, so a dead message is triageable with plain transport tooling. When a transport has no native sink and no destination is configured, the message now parks at the derived "{source}.deadletter" instead of being silently dropped. Failed attempts that will retry log WARN; the terminal decision logs ERROR. The in-memory transport's native dead-letter now honors the caller's (enriched) entry headers, matching Redis. Delivery semantics are never invisible: each subscription logs its effective topology (send destination, topic/subscriber group, concurrency, retry posture) at Info when it starts. Tests cover DeadLetterOn first-attempt dead-lettering, the global predicate, the forensics contract, the default backoff curve, derived-DLQ parking, and best-effort delay degradation. Full solution builds; in-memory 2011 green; live Redis 29 green. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio/Messaging/IMessageContext.cs | 51 +++++++- .../Messaging/InMemoryMessageTransport.cs | 4 +- src/Foundatio/Messaging/KnownHeaders.cs | 8 ++ src/Foundatio/Messaging/MessageBus.cs | 51 +++++++- src/Foundatio/Messaging/MessageClientCore.cs | 108 ++++++++++++---- .../Messaging/FailureHandlingTests.cs | 118 ++++++++++++++++++ .../Queue/MessageQueueTests.cs | 57 +++++++++ 7 files changed, 358 insertions(+), 39 deletions(-) create mode 100644 tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs diff --git a/src/Foundatio/Messaging/IMessageContext.cs b/src/Foundatio/Messaging/IMessageContext.cs index 4652f5550..4f3fee070 100644 --- a/src/Foundatio/Messaging/IMessageContext.cs +++ b/src/Foundatio/Messaging/IMessageContext.cs @@ -21,15 +21,41 @@ public sealed record RetryPolicy /// Maximum delivery attempts for a failing handler before the message is dead-lettered. Default 5. public int MaxAttempts { get; init; } = 5; - /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's own redelivery timing. - public Func? Backoff { get; init; } + /// + /// Delay before each redelivery given the 1-based attempt number. Defaults to + /// (immediate first retry, then 10s/20s/30s with jitter). Set null to defer to the transport's own redelivery timing. + /// + public Func? Backoff { get; init; } = DefaultBackoff; + + /// + /// Marks a handler failure as unrecoverable: when the predicate returns true the message is dead-lettered + /// immediately instead of retried (a poison message should not burn its attempt budget). Deserialization failures + /// are always unrecoverable regardless of this predicate. A subscription's + /// overrides this default. + /// + public Func? DeadLetterWhen { get; init; } /// /// Destination terminal (dead-lettered) messages are sent to when the transport has no native dead-letter sink. - /// Null drops terminal messages on such transports. Ignored when the transport supports native dead-lettering. + /// Null (default) derives "{source}.deadletter" per source. Ignored when the transport supports native dead-lettering. /// public string? DeadLetterDestination { get; init; } + /// + /// The default redelivery curve: an immediate first retry, then 10s/20s/30s (capped) with ±20% jitter — the delay + /// shape mature messaging stacks converged on. The attempt number is 1-based: the value is the delay applied after + /// that attempt failed. + /// + public static readonly Func DefaultBackoff = attempt => + { + if (attempt <= 1) + return TimeSpan.Zero; + + double seconds = Math.Min((attempt - 1) * 10, 30); + double jitter = 1 + (Random.Shared.NextDouble() * 0.4 - 0.2); + return TimeSpan.FromSeconds(seconds * jitter); + }; + /// Maximum attempts for a message whose type has no registered consumer before it is dead-lettered as "no-handler". Default 50. public int UnmatchedMaxAttempts { get; init; } = 50; @@ -59,20 +85,33 @@ public sealed record RejectOptions { /// /// When false (default) the message is returned for redelivery (a retry). When true the message is terminal: it - /// is moved to the transport's dead-letter sink where one exists, otherwise dropped. Terminal messages are never - /// redelivered. + /// is moved to the transport's native dead-letter sink where one exists, otherwise sent to the configured or + /// derived ("{source}.deadletter") dead-letter destination. Terminal messages are never redelivered. /// public bool Terminal { get; init; } - /// Reason carried to the dead-letter sink (where the transport supports one) for a terminal reject. + /// Reason carried to the dead-letter sink for a terminal reject. public string? Reason { get; init; } + /// + /// The failure behind a terminal reject; its type/message/stack are stamped as forensics headers on the + /// dead-lettered message so a dead message is triageable with plain transport tooling. + /// + public Exception? Exception { get; init; } + /// /// An explicit delay before the message is redelivered. Honored only for a non-terminal reject, served natively /// when the transport supports redelivery delay within its advertised maximum, otherwise through the runtime store. /// When null the transport's own redelivery timing applies. /// public TimeSpan? RedeliveryDelay { get; init; } + + /// + /// When true, a the transport cannot honor (no native support and no runtime store) + /// degrades to immediate redelivery instead of failing. The core's retry policy rejects with best-effort delays; + /// an explicit caller-specified delay defaults to strict. + /// + public bool BestEffortDelay { get; init; } } public interface IMessageContext diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index a2afdfd36..fce129528 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -229,7 +229,9 @@ public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationTo if (!state.InFlight.TryRemove(receipt.LockToken, out var inFlight) || !String.Equals(inFlight.Message.Id, entry.Id, StringComparison.Ordinal)) throw new ReceiptExpiredException(); - DeadLetter(state, inFlight.Message, reason); + // Dead-letter with the caller's entry headers (which may carry forensics stamped by the core), not the + // originally-stored ones. + DeadLetter(state, inFlight.Message with { Headers = entry.Headers }, reason); return Task.CompletedTask; } diff --git a/src/Foundatio/Messaging/KnownHeaders.cs b/src/Foundatio/Messaging/KnownHeaders.cs index d460072ee..0e4780acb 100644 --- a/src/Foundatio/Messaging/KnownHeaders.cs +++ b/src/Foundatio/Messaging/KnownHeaders.cs @@ -11,4 +11,12 @@ public static class KnownHeaders public const string Expiration = "message.expiration"; public const string Attempts = "message.attempts"; public const string DeadLetterReason = "message.dead_letter.reason"; + + // Forensics stamped by the core when a message is dead-lettered, so a dead message is triageable with plain + // transport tooling. These names are a compatibility contract; values are truncated to fit transport limits. + public const string DeadLetterExceptionType = "message.dead_letter.exception_type"; + public const string DeadLetterExceptionMessage = "message.dead_letter.exception_message"; + public const string DeadLetterExceptionStackTrace = "message.dead_letter.exception_stack"; + public const string DeadLetterFailedAt = "message.dead_letter.failed_at"; + public const string DeadLetterOriginalDestination = "message.dead_letter.original_destination"; } diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index 456ea36ac..87fe6d562 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -68,15 +68,27 @@ public sealed class MessageSubscriptionOptions /// public string? SubscriptionQualifier { get; set; } - /// Maximum messages this subscription processes concurrently per instance. Default 1. + /// + /// Maximum messages this subscription processes concurrently per instance. Default 1 — a deliberate divergence + /// from libraries that default higher: 1 is the only default that preserves per-handler ordering, each handler + /// already gets its own concurrent stream (10 handlers = 10 parallel consumers), and scaling out replicas scales + /// throughput without giving up ordering per instance. Raise it for handlers that are I/O-bound and order-agnostic. + /// public int MaxConcurrency { get; set; } = 1; /// Maximum delivery attempts before dead-lettering. Null uses the default . public int? MaxAttempts { get; set; } - /// Delay before each redelivery given the 1-based attempt number. Null defers to the transport's timing. + /// Delay before each redelivery given the 1-based attempt number. Null uses the default . public Func? RedeliveryBackoff { get; set; } + /// + /// Marks a handler failure as unrecoverable: when the predicate returns true the message is dead-lettered + /// immediately instead of retried. Null uses the default . Prefer + /// for the common by-type case. + /// + public Func? DeadLetterWhen { get; set; } + /// Whether messages auto-complete when the handler returns (default) or are settled manually. public AckMode AckMode { get; set; } = AckMode.Auto; @@ -94,6 +106,19 @@ public sealed class MessageSubscriptionOptions /// defaults to a per-channel key derived from the route. /// public string? Key { get; set; } + + /// + /// Dead-letters failures of type immediately instead of retrying — for + /// exceptions a retry can never fix (validation, malformed data). Composes: call once per exception type. + /// + public MessageSubscriptionOptions DeadLetterOn() where TException : Exception + { + var existing = DeadLetterWhen; + DeadLetterWhen = existing is null + ? static ex => ex is TException + : ex => existing(ex) || ex is TException; + return this; + } } /// A started subscription; disposing detaches the handler from the message type's delivery channels. @@ -177,13 +202,14 @@ public sealed record MessageBusOptions public sealed class MessageBus : IMessageBus { private readonly MessageClientCore _core; + private readonly ILogger _logger; public MessageBus(IMessageTransport transport, MessageBusOptions? options = null) { ArgumentNullException.ThrowIfNull(transport); options ??= new MessageBusOptions(); - var logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); - _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, logger, + _logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, _logger, static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); } @@ -238,6 +264,7 @@ public async Task SubscribeAsync(Func SubscribeAsync(Func? RedeliveryBackoff { get; init; } + public Func? DeadLetterWhen { get; init; } } /// @@ -263,7 +264,7 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, string source, Can // Retry so a node that does handle this type can pick it up; dead-letter as "no-handler" once the lenient // budget is exhausted so a genuinely orphaned type cannot loop forever. - await SettleFailedMessageAsync(message, _retryPolicy.UnmatchedMaxAttempts, _retryPolicy.UnmatchedBackoff, deadLetterReason: "no-handler", cancellationToken).AnyContext(); + await SettleFailedMessageAsync(message, unrecoverable: false, _retryPolicy.UnmatchedMaxAttempts, _retryPolicy.UnmatchedBackoff, deadLetterReason: "no-handler", exception: null, cancellationToken).AnyContext(); // Surface loudly. The throw is caught by the loop's per-message handling (SafeProcessAsync), so it never tears // down the receive loop or the other type handlers sharing this source. @@ -413,8 +414,26 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig activity?.SetErrorStatus(ex); int maxAttempts = config.MaxAttempts ?? _retryPolicy.MaxAttempts; var backoff = config.RedeliveryBackoff ?? _retryPolicy.Backoff; - _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}): {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); - await SettleFailedMessageAsync(message, maxAttempts, backoff, "handler-error", cancellationToken).AnyContext(); + + bool unrecoverable = false; + try + { + unrecoverable = (config.DeadLetterWhen ?? _retryPolicy.DeadLetterWhen)?.Invoke(ex) == true; + } + catch (Exception predicateEx) + { + _logger.LogError(predicateEx, "DeadLetterWhen predicate threw for message \"{MessageId}\"; treating the failure as retryable: {Message}", message.Id, predicateEx.Message); + } + + // A retry that can still happen is a warning; the terminal decision (unrecoverable or attempts exhausted) + // is the error worth alerting on. + if (unrecoverable || message.Attempts >= maxAttempts) + _logger.LogError(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}); dead-lettering: {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); + else + _logger.LogWarning(ex, "Handler failed for message \"{MessageId}\" from \"{Source}\" (attempt {Attempt} of {MaxAttempts}); will retry: {Message}", message.Id, config.Source, message.Attempts, maxAttempts, ex.Message); + + string reason = unrecoverable ? $"unrecoverable:{ex.GetType().Name}" : "handler-error"; + await SettleFailedMessageAsync(message, unrecoverable, maxAttempts, backoff, reason, ex, cancellationToken).AnyContext(); } finally { @@ -444,15 +463,17 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig return activity; } - private static Task SettleFailedMessageAsync(IMessageContext message, int maxAttempts, Func? backoff, string deadLetterReason, CancellationToken cancellationToken) + private static Task SettleFailedMessageAsync(IMessageContext message, bool unrecoverable, int maxAttempts, Func? backoff, string deadLetterReason, Exception? exception, CancellationToken cancellationToken) { if (message.IsHandled) return Task.CompletedTask; - if (message.Attempts >= maxAttempts) - return message.RejectAsync(new RejectOptions { Terminal = true, Reason = deadLetterReason }, cancellationToken); + if (unrecoverable || message.Attempts >= maxAttempts) + return message.RejectAsync(new RejectOptions { Terminal = true, Reason = deadLetterReason, Exception = exception }, cancellationToken); - return message.RejectAsync(new RejectOptions { RedeliveryDelay = backoff?.Invoke(message.Attempts) }, cancellationToken); + // Policy-driven delays are best-effort: a transport that can't honor the delay redelivers immediately rather + // than failing the settle (an explicit caller-requested delay stays strict). + return message.RejectAsync(new RejectOptions { RedeliveryDelay = backoff?.Invoke(message.Attempts), BestEffortDelay = true }, cancellationToken); } private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken) @@ -475,7 +496,7 @@ private async Task> CreateMessageContextAsync(TransportEnt var resolved = String.IsNullOrEmpty(typeName) ? null : _typeRegistry.Resolve(typeName); if (resolved is null || !typeof(T).IsAssignableFrom(resolved)) { - await DeadLetterPoisonMessageAsync(entry, "unresolved-type", cancellationToken).AnyContext(); + await DeadLetterPoisonMessageAsync(entry, "unresolved-type", exception: null, cancellationToken).AnyContext(); throw _exceptionFactory($"Unable to resolve a concrete type \"{typeName}\" assignable to \"{typeof(T).Name}\" for message \"{entry.Id}\".", null); } @@ -489,23 +510,24 @@ private async Task> CreateMessageContextAsync(TransportEnt } catch (Exception ex) { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", ex, cancellationToken).AnyContext(); throw _exceptionFactory($"Unable to deserialize message \"{entry.Id}\".", ex); } if (message is null) { - await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", cancellationToken).AnyContext(); + await DeadLetterPoisonMessageAsync(entry, "deserialize-failure", exception: null, cancellationToken).AnyContext(); throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); } return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); } - private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, CancellationToken cancellationToken) + private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, Exception? exception, CancellationToken cancellationToken) { MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination)); - return MessageContext.DeadLetterOrDropAsync(_transport, entry, reason, _retryPolicy.DeadLetterDestination, cancellationToken); + var enriched = entry with { Headers = MessageContext.BuildDeadLetterHeaders(entry, entry.DeliveryCount, exception, _timeProvider) }; + return MessageContext.DeadLetterOrDropAsync(_transport, enriched, reason, _retryPolicy.DeadLetterDestination, cancellationToken); } @@ -950,7 +972,8 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c if (options.Terminal) { MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); - await DeadLetterOrDropAsync(_transport, _entry, options.Reason, _deadLetterDestination, cancellationToken).AnyContext(); + var enriched = _entry with { Headers = BuildDeadLetterHeaders(_entry, Attempts, options.Exception, _timeProvider) }; + await DeadLetterOrDropAsync(_transport, enriched, options.Reason, _deadLetterDestination, cancellationToken).AnyContext(); return; } @@ -972,7 +995,17 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c } if (_runtimeStore is null) + { + // A best-effort delay (the core retry policy) degrades to immediate redelivery; an explicit caller delay + // stays strict because the caller is depending on the timing. + if (options.BestEffortDelay) + { + await _transport.AbandonAsync(_entry, cancellationToken).AnyContext(); + return; + } + throw new MessageBusException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); + } // Advance from the reconciled attempt count, not the raw transport DeliveryCount: the re-send produces a new // transport message whose native DeliveryCount resets to 1, so basing the next attempt on DeliveryCount would @@ -1004,9 +1037,9 @@ public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancella } // Terminal settlement. Prefer the transport's native dead-letter sink (preserves native DLQ tooling). When the - // transport has none, fall back to a configured core-managed dead-letter destination: copy the raw entry there - // (recording the reason) and complete the original. With neither, the message can't be parked, so it is completed - // (dropped) rather than throwing and stalling the consumer. + // transport has none, copy the raw entry to the configured dead-letter destination — or the derived + // "{source}.deadletter" when none is configured, so a dead message is always parked somewhere inspectable — + // recording the reason, then complete the original. internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, TransportEntry entry, string? reason, string? deadLetterDestination, CancellationToken cancellationToken) { if (transport is ISupportsDeadLetter deadLetter) @@ -1015,17 +1048,37 @@ internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, Tr return; } - if (!String.IsNullOrEmpty(deadLetterDestination)) + string destination = !String.IsNullOrEmpty(deadLetterDestination) ? deadLetterDestination : $"{entry.Destination}.deadletter"; + var headers = String.IsNullOrEmpty(reason) + ? entry.Headers + : entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); + await transport.SendAsync(destination, [new TransportMessage { Body = entry.Body, Headers = headers, MessageId = entry.Id }], new TransportSendOptions(), cancellationToken).AnyContext(); + await transport.CompleteAsync(entry, cancellationToken).AnyContext(); + } + + // Stamps the dead-letter forensics contract (see KnownHeaders) so a dead message is triageable — exception details, + // reconciled attempt count, where it was consumed from, and when it died. + internal static MessageHeaders BuildDeadLetterHeaders(TransportEntry entry, int attempts, Exception? exception, TimeProvider timeProvider) + { + var headers = entry.Headers.ToBuilder() + .Set(KnownHeaders.Attempts, attempts.ToString(CultureInfo.InvariantCulture)) + .Set(KnownHeaders.DeadLetterFailedAt, timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture)) + .Set(KnownHeaders.DeadLetterOriginalDestination, entry.Destination); + + if (exception is not null) { - var headers = String.IsNullOrEmpty(reason) - ? entry.Headers - : entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); - await transport.SendAsync(deadLetterDestination, [new TransportMessage { Body = entry.Body, Headers = headers, MessageId = entry.Id }], new TransportSendOptions(), cancellationToken).AnyContext(); - await transport.CompleteAsync(entry, cancellationToken).AnyContext(); - return; + headers.Set(KnownHeaders.DeadLetterExceptionType, exception.GetType().FullName ?? exception.GetType().Name); + headers.Set(KnownHeaders.DeadLetterExceptionMessage, Truncate(exception.Message, 1024)); + if (exception.StackTrace is { } stack) + headers.Set(KnownHeaders.DeadLetterExceptionStackTrace, Truncate(stack, 4096)); } - await transport.CompleteAsync(entry, cancellationToken).AnyContext(); + return headers.Build(); + } + + private static string Truncate(string value, int maxLength) + { + return value.Length <= maxLength ? value : value[..maxLength]; } private bool TryMarkHandled() @@ -1124,6 +1177,7 @@ internal sealed record MessageListenerRegistration public required int MaxConcurrency { get; init; } public required int? MaxAttempts { get; init; } public required bool HasRedeliveryBackoff { get; init; } + public required bool HasDeadLetterWhen { get; init; } public static MessageListenerRegistration Create(Delegate handler, ListenerConfig config) { @@ -1135,7 +1189,8 @@ public static MessageListenerRegistration Create(Delegate handler, ListenerConfi AckMode = config.AckMode, MaxConcurrency = Math.Max(1, config.MaxConcurrency), MaxAttempts = config.MaxAttempts, - HasRedeliveryBackoff = config.RedeliveryBackoff is not null + HasRedeliveryBackoff = config.RedeliveryBackoff is not null, + HasDeadLetterWhen = config.DeadLetterWhen is not null }; } @@ -1147,6 +1202,7 @@ public bool Matches(MessageListenerRegistration other) && AckMode == other.AckMode && MaxConcurrency == other.MaxConcurrency && MaxAttempts == other.MaxAttempts - && HasRedeliveryBackoff == other.HasRedeliveryBackoff; + && HasRedeliveryBackoff == other.HasRedeliveryBackoff + && HasDeadLetterWhen == other.HasDeadLetterWhen; } } diff --git a/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs new file mode 100644 index 000000000..837f05fe5 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs @@ -0,0 +1,118 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class FailureHandlingTests +{ + [Fact] + public async Task DeadLetterOn_MatchingException_DeadLettersOnFirstAttemptAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + int attempts = 0; + + await using var subscription = await bus.SubscribeAsync((_, _) => + { + Interlocked.Increment(ref attempts); + throw new ArgumentException("bad data a retry can never fix"); + }, new MessageSubscriptionOptions { MaxAttempts = 5 }.DeadLetterOn(), cancellationToken); + + await bus.SendAsync(new FailingItem { Data = "poison" }, cancellationToken: cancellationToken); + + var stats = await WaitForDeadLetterAsync(transport, "failing-item", cancellationToken); + Assert.Equal(1, stats.Deadletter); + Assert.Equal(1, Volatile.Read(ref attempts)); // never retried + + var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync("failing-item", new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); + Assert.Equal("unrecoverable:ArgumentException", dead.Headers[KnownHeaders.DeadLetterReason]); + } + + [Fact] + public async Task DeadLetterWhen_GlobalPolicy_AppliesWhenSubscriptionDoesNotOverrideAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport, new MessageBusOptions + { + RetryPolicy = new RetryPolicy { DeadLetterWhen = ex => ex is InvalidOperationException } + }); + int attempts = 0; + + await using var subscription = await bus.SubscribeAsync((_, _) => + { + Interlocked.Increment(ref attempts); + throw new InvalidOperationException("unrecoverable per global policy"); + }, cancellationToken: cancellationToken); + + await bus.SendAsync(new FailingItem { Data = "poison" }, cancellationToken: cancellationToken); + + var stats = await WaitForDeadLetterAsync(transport, "failing-item", cancellationToken); + Assert.Equal(1, stats.Deadletter); + Assert.Equal(1, Volatile.Read(ref attempts)); + } + + [Fact] + public async Task DeadLetter_StampsForensicsHeadersAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + + await using var subscription = await bus.SubscribeAsync((_, _) => + throw new InvalidOperationException("the failure detail"), + new MessageSubscriptionOptions { MaxAttempts = 1 }, cancellationToken); + + await bus.SendAsync(new FailingItem { Data = "doomed" }, cancellationToken: cancellationToken); + + await WaitForDeadLetterAsync(transport, "failing-item", cancellationToken); + var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync("failing-item", new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); + + Assert.Equal(typeof(InvalidOperationException).FullName, dead.Headers[KnownHeaders.DeadLetterExceptionType]); + Assert.Equal("the failure detail", dead.Headers[KnownHeaders.DeadLetterExceptionMessage]); + Assert.NotEmpty(dead.Headers[KnownHeaders.DeadLetterExceptionStackTrace]); + Assert.Equal("failing-item", dead.Headers[KnownHeaders.DeadLetterOriginalDestination]); + Assert.NotEmpty(dead.Headers[KnownHeaders.DeadLetterFailedAt]); + Assert.Equal("1", dead.Headers[KnownHeaders.Attempts]); + } + + [Fact] + public void DefaultBackoff_MatchesTheConvergedCurve() + { + // Immediate first retry, then 10s/20s/30s (capped) with ±20% jitter. + Assert.Equal(TimeSpan.Zero, RetryPolicy.DefaultBackoff(1)); + + foreach ((int attempt, double expectedSeconds) in new[] { (2, 10d), (3, 20d), (4, 30d), (7, 30d) }) + { + var delay = RetryPolicy.DefaultBackoff(attempt); + Assert.InRange(delay.TotalSeconds, expectedSeconds * 0.8, expectedSeconds * 1.2); + } + + // The default policy uses the curve. + Assert.Same(RetryPolicy.DefaultBackoff, new RetryPolicy().Backoff); + } + + private static async Task WaitForDeadLetterAsync(InMemoryMessageTransport transport, string destination, CancellationToken cancellationToken) + { + var stats = await transport.GetStatsAsync(destination, cancellationToken); + long deadline = Environment.TickCount64 + 10_000; + while (stats.Deadletter == 0 && Environment.TickCount64 < deadline) + { + await Task.Delay(25, cancellationToken); + stats = await transport.GetStatsAsync(destination, cancellationToken); + } + + return stats; + } + + [MessageRoute("failing-item")] + private sealed class FailingItem + { + public string? Data { get; set; } + } +} diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 354b36c63..c0479a53f 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -812,6 +812,63 @@ public async Task RejectAsync_Terminal_WithoutNativeDeadLetter_SendsToConfigured Assert.Equal("validation", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterReason)); } + [Fact] + public async Task RejectAsync_Terminal_WithoutNativeDeadLetterOrConfig_DerivesDeadLetterDestinationAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new NoDeadLetterTransport(); + // No DeadLetterDestination configured: the terminal message must be parked at "{source}.deadletter", not dropped. + await using var queue = new MessageBus(transport); + + await queue.SendAsync(new PreviewWorkItem { Data = "bad" }, cancellationToken: cancellationToken); + await using var collector = await MessageCollector.StartAsync(queue, cancellationToken: cancellationToken); + var message = await collector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); + Assert.NotNull(message); + + await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation", Exception = new InvalidOperationException("boom") }, cancellationToken); + + await using var deadCollector = await MessageCollector.StartAsync(queue, destination: "preview-work-item.deadletter", cancellationToken: cancellationToken); + var dead = await deadCollector.NextAsync(TimeSpan.FromSeconds(2), cancellationToken); + Assert.NotNull(dead); + Assert.Equal("validation", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterReason)); + Assert.Equal(typeof(InvalidOperationException).FullName, dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterExceptionType)); + Assert.Equal("preview-work-item", dead.Headers.GetValueOrDefault(KnownHeaders.DeadLetterOriginalDestination)); + } + + [Fact] + public async Task RetryPolicy_BackoffOnTransportWithoutDelaySupport_DegradesToImmediateRedeliveryAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + // BasicQueueTransport has no native redelivery delay and no runtime store is registered, so the default + // backoff curve (10s+ after the second attempt) cannot be honored — the policy retry must degrade to + // immediate redelivery rather than failing the settle, and still reach dead-letter after MaxAttempts. + await using var transport = new BasicQueueTransport(); + await using var queue = new MessageBus(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(15)); + int attempts = 0; + + await using var consumer = await queue.SubscribeAsync((_, _) => + { + Interlocked.Increment(ref attempts); + throw new InvalidOperationException("always fails"); + }, new MessageSubscriptionOptions { MaxAttempts = 3 }, cts.Token); + + await queue.SendAsync(new PreviewWorkItem { Data = "doomed" }, cancellationToken: cts.Token); + + // All three attempts happen without a 10s stall, ending in the transport's native dead-letter sink. + var stats = await transport.GetStatsAsync("preview-work-item", cts.Token); + long deadline = Environment.TickCount64 + 10_000; + while (stats.Deadletter == 0 && Environment.TickCount64 < deadline) + { + await Task.Delay(25, cts.Token); + stats = await transport.GetStatsAsync("preview-work-item", cts.Token); + } + + Assert.Equal(1, stats.Deadletter); + Assert.Equal(3, Volatile.Read(ref attempts)); + } + [Fact] public async Task StartConsumerAsync_UsesDefaultRetryPolicyMaxAttempts_WhenConsumerDoesNotOverrideAsync() { From 81efb18fd822bc09dfcf2100d2e30cc121fd3e5a Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 2 Jul 2026 01:13:48 -0500 Subject: [PATCH 43/57] Address review findings on the failure-path conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All from an adversarial review of the previous commit (9 confirmed findings): - Derived dead-letter parking is best-effort: a park-send failure logs ERROR and drops (completing the original) instead of throwing — a terminal settle must never stall the consumer loop. MessageContext carries a logger for this. - The runtime-store retry fallback only serves queue-channel entries: a subscription-channel entry's Destination is the opaque topic-qualified address, and a queue re-send to that name would land where no subscription group reads. Policy (best-effort) delays on subscription channels degrade to immediate redelivery; explicit delays fail with a precise error. - The exhausted attempt count is recorded in a forensics-only header (message.dead_letter.attempts) instead of overwriting message.attempts, so a replayed dead-letter starts with a fresh retry budget; exception-less deaths (no-handler, unresolved-type) clear any stale exception forensics. - A handler that settles manually and then throws now logs a distinct "threw after settling" warning instead of a retry/dead-letter that the settle path will skip. - The unmatched-type path joins the WARN-while-retryable / ERROR-when-terminal convention (previously every retryable no-handler attempt logged ERROR via the loop), and the loop no longer double-logs UnhandledMessageTypeException. - LogSubscription logs the effective (clamped) concurrency; Key documents that shared-key subscriptions must configure identical failure policies; MessageBusOptions.RuntimeStore documents the pump requirement for hand-wired options. Full solution builds; in-memory 2011 green; live Redis 29 green. Co-Authored-By: Claude Opus 4.8 --- src/Foundatio/Messaging/KnownHeaders.cs | 1 + src/Foundatio/Messaging/MessageBus.cs | 11 ++- src/Foundatio/Messaging/MessageClientCore.cs | 76 +++++++++++++++---- .../Messaging/FailureHandlingTests.cs | 4 +- 4 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/Foundatio/Messaging/KnownHeaders.cs b/src/Foundatio/Messaging/KnownHeaders.cs index 0e4780acb..f8dd99f0d 100644 --- a/src/Foundatio/Messaging/KnownHeaders.cs +++ b/src/Foundatio/Messaging/KnownHeaders.cs @@ -14,6 +14,7 @@ public static class KnownHeaders // Forensics stamped by the core when a message is dead-lettered, so a dead message is triageable with plain // transport tooling. These names are a compatibility contract; values are truncated to fit transport limits. + public const string DeadLetterAttempts = "message.dead_letter.attempts"; public const string DeadLetterExceptionType = "message.dead_letter.exception_type"; public const string DeadLetterExceptionMessage = "message.dead_letter.exception_message"; public const string DeadLetterExceptionStackTrace = "message.dead_letter.exception_stack"; diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index 87fe6d562..1082ed32f 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -103,7 +103,8 @@ public sealed class MessageSubscriptionOptions /// /// Consumer identity. Subscriptions sharing a key on the same channel form one consumer group and compete; - /// defaults to a per-channel key derived from the route. + /// defaults to a per-channel key derived from the route. Subscriptions sharing a key must configure identical + /// failure policies — only the presence of a backoff/DeadLetterWhen is verified, not the delegate itself. /// public string? Key { get; set; } @@ -183,6 +184,12 @@ public sealed record MessageBusOptions public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); + /// + /// Enables durable scheduling: delayed sends beyond a transport ceiling and store-parked retry delays are written + /// here and drained by the job runtime pump. The DI builder registers the pump automatically with the store; when + /// wiring options by hand, ensure a pump (JobRuntimePumpService / JobScheduleProcessor) is running or parked + /// messages will never be dispatched. + /// public IJobRuntimeStore? RuntimeStore { get; init; } public RetryPolicy RetryPolicy { get; init; } = new(); @@ -361,7 +368,7 @@ private void LogSubscription(ListenerConfig send, ListenerConfig publish) { _logger.LogInformation( "Subscribed {MessageType}: send={Destination}, publish={Topic}/{Subscription}, concurrency={MaxConcurrency}, attempts={MaxAttempts}, ack={AckMode}", - send.MessageType.Name, send.Source, publish.Topic, publish.Subscription, send.MaxConcurrency, send.MaxAttempts?.ToString() ?? "default", send.AckMode); + send.MessageType.Name, send.Source, publish.Topic, publish.Subscription, Math.Max(1, send.MaxConcurrency), send.MaxAttempts?.ToString() ?? "default", send.AckMode); } private Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index fc0067cc6..623e49692 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -262,12 +262,18 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, string source, Can var message = CreateMessageContext(entry, cancellationToken); + // Same WARN-while-retryable / ERROR-when-terminal convention as handler failures. + if (message.Attempts >= _retryPolicy.UnmatchedMaxAttempts) + _logger.LogError("No consumer registered for message type \"{MessageType}\" on \"{Source}\" (attempt {Attempt} of {MaxAttempts}); dead-lettering as no-handler", message.MessageType, source, message.Attempts, _retryPolicy.UnmatchedMaxAttempts); + else + _logger.LogWarning("No consumer registered for message type \"{MessageType}\" on \"{Source}\" (attempt {Attempt} of {MaxAttempts}); will retry", message.MessageType, source, message.Attempts, _retryPolicy.UnmatchedMaxAttempts); + // Retry so a node that does handle this type can pick it up; dead-letter as "no-handler" once the lenient // budget is exhausted so a genuinely orphaned type cannot loop forever. await SettleFailedMessageAsync(message, unrecoverable: false, _retryPolicy.UnmatchedMaxAttempts, _retryPolicy.UnmatchedBackoff, deadLetterReason: "no-handler", exception: null, cancellationToken).AnyContext(); - // Surface loudly. The throw is caught by the loop's per-message handling (SafeProcessAsync), so it never tears - // down the receive loop or the other type handlers sharing this source. + // Surface to direct callers. The throw is caught (and not re-logged) by the loop's per-message handling + // (SafeProcessAsync), so it never tears down the receive loop or the other type handlers sharing this source. throw new UnhandledMessageTypeException(message.MessageType, source); } @@ -388,6 +394,11 @@ private async Task SafeProcessAsync(TransportEntry entry, Func(TMessage message, ListenerConfig catch (Exception ex) { activity?.SetErrorStatus(ex); + + // The handler already settled (e.g. terminal-rejected a poison payload, then rethrew): the settle path + // will skip, so don't log a retry/dead-letter that won't happen. + if (message.IsHandled) + { + _logger.LogWarning(ex, "Handler threw after settling message \"{MessageId}\" from \"{Source}\"; no further settlement will occur: {Message}", message.Id, config.Source, ex.Message); + return; + } + int maxAttempts = config.MaxAttempts ?? _retryPolicy.MaxAttempts; var backoff = config.RedeliveryBackoff ?? _retryPolicy.Backoff; @@ -479,7 +499,7 @@ private static Task SettleFailedMessageAsync(IMessageContext message, bool unrec private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken) { MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); - return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); + return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger); } private async Task> CreateMessageContextAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class @@ -520,14 +540,14 @@ private async Task> CreateMessageContextAsync(TransportEnt throw _exceptionFactory($"Message \"{entry.Id}\" deserialized to null.", null); } - return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination); + return new MessageContext(_transport, entry, message, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger); } private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, Exception? exception, CancellationToken cancellationToken) { MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination)); var enriched = entry with { Headers = MessageContext.BuildDeadLetterHeaders(entry, entry.DeliveryCount, exception, _timeProvider) }; - return MessageContext.DeadLetterOrDropAsync(_transport, enriched, reason, _retryPolicy.DeadLetterDestination, cancellationToken); + return MessageContext.DeadLetterOrDropAsync(_transport, enriched, reason, _retryPolicy.DeadLetterDestination, _logger, cancellationToken); } @@ -925,15 +945,17 @@ internal class MessageContext : IMessageContext private readonly IJobRuntimeStore? _runtimeStore; private readonly TimeProvider _timeProvider; private readonly string? _deadLetterDestination; + private readonly ILogger _logger; private int _isHandled; - public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) + public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) { _transport = transport; _entry = entry; _runtimeStore = runtimeStore; _timeProvider = timeProvider ?? TimeProvider.System; _deadLetterDestination = deadLetterDestination; + _logger = logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance; CancellationToken = cancellationToken; } @@ -973,7 +995,7 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c { MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); var enriched = _entry with { Headers = BuildDeadLetterHeaders(_entry, Attempts, options.Exception, _timeProvider) }; - await DeadLetterOrDropAsync(_transport, enriched, options.Reason, _deadLetterDestination, cancellationToken).AnyContext(); + await DeadLetterOrDropAsync(_transport, enriched, options.Reason, _deadLetterDestination, _logger, cancellationToken).AnyContext(); return; } @@ -994,7 +1016,11 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c return; } - if (_runtimeStore is null) + // The runtime-store fallback re-sends the message as a plain queue send, which only makes sense for a + // queue-channel entry: a subscription-channel entry's Destination is the opaque topic-qualified address, and a + // queue send to that name would land where no subscription group reads. + bool isSubscriptionSource = SubscriptionAddress.TryParse(_entry.Destination, out _, out _); + if (_runtimeStore is null || isSubscriptionSource) { // A best-effort delay (the core retry policy) degrades to immediate redelivery; an explicit caller delay // stays strict because the caller is depending on the timing. @@ -1004,7 +1030,7 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c return; } - throw new MessageBusException($"Delayed redelivery requires either native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store."); + throw new MessageBusException($"Delayed redelivery of \"{_entry.Destination}\" requires native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum){(isSubscriptionSource ? "" : " or a registered job runtime store")}."); } // Advance from the reconciled attempt count, not the raw transport DeliveryCount: the re-send produces a new @@ -1039,8 +1065,9 @@ public Task RenewLockAsync(TimeSpan? duration = null, CancellationToken cancella // Terminal settlement. Prefer the transport's native dead-letter sink (preserves native DLQ tooling). When the // transport has none, copy the raw entry to the configured dead-letter destination — or the derived // "{source}.deadletter" when none is configured, so a dead message is always parked somewhere inspectable — - // recording the reason, then complete the original. - internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, TransportEntry entry, string? reason, string? deadLetterDestination, CancellationToken cancellationToken) + // recording the reason, then complete the original. Parking is best-effort: a park failure logs and drops rather + // than throwing, because a terminal settle must never stall the consumer loop. + internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, TransportEntry entry, string? reason, string? deadLetterDestination, ILogger logger, CancellationToken cancellationToken) { if (transport is ISupportsDeadLetter deadLetter) { @@ -1052,16 +1079,26 @@ internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, Tr var headers = String.IsNullOrEmpty(reason) ? entry.Headers : entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); - await transport.SendAsync(destination, [new TransportMessage { Body = entry.Body, Headers = headers, MessageId = entry.Id }], new TransportSendOptions(), cancellationToken).AnyContext(); + + try + { + await transport.SendAsync(destination, [new TransportMessage { Body = entry.Body, Headers = headers, MessageId = entry.Id }], new TransportSendOptions(), cancellationToken).AnyContext(); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to park dead-lettered message \"{MessageId}\" at \"{Destination}\"; dropping it: {Message}", entry.Id, destination, ex.Message); + } + await transport.CompleteAsync(entry, cancellationToken).AnyContext(); } // Stamps the dead-letter forensics contract (see KnownHeaders) so a dead message is triageable — exception details, - // reconciled attempt count, where it was consumed from, and when it died. + // reconciled attempt count, where it was consumed from, and when it died. The attempt count goes in a forensics + // header (never message.attempts) so a replayed message starts with a fresh retry budget. internal static MessageHeaders BuildDeadLetterHeaders(TransportEntry entry, int attempts, Exception? exception, TimeProvider timeProvider) { var headers = entry.Headers.ToBuilder() - .Set(KnownHeaders.Attempts, attempts.ToString(CultureInfo.InvariantCulture)) + .Set(KnownHeaders.DeadLetterAttempts, attempts.ToString(CultureInfo.InvariantCulture)) .Set(KnownHeaders.DeadLetterFailedAt, timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture)) .Set(KnownHeaders.DeadLetterOriginalDestination, entry.Destination); @@ -1072,6 +1109,13 @@ internal static MessageHeaders BuildDeadLetterHeaders(TransportEntry entry, int if (exception.StackTrace is { } stack) headers.Set(KnownHeaders.DeadLetterExceptionStackTrace, Truncate(stack, 4096)); } + else + { + // A death with no exception (no-handler, unresolved-type) must not carry stale forensics from a previous one. + headers.Remove(KnownHeaders.DeadLetterExceptionType); + headers.Remove(KnownHeaders.DeadLetterExceptionMessage); + headers.Remove(KnownHeaders.DeadLetterExceptionStackTrace); + } return headers.Build(); } @@ -1096,8 +1140,8 @@ private static int ParseAttemptsHeader(MessageHeaders headers) internal sealed class MessageContext : MessageContext, IMessageContext where T : class { - public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null) - : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination) + public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) + : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination, logger) { Message = message; } diff --git a/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs index 837f05fe5..409146645 100644 --- a/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs +++ b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs @@ -78,7 +78,9 @@ public async Task DeadLetter_StampsForensicsHeadersAsync() Assert.NotEmpty(dead.Headers[KnownHeaders.DeadLetterExceptionStackTrace]); Assert.Equal("failing-item", dead.Headers[KnownHeaders.DeadLetterOriginalDestination]); Assert.NotEmpty(dead.Headers[KnownHeaders.DeadLetterFailedAt]); - Assert.Equal("1", dead.Headers[KnownHeaders.Attempts]); + // The exhausted count is forensics-only: message.attempts is left alone so a replayed message starts fresh. + Assert.Equal("1", dead.Headers[KnownHeaders.DeadLetterAttempts]); + Assert.False(dead.Headers.ContainsKey(KnownHeaders.Attempts)); } [Fact] From 483849474dc82b6199ad9da3fbcc20db59e80865 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 2 Jul 2026 10:28:41 -0500 Subject: [PATCH 44/57] Add Foundatio.Testing: a messaging test harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Foundatio.Testing package with MessagingTestHarness: run the real MessageBus over a recording in-memory transport, await quiescence, and assert on what actually moved through the bus. - RecordingMessageTransport (internal) decorates an owned InMemoryMessageTransport with its full capability set, recording sends by role (queue -> Sent, topic -> Published) and settlements (Handled, Abandoned, DeadLettered with reason + delivery count), and tracking every destination/source it has seen for idle detection. - Typed accessors (Sent/Published/Handled/DeadLettered) filter by the message.type wire discriminator and deserialize with the same serializer/type registry the bus uses, so assertions read as domain objects. - WaitForIdleAsync polls aggregate stats until nothing is queued or in flight, requiring two consecutive idle observations so a settling message that cascades into a new send is not missed; timing out throws with the busy destinations and their pending counts named. Delayed redeliveries live only in the inner transport's timer (neither queued nor working), so the decorator tracks scheduled-redelivery markers with a small grace window — without this, WaitForIdle would report idle while a retry was pending. - Messaging.UseTestHarness() registers the harness and points the bus at its transport; the harness resolves the container's serializer/registry so typed accessors agree with the wire format. This makes the core-owned failure path directly assertable in user tests: a poison message's retries (Abandoned) and terminal dead-letter (reason + forensics headers) are recorded facts, not log lines. Full solution builds; in-memory 2015 green; live Redis 29 green. Co-Authored-By: Claude Fable 5 --- Foundatio.All.slnx | 1 + Foundatio.slnx | 1 + .../Foundatio.Testing.csproj | 8 + src/Foundatio.Testing/MessagingTestHarness.cs | 150 ++++++++++++++ .../RecordingMessageTransport.cs | 188 ++++++++++++++++++ .../TestingFoundatioBuilderExtensions.cs | 26 +++ tests/Foundatio.Tests/Foundatio.Tests.csproj | 1 + .../Messaging/MessagingTestHarnessTests.cs | 167 ++++++++++++++++ 8 files changed, 542 insertions(+) create mode 100644 src/Foundatio.Testing/Foundatio.Testing.csproj create mode 100644 src/Foundatio.Testing/MessagingTestHarness.cs create mode 100644 src/Foundatio.Testing/RecordingMessageTransport.cs create mode 100644 src/Foundatio.Testing/TestingFoundatioBuilderExtensions.cs create mode 100644 tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs diff --git a/Foundatio.All.slnx b/Foundatio.All.slnx index 0dcdddb02..a6974581f 100644 --- a/Foundatio.All.slnx +++ b/Foundatio.All.slnx @@ -71,6 +71,7 @@ + diff --git a/Foundatio.slnx b/Foundatio.slnx index f90428570..d6d71db09 100644 --- a/Foundatio.slnx +++ b/Foundatio.slnx @@ -30,6 +30,7 @@ + diff --git a/src/Foundatio.Testing/Foundatio.Testing.csproj b/src/Foundatio.Testing/Foundatio.Testing.csproj new file mode 100644 index 000000000..1989146f8 --- /dev/null +++ b/src/Foundatio.Testing/Foundatio.Testing.csproj @@ -0,0 +1,8 @@ + + + Test harness for Foundatio messaging: run the real message bus over a recording in-memory transport, await quiescence, and assert on the messages that were sent, published, handled, retried, or dead-lettered. + + + + + diff --git a/src/Foundatio.Testing/MessagingTestHarness.cs b/src/Foundatio.Testing/MessagingTestHarness.cs new file mode 100644 index 000000000..204d5c665 --- /dev/null +++ b/src/Foundatio.Testing/MessagingTestHarness.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Serializer; + +namespace Foundatio.Messaging.Testing; + +/// +/// One recorded message movement: a send/publish accepted by the transport, or a settlement (handled, abandoned for +/// retry, dead-lettered) of a delivered message. +/// +public sealed record RecordedMessage +{ + public required string Destination { get; init; } + + /// + /// The role the message was sent to (Queue for sends, Topic for publishes). Settlement recordings always report + /// Queue: every delivery settles on a queue-shaped channel, including topic deliveries via their subscriptions. + /// + public required DestinationRole Role { get; init; } + + public string? MessageType { get; init; } + public required ReadOnlyMemory Body { get; init; } + public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; + + /// The dead-letter reason, for dead-lettered recordings. + public string? Reason { get; init; } + + /// The delivery count at settlement, for settlement recordings. + public int Attempts { get; init; } +} + +/// +/// Deterministic messaging tests without sleeps: the harness runs the real bus over a recording in-memory transport, +/// so tests act (send/publish), until every queue and in-flight handler drains, then +/// assert on what actually happened — including the core retry/dead-letter path (a message redelivered N times and +/// then dead-lettered is directly assertable). +/// +/// var services = new ServiceCollection(); +/// services.AddFoundatio() +/// .Messaging.UseTestHarness() +/// .Messaging.AddHandler<OrderPlaced, SendConfirmationHandler>(); +/// // start hosted services, then: +/// await bus.PublishAsync(new OrderPlaced(42)); +/// await harness.WaitForIdleAsync(); +/// Assert.Single(harness.Published<OrderPlaced>()); +/// Assert.Empty(harness.DeadLetteredMessages); +/// +/// +public sealed class MessagingTestHarness : IAsyncDisposable +{ + private static readonly TimeSpan DefaultIdleTimeout = TimeSpan.FromSeconds(30); + + private readonly RecordingMessageTransport _transport; + private readonly ISerializer _serializer; + private readonly IMessageTypeRegistry _typeRegistry; + + public MessagingTestHarness(ISerializer? serializer = null, IMessageTypeRegistry? typeRegistry = null, TimeProvider? timeProvider = null) + { + _serializer = serializer ?? DefaultSerializer.Instance; + _typeRegistry = typeRegistry ?? new MessageTypeRegistry(); + _transport = new RecordingMessageTransport(timeProvider); + } + + /// The transport to run the bus over (an in-memory transport that records every movement). + public IMessageTransport Transport => _transport; + + /// Every message accepted by a queue-role send (a command on its way to one handler). + public IReadOnlyList SentMessages => _transport.Sent; + + /// Every message accepted by a topic-role send (an event on its way to each subscriber). + public IReadOnlyList PublishedMessages => _transport.Published; + + /// Every delivered message that settled as completed (handled successfully or auto-acked). + public IReadOnlyList HandledMessages => _transport.Handled; + + /// Every delivered message returned for redelivery (a retry). + public IReadOnlyList AbandonedMessages => _transport.Abandoned; + + /// Every delivered message that settled terminally into the dead-letter sink. + public IReadOnlyList DeadLetteredMessages => _transport.DeadLettered; + + /// The sent (queue-role) messages of type , deserialized. + public IReadOnlyList Sent() where T : class => Deserialize(_transport.Sent); + + /// The published (topic-role) messages of type , deserialized. + public IReadOnlyList Published() where T : class => Deserialize(_transport.Published); + + /// The successfully handled messages of type , deserialized. + public IReadOnlyList Handled() where T : class => Deserialize(_transport.Handled); + + /// The dead-lettered messages of type , deserialized. + public IReadOnlyList DeadLettered() where T : class => Deserialize(_transport.DeadLettered); + + /// + /// Waits until the transport is quiescent — every known destination has nothing queued and nothing in flight — + /// so assertions observe the final state. Returns quickly when already idle (fast negative assertions). Throws + /// naming the still-busy destinations when the timeout (default 30s) lapses. + /// Store-parked work (delayed sends / delayed retries through a runtime store) is not transport activity; drain + /// it explicitly via the job schedule processor before waiting. + /// + public async Task WaitForIdleAsync(TimeSpan? timeout = null, CancellationToken cancellationToken = default) + { + long deadline = Environment.TickCount64 + (long)(timeout ?? DefaultIdleTimeout).TotalMilliseconds; + int stableChecks = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var pending = await _transport.GetPendingAsync(cancellationToken).ConfigureAwait(false); + + if (pending.Count == 0) + { + // Require two consecutive idle observations: a settling message can synchronously cascade into a new + // send, which a single snapshot could miss. + if (++stableChecks >= 2) + return; + } + else + { + stableChecks = 0; + + if (Environment.TickCount64 >= deadline) + { + var detail = new StringBuilder("The message bus did not become idle in time. Still busy: "); + detail.AppendJoin(", ", pending.Select(p => $"{p.Name} (queued={p.Queued}, working={p.Working})")); + throw new TimeoutException(detail.ToString()); + } + } + + await Task.Delay(25, cancellationToken).ConfigureAwait(false); + } + } + + public ValueTask DisposeAsync() => _transport.DisposeAsync(); + + private IReadOnlyList Deserialize(IReadOnlyList recordings) where T : class + { + string typeName = _typeRegistry.GetName(typeof(T)); + return recordings + .Where(r => String.Equals(r.MessageType, typeName, StringComparison.Ordinal)) + .Select(r => _serializer.Deserialize(r.Body, typeof(T)) as T) + .Where(m => m is not null) + .Select(m => m!) + .ToList(); + } +} diff --git a/src/Foundatio.Testing/RecordingMessageTransport.cs b/src/Foundatio.Testing/RecordingMessageTransport.cs new file mode 100644 index 000000000..868a991a7 --- /dev/null +++ b/src/Foundatio.Testing/RecordingMessageTransport.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; + +namespace Foundatio.Messaging.Testing; + +/// +/// The harness transport: a fully-capable in-memory transport that records every send and settlement so tests can +/// assert on what actually moved through the bus, and tracks the destinations/sources it has seen so +/// can detect quiescence. +/// +internal sealed class RecordingMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, + ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, ISupportsPriority, + ISupportsExpiration, ISupportsProvisioning, ITransportInfo +{ + // A delayed redelivery lives only in the inner transport's timer until it fires — neither queued nor in flight — + // so idle detection would report quiescent while a retry is pending. Give the timer this long past its due time to + // materialize the redelivered message back into stats before the pending marker is dropped. + private static readonly TimeSpan _redeliveryGrace = TimeSpan.FromMilliseconds(250); + + private readonly InMemoryMessageTransport _inner; + private readonly TimeProvider _timeProvider; + private readonly ConcurrentQueue _sent = new(); + private readonly ConcurrentQueue _published = new(); + private readonly ConcurrentQueue _handled = new(); + private readonly ConcurrentQueue _abandoned = new(); + private readonly ConcurrentQueue _deadLettered = new(); + private readonly ConcurrentDictionary _knownNames = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _pendingRedeliveries = new(); + + public RecordingMessageTransport(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _inner = new InMemoryMessageTransport(timeProvider); + } + + public IReadOnlyList Sent => [.. _sent]; + public IReadOnlyList Published => [.. _published]; + public IReadOnlyList Handled => [.. _handled]; + public IReadOnlyList Abandoned => [.. _abandoned]; + public IReadOnlyList DeadLettered => [.. _deadLettered]; + + public DeliveryGuarantee DeliveryGuarantee => _inner.DeliveryGuarantee; + public OrderingGuarantee Ordering => _inner.Ordering; + public IReadOnlySet SupportedRoles => _inner.SupportedRoles; + public int? MaxBatchSize => _inner.MaxBatchSize; + public long? MaxMessageBytes => _inner.MaxMessageBytes; + public TimeSpan? MaxVisibilityTimeout => _inner.MaxVisibilityTimeout; + public TimeSpan? MaxRedeliveryDelay => _inner.MaxRedeliveryDelay; + + public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + var result = await _inner.SendAsync(destination, messages, options, ct).ConfigureAwait(false); + + _knownNames.TryAdd(destination, 0); + var recordings = options.DestinationRole == DestinationRole.Topic ? _published : _sent; + foreach (var message in messages) + { + recordings.Enqueue(new RecordedMessage + { + Destination = destination, + Role = options.DestinationRole, + MessageType = message.Headers.GetValueOrDefault(KnownHeaders.MessageType), + Body = message.Body, + Headers = message.Headers + }); + } + + return result; + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct = default) + { + _knownNames.TryAdd(source, 0); + return _inner.ReceiveAsync(source, request, ct); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default) + { + _knownNames.TryAdd(source, 0); + return _inner.ReceiveAsync(source, request, visibility, ct); + } + + public Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct = default) + { + _knownNames.TryAdd(source, 0); + return _inner.SubscribeAsync(source, onMessage, options, ct); + } + + public async Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + await _inner.CompleteAsync(entry, ct).ConfigureAwait(false); + _handled.Enqueue(Record(entry)); + } + + public async Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + await _inner.AbandonAsync(entry, ct).ConfigureAwait(false); + _abandoned.Enqueue(Record(entry)); + } + + public async Task AbandonAsync(TransportEntry entry, TimeSpan redeliveryDelay, CancellationToken ct = default) + { + Guid pendingToken = Guid.NewGuid(); + if (redeliveryDelay > TimeSpan.Zero) + _pendingRedeliveries[pendingToken] = (entry.Destination, _timeProvider.GetUtcNow().Add(redeliveryDelay)); + + try + { + await _inner.AbandonAsync(entry, redeliveryDelay, ct).ConfigureAwait(false); + } + catch + { + _pendingRedeliveries.TryRemove(pendingToken, out _); + throw; + } + + _abandoned.Enqueue(Record(entry)); + } + + public async Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationToken ct = default) + { + await _inner.DeadLetterAsync(entry, reason, ct).ConfigureAwait(false); + _deadLettered.Enqueue(Record(entry) with { Reason = reason }); + } + + public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct = default) + => _inner.ReceiveDeadLetteredAsync(destination, request, ct); + + public Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct = default) + => _inner.RenewLockAsync(entry, duration, ct); + + public Task GetStatsAsync(string destination, CancellationToken ct = default) + => _inner.GetStatsAsync(destination, ct); + + public Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct = default) + { + foreach (var declaration in declarations) + _knownNames.TryAdd(declaration.Name, 0); + return _inner.EnsureAsync(declarations, ct); + } + + public Task DeleteAsync(string name, CancellationToken ct = default) => _inner.DeleteAsync(name, ct); + + public Task ExistsAsync(string name, CancellationToken ct = default) => _inner.ExistsAsync(name, ct); + + public ValueTask DisposeAsync() => _inner.DisposeAsync(); + + // Aggregate pending work across every destination/source this transport has seen; idle means nothing queued, + // nothing in flight, and no delayed redelivery still waiting on its timer. + public async Task> GetPendingAsync(CancellationToken ct = default) + { + var now = _timeProvider.GetUtcNow(); + var scheduled = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var redelivery in _pendingRedeliveries) + { + if (now >= redelivery.Value.DueAt + _redeliveryGrace) + _pendingRedeliveries.TryRemove(redelivery.Key, out _); + else + scheduled[redelivery.Value.Destination] = scheduled.GetValueOrDefault(redelivery.Value.Destination) + 1; + } + + var pending = new List<(string, long, long)>(); + foreach (string name in _knownNames.Keys.OrderBy(n => n, StringComparer.Ordinal)) + { + var stats = await _inner.GetStatsAsync(name, ct).ConfigureAwait(false); + long queued = stats.Queued + scheduled.GetValueOrDefault(name); + if (queued > 0 || stats.Working > 0) + pending.Add((name, queued, stats.Working)); + } + + return pending; + } + + private static RecordedMessage Record(TransportEntry entry) => new() + { + Destination = entry.Destination, + Role = DestinationRole.Queue, + MessageType = entry.Headers.GetValueOrDefault(KnownHeaders.MessageType), + Body = entry.Body, + Headers = entry.Headers, + Attempts = entry.DeliveryCount + }; +} diff --git a/src/Foundatio.Testing/TestingFoundatioBuilderExtensions.cs b/src/Foundatio.Testing/TestingFoundatioBuilderExtensions.cs new file mode 100644 index 000000000..cb899d6f2 --- /dev/null +++ b/src/Foundatio.Testing/TestingFoundatioBuilderExtensions.cs @@ -0,0 +1,26 @@ +using System; +using Foundatio.Messaging; +using Foundatio.Messaging.Testing; +using Foundatio.Serializer; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace Foundatio; + +public static class TestingFoundatioBuilderExtensions +{ + /// + /// Runs messaging over a recording in-memory transport for tests. Resolve from + /// the container to await quiescence () and assert on the + /// messages that were sent, published, handled, retried, or dead-lettered. + /// + public static FoundatioBuilder UseTestHarness(this FoundatioBuilder.MessagingBuilder builder) + { + var services = ((IFoundatioBuilder)builder).Services; + services.TryAddSingleton(sp => new MessagingTestHarness( + sp.GetService(), + sp.GetService(), + sp.GetService())); + return builder.UseTransport(sp => sp.GetRequiredService().Transport); + } +} diff --git a/tests/Foundatio.Tests/Foundatio.Tests.csproj b/tests/Foundatio.Tests/Foundatio.Tests.csproj index cd934dc96..fb17d2e29 100644 --- a/tests/Foundatio.Tests/Foundatio.Tests.csproj +++ b/tests/Foundatio.Tests/Foundatio.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs b/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs new file mode 100644 index 000000000..008aa67c5 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Messaging.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class MessagingTestHarnessTests +{ + [Fact] + public async Task Harness_RecordsSendPublishAndHandledWithTypedAccessAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var harness = new MessagingTestHarness(); + await using var bus = new MessageBus(harness.Transport, new MessageBusOptions { OwnsTransport = false }); + + var handled = new List(); + await using var subscription = await bus.SubscribeAsync((context, _) => + { + lock (handled) + handled.Add(context.Message.Id); + return Task.CompletedTask; + }, cancellationToken: cancellationToken); + + await bus.SendAsync(new HarnessOrder { Id = "cmd" }, cancellationToken: cancellationToken); + await bus.PublishAsync(new HarnessOrder { Id = "evt" }, cancellationToken: cancellationToken); + await harness.WaitForIdleAsync(cancellationToken: cancellationToken); + + // Sends and publishes are recorded separately, deserialized back to the message type. + Assert.Equal("cmd", Assert.Single(harness.Sent()).Id); + Assert.Equal("evt", Assert.Single(harness.Published()).Id); + Assert.Equal(2, harness.Handled().Count); + Assert.Equal(2, handled.Count); + + // Raw recordings carry the route and role for topology assertions. + var sent = Assert.Single(harness.SentMessages); + Assert.Equal("harness-orders", sent.Destination); + Assert.Equal(DestinationRole.Queue, sent.Role); + var published = Assert.Single(harness.PublishedMessages); + Assert.Equal("harness-orders", published.Destination); + Assert.Equal(DestinationRole.Topic, published.Role); + + // Negative assertions are immediate once idle. + Assert.Empty(harness.DeadLetteredMessages); + Assert.Empty(harness.AbandonedMessages); + Assert.Empty(harness.Sent()); + } + + [Fact] + public async Task Harness_RetryCycleEndsInDeadLetterAndIsFullyObservableAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var harness = new MessagingTestHarness(); + await using var bus = new MessageBus(harness.Transport, new MessageBusOptions { OwnsTransport = false }); + + int attempts = 0; + await using var subscription = await bus.SubscribeAsync((_, _) => + { + Interlocked.Increment(ref attempts); + throw new InvalidOperationException("always fails"); + }, new MessageSubscriptionOptions { MaxAttempts = 3, RedeliveryBackoff = _ => TimeSpan.Zero }, cancellationToken); + + await bus.SendAsync(new HarnessOrder { Id = "poison" }, cancellationToken: cancellationToken); + await harness.WaitForIdleAsync(cancellationToken: cancellationToken); + + // The whole failure path is assertable: two retries, then terminal dead-letter with the reason and forensics. + Assert.Equal(3, Volatile.Read(ref attempts)); + Assert.Equal(2, harness.AbandonedMessages.Count); + var dead = Assert.Single(harness.DeadLetteredMessages); + Assert.Equal("handler-error", dead.Reason); + Assert.Equal(3, dead.Attempts); + Assert.Equal(typeof(InvalidOperationException).FullName, dead.Headers[KnownHeaders.DeadLetterExceptionType]); + Assert.Equal("poison", Assert.Single(harness.DeadLettered()).Id); + Assert.Empty(harness.HandledMessages); + } + + [Fact] + public async Task WaitForIdle_CoversDelayedRedeliveriesAndTimesOutWithDiagnosticsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var harness = new MessagingTestHarness(); + await using var bus = new MessageBus(harness.Transport, new MessageBusOptions { OwnsTransport = false }); + + int attempts = 0; + await using var subscription = await bus.SubscribeAsync((_, _) => + { + if (Interlocked.Increment(ref attempts) == 1) + throw new InvalidOperationException("fails once"); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { MaxAttempts = 2, RedeliveryBackoff = _ => TimeSpan.FromMilliseconds(500) }, cancellationToken); + + await bus.SendAsync(new HarnessOrder { Id = "retry-me" }, cancellationToken: cancellationToken); + + // The retry is parked in a redelivery timer (neither queued nor in flight); the harness must still see it. + await harness.WaitForIdleAsync(cancellationToken: cancellationToken); + Assert.Equal(2, Volatile.Read(ref attempts)); + Assert.Single(harness.Handled()); + + // A destination that never drains fails with the busy destinations named. + await using var stuck = await bus.SubscribeAsync((_, handlerToken) => Task.Delay(Timeout.Infinite, handlerToken), + cancellationToken: cancellationToken); + await bus.SendAsync(new HarnessOther { Id = "stuck" }, cancellationToken: cancellationToken); + + var timeout = await Assert.ThrowsAsync(() => harness.WaitForIdleAsync(TimeSpan.FromSeconds(2), cancellationToken)); + Assert.Contains("harness-other", timeout.Message); + } + + [Fact] + public async Task UseTestHarness_WiresDeclarativeHandlersOverTheRecordingTransportAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddFoundatio() + .Messaging.UseTestHarness() + .Messaging.AddHandler(); + + await using var provider = services.BuildServiceProvider(); + var hosted = provider.GetServices().ToList(); + foreach (var service in hosted) + await service.StartAsync(cancellationToken); + + try + { + var harness = provider.GetRequiredService(); + var bus = provider.GetRequiredService(); + Assert.Same(harness.Transport, provider.GetRequiredService()); + + await bus.SendAsync(new HarnessOrder { Id = "from-di" }, cancellationToken: cancellationToken); + await harness.WaitForIdleAsync(cancellationToken: cancellationToken); + + Assert.Equal("from-di", Assert.Single(harness.Sent()).Id); + Assert.Equal("from-di", Assert.Single(harness.Handled()).Id); + Assert.Equal("from-di", Assert.Single(RecordingOrderHandler.Handled)); + } + finally + { + foreach (var service in hosted) + await service.StopAsync(cancellationToken); + } + } + + [MessageRoute("harness-orders")] + public class HarnessOrder { public string Id { get; set; } = ""; } + + [MessageRoute("harness-other")] + public class HarnessOther { public string Id { get; set; } = ""; } + + private sealed class RecordingOrderHandler : IMessageHandler + { + public static readonly List Handled = []; + + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) + { + lock (Handled) + Handled.Add(context.Message.Id); + return Task.CompletedTask; + } + } +} From 65ae7046bb2a021ce3c340f760c44794206f2ae8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 2 Jul 2026 15:33:42 -0500 Subject: [PATCH 45/57] Address review findings on the messaging test harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From an adversarial review of the previous commit: - WaitForIdleAsync honors Timeout.InfiniteTimeSpan as wait-until-idle (previously the -1ms sentinel produced an already-lapsed deadline, so the call threw TimeoutException immediately whenever the bus happened to be busy); other negative timeouts are rejected with ArgumentOutOfRangeException up front. - Added the missing Abandoned() typed accessor so the retry path has the same typed access as sent/published/handled/dead-lettered. - DeadLetteredMessages documents its boundary: it records deaths made through the transport API; broker-internal deaths (e.g. TimeToLive lapsing before delivery happens inside the transport's receive path) surface in stats and ReceiveDeadLetteredAsync but are not recordable by a decorator. Verified-and-rejected (no change): the wall-clock redelivery-marker grace and the settle-then-record ordering are unreachable races — the redelivery timer and the idle poller share the same timer queue/thread pool (FIFO ordering means the lateness that would delay the timer delays the pruning polls behind it), and the inner settle is fully synchronous so the recording enqueue has no yield point after it. Full solution builds; in-memory 2015 green. Co-Authored-By: Claude Fable 5 --- src/Foundatio.Testing/MessagingTestHarness.cs | 21 ++++++++++++++++--- .../Messaging/MessagingTestHarnessTests.cs | 6 ++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/Foundatio.Testing/MessagingTestHarness.cs b/src/Foundatio.Testing/MessagingTestHarness.cs index 204d5c665..9c093f3c9 100644 --- a/src/Foundatio.Testing/MessagingTestHarness.cs +++ b/src/Foundatio.Testing/MessagingTestHarness.cs @@ -80,7 +80,12 @@ public MessagingTestHarness(ISerializer? serializer = null, IMessageTypeRegistry /// Every delivered message returned for redelivery (a retry). public IReadOnlyList AbandonedMessages => _transport.Abandoned; - /// Every delivered message that settled terminally into the dead-letter sink. + /// + /// Every delivered message that settled terminally into the dead-letter sink. Records deaths made through the + /// transport API (the core's retry-exhausted/unrecoverable path); a broker-internal death — such as a message + /// whose TimeToLive lapsed before delivery — is visible in destination stats and ReceiveDeadLetteredAsync but + /// not recorded here. + /// public IReadOnlyList DeadLetteredMessages => _transport.DeadLettered; /// The sent (queue-role) messages of type , deserialized. @@ -92,19 +97,29 @@ public MessagingTestHarness(ISerializer? serializer = null, IMessageTypeRegistry /// The successfully handled messages of type , deserialized. public IReadOnlyList Handled() where T : class => Deserialize(_transport.Handled); + /// The retried (abandoned for redelivery) messages of type , deserialized. + public IReadOnlyList Abandoned() where T : class => Deserialize(_transport.Abandoned); + /// The dead-lettered messages of type , deserialized. public IReadOnlyList DeadLettered() where T : class => Deserialize(_transport.DeadLettered); /// /// Waits until the transport is quiescent — every known destination has nothing queued and nothing in flight — /// so assertions observe the final state. Returns quickly when already idle (fast negative assertions). Throws - /// naming the still-busy destinations when the timeout (default 30s) lapses. + /// naming the still-busy destinations when the timeout (default 30s) lapses; + /// waits until idle or cancellation. /// Store-parked work (delayed sends / delayed retries through a runtime store) is not transport activity; drain /// it explicitly via the job schedule processor before waiting. /// public async Task WaitForIdleAsync(TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - long deadline = Environment.TickCount64 + (long)(timeout ?? DefaultIdleTimeout).TotalMilliseconds; + var effectiveTimeout = timeout ?? DefaultIdleTimeout; + if (effectiveTimeout < TimeSpan.Zero && effectiveTimeout != Timeout.InfiniteTimeSpan) + throw new ArgumentOutOfRangeException(nameof(timeout), timeout, "Timeout must be non-negative or Timeout.InfiniteTimeSpan."); + + long deadline = effectiveTimeout == Timeout.InfiniteTimeSpan + ? Int64.MaxValue + : Environment.TickCount64 + (long)effectiveTimeout.TotalMilliseconds; int stableChecks = 0; while (true) diff --git a/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs b/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs index 008aa67c5..5b73010eb 100644 --- a/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs +++ b/tests/Foundatio.Tests/Messaging/MessagingTestHarnessTests.cs @@ -72,6 +72,8 @@ public async Task Harness_RetryCycleEndsInDeadLetterAndIsFullyObservableAsync() // The whole failure path is assertable: two retries, then terminal dead-letter with the reason and forensics. Assert.Equal(3, Volatile.Read(ref attempts)); Assert.Equal(2, harness.AbandonedMessages.Count); + Assert.All(harness.Abandoned(), m => Assert.Equal("poison", m.Id)); + Assert.Equal(2, harness.Abandoned().Count); var dead = Assert.Single(harness.DeadLetteredMessages); Assert.Equal("handler-error", dead.Reason); Assert.Equal(3, dead.Attempts); @@ -102,6 +104,10 @@ public async Task WaitForIdle_CoversDelayedRedeliveriesAndTimesOutWithDiagnostic Assert.Equal(2, Volatile.Read(ref attempts)); Assert.Single(harness.Handled()); + // Timeout.InfiniteTimeSpan means wait-until-idle; other negative timeouts are rejected up front. + await harness.WaitForIdleAsync(Timeout.InfiniteTimeSpan, cancellationToken); + await Assert.ThrowsAsync(() => harness.WaitForIdleAsync(TimeSpan.FromMilliseconds(-2), cancellationToken)); + // A destination that never drains fails with the busy destinations named. await using var stuck = await bus.SubscribeAsync((_, handlerToken) => Task.Delay(Timeout.Infinite, handlerToken), cancellationToken: cancellationToken); From 93214d0611d28a363ed999803fa5ba9bca0f5d71 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 7 Jul 2026 00:12:45 -0500 Subject: [PATCH 46/57] Fix CI: tolerate all-skipped runs in the env-gated integration test projects Foundatio.Redis.Tests and Foundatio.Aws.Tests skip every test when their FOUNDATIO_*_CONNECTION_STRING variables are unset, and Microsoft.Testing.Platform fails a zero-tests session with exit code 8, breaking the build job. Ignore that exit code per project via TestingPlatformCommandLineArguments, the documented mechanism for intentionally all-skipped assemblies. Co-Authored-By: Claude Fable 5 --- tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj | 4 ++++ tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj b/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj index 8fe4f59ce..d6d9f3e74 100644 --- a/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj +++ b/tests/Foundatio.Aws.Tests/Foundatio.Aws.Tests.csproj @@ -1,4 +1,8 @@ + + + $(TestingPlatformCommandLineArguments) --ignore-exit-code 8 + diff --git a/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj b/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj index 33f37fc01..3d2ab2138 100644 --- a/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj +++ b/tests/Foundatio.Redis.Tests/Foundatio.Redis.Tests.csproj @@ -1,4 +1,8 @@ + + + $(TestingPlatformCommandLineArguments) --ignore-exit-code 8 + From 64414cb7ab4212ce8b467fefdb8babab980a94dc Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 17:45:11 -0500 Subject: [PATCH 47/57] Remove aspirational send options: DeduplicationId and PartitionKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeduplicationId was settable but no transport implements broker dedup — it only leaked into the message id, aliasing distinct messages. PartitionKey was declared on TransportSendOptions and read nowhere. Both come back only when a capability contract and conformance tests exist for them (review feedback #10). Co-Authored-By: Claude Fable 5 --- src/Foundatio/Messaging/InMemoryMessageTransport.cs | 3 +-- src/Foundatio/Messaging/MessageBus.cs | 4 ---- src/Foundatio/Messaging/MessageClientCore.cs | 11 +++-------- src/Foundatio/Messaging/MessageTransport.cs | 2 -- tests/Foundatio.Tests/Queue/BasicQueueTransport.cs | 2 +- 5 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index fce129528..8494c8219 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -63,8 +63,7 @@ public Task SendAsync(string destination, IReadOnlyListOverrides the routed destination for this send. public string? Destination { get; init; } public MessageHeaders? Headers { get; init; } @@ -31,7 +30,6 @@ public sealed record MessagePublishOptions public DateTimeOffset? DeliverAt { get; init; } public TimeSpan? TimeToLive { get; init; } public string? CorrelationId { get; init; } - public string? DeduplicationId { get; init; } /// Overrides the routed topic for this publish. public string? Topic { get; init; } public MessageHeaders? Headers { get; init; } @@ -423,7 +421,6 @@ private static MessageEnvelopeOptions ToEnvelope(MessageSendOptions options) DeliverAt = options.DeliverAt, TimeToLive = options.TimeToLive, CorrelationId = options.CorrelationId, - DeduplicationId = options.DeduplicationId, Headers = options.Headers }; } @@ -437,7 +434,6 @@ private static MessageEnvelopeOptions ToEnvelope(MessagePublishOptions options) DeliverAt = options.DeliverAt, TimeToLive = options.TimeToLive, CorrelationId = options.CorrelationId, - DeduplicationId = options.DeduplicationId, Headers = options.Headers }; } diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 623e49692..461c2ad75 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -40,7 +40,6 @@ internal sealed record MessageEnvelopeOptions public DateTimeOffset? DeliverAt { get; init; } public TimeSpan? TimeToLive { get; init; } public string? CorrelationId { get; init; } - public string? DeduplicationId { get; init; } public MessageHeaders? Headers { get; init; } } @@ -114,7 +113,7 @@ public async Task SendAsync(ScheduledDispatchKind kind, Type messageType ValidateCapabilities(options.Priority, options.TimeToLive); var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; - string messageId = options.DeduplicationId ?? Guid.NewGuid().ToString("N"); + string messageId = Guid.NewGuid().ToString("N"); var transportMessage = CreateTransportMessage(message, messageType, options, messageId); if (ensureDestination is not null) @@ -136,15 +135,12 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; var grouped = new Dictionary>(StringComparer.Ordinal); - int index = 0; foreach (var message in messages) { ArgumentNullException.ThrowIfNull(message); Type messageType = declaredType ?? message.GetType(); string destination = resolveDestination(messageType); - string? messageId = options.DeduplicationId is null ? null : $"{options.DeduplicationId}:{index}"; - index++; if (!grouped.TryGetValue(destination, out var transportMessages)) { @@ -152,7 +148,7 @@ public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable grouped.Add(destination, transportMessages); } - transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId)); + transportMessages.Add(CreateTransportMessage(message, messageType, options, messageId: null)); } foreach (var group in grouped) @@ -595,8 +591,7 @@ private TransportSendOptions BuildSendOptions(MessageEnvelopeOptions options) return new TransportSendOptions { Priority = options.Priority, - DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _timeProvider.GetUtcNow().Add(delay) : null), - DeduplicationId = options.DeduplicationId + DeliverAt = options.DeliverAt ?? (options.Delay is { } delay ? _timeProvider.GetUtcNow().Add(delay) : null) }; } diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 1ba7fe621..72aacba40 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -52,8 +52,6 @@ public sealed record TransportSendOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public DateTimeOffset? DeliverAt { get; init; } - public string? DeduplicationId { get; init; } - public string? PartitionKey { get; init; } /// /// The role of the destination being sent to. Lets a transport route the send without inferring (for example, a diff --git a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs index 1fa97aaf4..320f10266 100644 --- a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs +++ b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs @@ -26,7 +26,7 @@ public Task SendAsync(string destination, IReadOnlyList Date: Thu, 9 Jul 2026 17:57:38 -0500 Subject: [PATCH 48/57] Role-aware transport capabilities; fix silent delay drop on AWS delayed publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transport-wide marker interfaces (ISupportsDelayedDelivery/Priority/Expiration) could not express that SQS queues take a native DelaySeconds while SNS topics have no delay at all: a delayed publish within 15 minutes took the native path and SNS published immediately, silently dropping the delay. Capabilities and limits (delayed delivery + ceiling, priority, expiration, ordering, batch and size limits) now live on a TransportCapabilities record that ITransportInfo returns per destination role, and every core send-path decision — native-vs- runtime-store scheduling, priority/TTL validation, size checks, batch chunking — asks for the role it is actually targeting. Transports that cannot honor a future DeliverAt now refuse it loudly instead of accepting and dropping it (AWS topic branch, Redis Streams), with a new conformance fact enforcing that contract, plus a core regression test proving a delayed publish on a queue-delay-only transport routes through the runtime store. (Review feedback #2.) Co-Authored-By: Claude Fable 5 --- src/Foundatio.Aws/AwsMessageTransport.cs | 33 +++++++-- .../Messaging/RedisStreamsMessageTransport.cs | 14 +++- .../MessageTransportConformanceTests.cs | 57 ++++++++++++++-- .../RecordingMessageTransport.cs | 8 +-- .../Messaging/InMemoryMessageTransport.cs | 16 +++-- src/Foundatio/Messaging/MessageClientCore.cs | 39 +++++++---- src/Foundatio/Messaging/MessageTransport.cs | 59 ++++++++++++---- .../RedisJobStoreIntegrationTests.cs | 10 ++- .../Foundatio.Tests/Messaging/PubSubTests.cs | 68 +++++++++++++++++++ .../Queue/MessageQueueTests.cs | 12 +++- 10 files changed, 258 insertions(+), 58 deletions(-) diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index eb6ded841..ba48fedff 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -24,12 +24,13 @@ namespace Foundatio.Messaging; /// /// /// Capability mapping: pull receive (SQS long poll), visibility timeout, redelivery delay (ChangeMessageVisibility, -/// 12h cap), delayed delivery (SQS DelaySeconds, 15-minute cap), provisioning, and stats. SQS has no per-message +/// 12h cap), delayed delivery on queues only (SQS DelaySeconds, 15-minute cap — SNS topics have no native delay, so +/// delayed publishes route through the runtime-store fallback), provisioning, and stats. SQS has no per-message /// priority, per-message TTL, or push delivery, and no transport-native dead-letter that the core controls the timing /// of, so those capabilities are intentionally not implemented (the core owns retry/dead-lettering). /// public sealed class AwsMessageTransport : IMessageTransport, ISupportsPull, ISupportsVisibilityTimeout, - ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsDelayedDelivery, ISupportsProvisioning, ISupportsStats, ITransportInfo + ISupportsLockRenewal, ISupportsRedeliveryDelay, ISupportsProvisioning, ISupportsStats, ITransportInfo { private const string HeadersAttributeName = "fnd.headers"; private const string EncodingAttributeName = "fnd.encoding"; @@ -58,13 +59,27 @@ public AwsMessageTransport(AwsMessageTransportOptions options) public AwsMessageTransport(string connectionString) : this(AwsMessageTransportOptions.FromConnectionString(connectionString)) { } + // Capabilities differ by role: SQS queues take a native DelaySeconds (15-minute cap), SNS topics have no native + // delay at all — a delayed publish must route through the runtime-store fallback, never silently drop the delay. + // The 256 KB body limit applies to both services. + private static readonly TransportCapabilities _queueCapabilities = new() + { + DelayedDelivery = true, + MaxDeliveryDelay = TimeSpan.FromMinutes(15), // SQS DelaySeconds maximum + MaxMessageBytes = 262144 // 256 KB SQS limit + }; + + private static readonly TransportCapabilities _topicCapabilities = new() + { + MaxMessageBytes = 262144 // 256 KB SNS limit + }; + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; - public OrderingGuarantee Ordering => OrderingGuarantee.None; public IReadOnlySet SupportedRoles => _supportedRoles; - public int? MaxBatchSize => null; // sends are issued per message - public long? MaxMessageBytes => 262144; // 256 KB SQS/SNS limit - public TimeSpan? MaxDeliveryDelay => TimeSpan.FromMinutes(15); // SQS DelaySeconds maximum + public TransportCapabilities GetCapabilities(DestinationRole role) => + role == DestinationRole.Topic ? _topicCapabilities : _queueCapabilities; + public TimeSpan? MaxRedeliveryDelay => TimeSpan.FromHours(12); // SQS ChangeMessageVisibility maximum public TimeSpan? MaxVisibilityTimeout => TimeSpan.FromHours(12); // SQS visibility maximum @@ -80,6 +95,12 @@ public async Task SendAsync(string destination, IReadOnlyList DateTimeOffset.UtcNow) + throw new NotSupportedException($"Transport \"{nameof(AwsMessageTransport)}\" does not support delayed delivery for Topic destinations (SNS has no native delay). Register a job runtime store so delayed publishes use the scheduled-dispatch fallback."); + string topicArn = await ResolveTopicArnAsync(destination, ct).ConfigureAwait(false); foreach (var message in messages) { diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index 1ac2c4153..b5fd141ea 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -51,11 +51,13 @@ public RedisStreamsMessageTransport(RedisStreamsMessageTransportOptions options) _consumer = !String.IsNullOrEmpty(options.ConsumerName) ? options.ConsumerName : $"c-{Guid.NewGuid():N}"[..16]; } + // Streams append FIFO; there is no native priority, per-message expiration, or delayed delivery (delays route + // through the runtime-store fallback), and no broker-imposed size or batch limits. + private static readonly TransportCapabilities _capabilities = new() { Ordering = OrderingGuarantee.Fifo }; + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; - public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; public IReadOnlySet SupportedRoles => _supportedRoles; - public int? MaxBatchSize => null; - public long? MaxMessageBytes => null; + public TransportCapabilities GetCapabilities(DestinationRole role) => _capabilities; public TimeSpan? MaxRedeliveryDelay => null; // lease is tracked in Redis, so any delay is honored public TimeSpan? MaxVisibilityTimeout => null; @@ -65,6 +67,12 @@ public async Task SendAsync(string destination, IReadOnlyList _timeProvider.GetUtcNow()) + throw new NotSupportedException($"Transport \"{nameof(RedisStreamsMessageTransport)}\" does not support native delayed delivery. Register a job runtime store so delayed sends use the scheduled-dispatch fallback."); + // The stream IS the queue/topic; subscriptions read it through their own group. The caller-stated role picks // the stream namespace so a queue and a topic sharing a route name never cross-deliver. RedisKey streamKey = options.DestinationRole == DestinationRole.Topic ? TopicStreamKey(destination) : QueueStreamKey(destination); diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 2dd343bf2..01dc9e081 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -59,7 +59,7 @@ public virtual async Task CanSendAndReceiveBatchAsync() // Only assert positional FIFO order when the transport actually guarantees ordering; a best-effort // (OrderingGuarantee.None) transport may legitimately deliver out of order. - if (transport is not ITransportInfo { Ordering: OrderingGuarantee.None }) + if (GetCapabilities(transport, DestinationRole.Queue).Ordering != OrderingGuarantee.None) { Assert.Equal("one", ReadBody(entries[0])); Assert.Equal("two", ReadBody(entries[1])); @@ -214,13 +214,51 @@ await EnsureAsync(transport, } } + [Fact] + public virtual async Task SendAsync_ToTopic_WithDeliverAt_WithoutNativeDelay_ThrowsAsync() + { + var transport = CreateTransport(); + if (transport is null) + { + Assert.Skip("No transport configured."); + return; + } + + if (transport is ITransportInfo { SupportedRoles: { } roles } && !roles.Contains(DestinationRole.Topic)) + { + Assert.Skip("Transport does not support topic destinations."); + return; + } + + if (GetCapabilities(transport, DestinationRole.Topic).DelayedDelivery) + { + Assert.Skip("Transport honors delayed delivery natively for topics; nothing to refuse."); + return; + } + + try + { + // A transport that cannot honor DeliverAt for a role must refuse it, never publish immediately and + // silently drop the delay — the core only routes a delayed send here when the role advertises the + // capability, so acceptance would mean a lost delay (the AWS SNS delayed-publish bug shape). + await Assert.ThrowsAsync(() => transport.SendAsync("delayed-topic", + [CreateMessage("later")], + new TransportSendOptions { DestinationRole = DestinationRole.Topic, DeliverAt = DateTimeOffset.UtcNow.AddMinutes(5) }, + TestCancellationToken)); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + [Fact] public virtual async Task ReceiveAsync_RespectsPriorityAsync() { var transport = CreateTransport(); - if (transport is not ISupportsPull pull || transport is not ISupportsPriority) + if (transport is not ISupportsPull pull || !GetCapabilities(transport, DestinationRole.Queue).Priority) { - Assert.Skip("Transport does not support pull receive with priority (ISupportsPull + ISupportsPriority)."); + Assert.Skip("Transport does not support pull receive with queue priority (ISupportsPull + Priority capability)."); return; } @@ -255,9 +293,9 @@ public virtual async Task ReceiveAsync_RespectsPriorityAsync() public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() { var transport = CreateTransport(); - if (transport is not ISupportsPull pull || transport is not ISupportsDelayedDelivery) + if (transport is not ISupportsPull pull || !GetCapabilities(transport, DestinationRole.Queue).DelayedDelivery) { - Assert.Skip("Transport does not support pull receive with delayed delivery (ISupportsPull + ISupportsDelayedDelivery)."); + Assert.Skip("Transport does not support pull receive with native queue delayed delivery (ISupportsPull + DelayedDelivery capability)."); return; } @@ -314,9 +352,9 @@ public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsync() { var transport = CreateTransport(); - if (transport is not ISupportsPull pull || transport is not ISupportsExpiration || transport is not ISupportsStats stats) + if (transport is not ISupportsPull pull || !GetCapabilities(transport, DestinationRole.Queue).Expiration || transport is not ISupportsStats stats) { - Assert.Skip("Transport does not support pull receive with expiration and stats (ISupportsPull + ISupportsExpiration + ISupportsStats)."); + Assert.Skip("Transport does not support pull receive with expiration and stats (ISupportsPull + Expiration capability + ISupportsStats)."); return; } @@ -582,6 +620,11 @@ private async Task AssertQueueDrainedAsync(ISupportsStats stats, string destinat Assert.Equal(0, current.Working); } + private static TransportCapabilities GetCapabilities(IMessageTransport transport, DestinationRole role) + { + return transport is ITransportInfo info ? info.GetCapabilities(role) : TransportCapabilities.None; + } + private static async Task EnsureAsync(IMessageTransport transport, params DestinationDeclaration[] declarations) { if (transport is ISupportsProvisioning provisioning) diff --git a/src/Foundatio.Testing/RecordingMessageTransport.cs b/src/Foundatio.Testing/RecordingMessageTransport.cs index 868a991a7..007ef1b16 100644 --- a/src/Foundatio.Testing/RecordingMessageTransport.cs +++ b/src/Foundatio.Testing/RecordingMessageTransport.cs @@ -14,8 +14,8 @@ namespace Foundatio.Messaging.Testing; /// can detect quiescence. /// internal sealed class RecordingMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, - ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, ISupportsPriority, - ISupportsExpiration, ISupportsProvisioning, ITransportInfo + ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, + ISupportsProvisioning, ITransportInfo { // A delayed redelivery lives only in the inner transport's timer until it fires — neither queued nor in flight — // so idle detection would report quiescent while a retry is pending. Give the timer this long past its due time to @@ -45,10 +45,8 @@ public RecordingMessageTransport(TimeProvider? timeProvider = null) public IReadOnlyList DeadLettered => [.. _deadLettered]; public DeliveryGuarantee DeliveryGuarantee => _inner.DeliveryGuarantee; - public OrderingGuarantee Ordering => _inner.Ordering; public IReadOnlySet SupportedRoles => _inner.SupportedRoles; - public int? MaxBatchSize => _inner.MaxBatchSize; - public long? MaxMessageBytes => _inner.MaxMessageBytes; + public TransportCapabilities GetCapabilities(DestinationRole role) => _inner.GetCapabilities(role); public TimeSpan? MaxVisibilityTimeout => _inner.MaxVisibilityTimeout; public TimeSpan? MaxRedeliveryDelay => _inner.MaxRedeliveryDelay; diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index 8494c8219..eac68c589 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -10,10 +10,19 @@ namespace Foundatio.Messaging; -public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, ISupportsPriority, ISupportsExpiration, ISupportsProvisioning, ITransportInfo +public sealed class InMemoryMessageTransport : IMessageTransport, ISupportsPull, ISupportsPush, ISupportsVisibilityTimeout, ISupportsDeadLetter, ISupportsRedeliveryDelay, ISupportsLockRenewal, ISupportsStats, ISupportsProvisioning, ITransportInfo { private static readonly TimeSpan _defaultLockRenewal = TimeSpan.FromMinutes(1); + // Priority and expiration are honored on every role; there is no native delayed delivery (delays route through + // the runtime-store fallback) and no broker-imposed size or batch limits. + private static readonly TransportCapabilities _capabilities = new() + { + Priority = true, + Expiration = true, + Ordering = OrderingGuarantee.Fifo + }; + private static readonly IReadOnlySet _supportedRoles = new HashSet { DestinationRole.Queue, @@ -36,10 +45,9 @@ public InMemoryMessageTransport(TimeProvider? timeProvider = null) } public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; - public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; public IReadOnlySet SupportedRoles => _supportedRoles; - public int? MaxBatchSize => null; - public long? MaxMessageBytes => null; + + public TransportCapabilities GetCapabilities(DestinationRole role) => _capabilities; // The in-memory transport has no broker-imposed ceiling on visibility or redelivery delay. public TimeSpan? MaxVisibilityTimeout => null; diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 461c2ad75..46fcb4328 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -110,7 +110,7 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc public async Task SendAsync(ScheduledDispatchKind kind, Type messageType, object message, MessageEnvelopeOptions options, string destination, Func? ensureDestination, CancellationToken cancellationToken) { ThrowIfDisposed(); - ValidateCapabilities(options.Priority, options.TimeToLive); + ValidateCapabilities(RoleFor(kind), options.Priority, options.TimeToLive); var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; string messageId = Guid.NewGuid().ToString("N"); @@ -131,7 +131,7 @@ public async Task SendAsync(ScheduledDispatchKind kind, Type messageType public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable messages, Type? declaredType, MessageEnvelopeOptions options, Func resolveDestination, Func? ensureDestination, CancellationToken cancellationToken) { ThrowIfDisposed(); - ValidateCapabilities(options.Priority, options.TimeToLive); + ValidateCapabilities(RoleFor(kind), options.Priority, options.TimeToLive); var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; var grouped = new Dictionary>(StringComparer.Ordinal); @@ -595,13 +595,22 @@ private TransportSendOptions BuildSendOptions(MessageEnvelopeOptions options) }; } - private void ValidateCapabilities(MessagePriority priority, TimeSpan? timeToLive) + // Capabilities are role-aware: the same transport can honor a feature on queues but not topics (SQS DelaySeconds + // vs. SNS publish), so every send-path decision asks for the destination role it is actually targeting. + private TransportCapabilities CapabilitiesFor(DestinationRole role) { - if (priority != MessagePriority.Normal && _transport is not ISupportsPriority) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority."); + return _transport is ITransportInfo info ? info.GetCapabilities(role) : TransportCapabilities.None; + } + + private void ValidateCapabilities(DestinationRole role, MessagePriority priority, TimeSpan? timeToLive) + { + var capabilities = CapabilitiesFor(role); + + if (priority != MessagePriority.Normal && !capabilities.Priority) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message priority for {role} destinations."); - if (timeToLive is not null && _transport is not ISupportsExpiration) - throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration."); + if (timeToLive is not null && !capabilities.Expiration) + throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration for {role} destinations."); } private async Task TryScheduleAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) @@ -634,25 +643,27 @@ private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out if (options.DeliverAt is null || dueUtc <= now) return false; - // A transport can deliver natively only up to its advertised maximum; a delay longer than the broker supports + // A destination can deliver natively only up to its advertised maximum; a delay longer than the broker supports // (e.g. SQS caps DelaySeconds at 15 minutes) must route through the durable runtime store rather than be - // silently truncated to the broker's ceiling. - if (_transport is ISupportsDelayedDelivery delayed && (delayed.MaxDeliveryDelay is not { } max || dueUtc - now <= max)) + // silently truncated to the broker's ceiling. The check is per destination role: a transport whose queues take + // a native delay may still have topics that cannot (SQS vs. SNS), and those publishes must fall back too. + var capabilities = CapabilitiesFor(options.DestinationRole); + if (capabilities.DelayedDelivery && (capabilities.MaxDeliveryDelay is not { } max || dueUtc - now <= max)) return false; if (_runtimeStore is null) - throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" (within its supported maximum) or a registered job runtime store.", null); + throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" for {options.DestinationRole} destinations (within its supported maximum) or a registered job runtime store.", null); return true; } private async Task> SendChunkedAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { - var info = _transport as ITransportInfo; + var capabilities = CapabilitiesFor(options.DestinationRole); // Enforce a transport-declared maximum message size up front with a clear error, rather than letting an opaque // broker rejection surface mid-send (the limit is advertised, so honor it). - if (info?.MaxMessageBytes is { } maxBytes) + if (capabilities.MaxMessageBytes is { } maxBytes) { foreach (var message in messages) { @@ -662,7 +673,7 @@ private async Task> SendChunkedAsync(string destin } // Respect a transport-declared maximum batch size by splitting oversized sends into chunks. - int? maxBatchSize = info?.MaxBatchSize; + int? maxBatchSize = capabilities.MaxBatchSize; if (maxBatchSize is not { } limit || limit <= 0 || messages.Count <= limit) { var result = await _transport.SendAsync(destination, messages, options, cancellationToken).AnyContext(); diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 72aacba40..11f53997c 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -151,13 +151,54 @@ public sealed record PushOptions public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(1); } +/// +/// The capability and limit facts a transport advertises for one . Capabilities vary by +/// role on real brokers (SQS queues take DelaySeconds; SNS topics have no native delay), so the core asks per role +/// via rather than reading transport-wide flags. Anything not advertised +/// here is treated as unsupported: the core validates, falls back, or throws instead of letting the broker silently +/// drop a requested behavior. +/// +public sealed record TransportCapabilities +{ + /// Capabilities of a transport (or role) that advertises nothing: every feature routes through core fallbacks or fails validation. + public static readonly TransportCapabilities None = new(); + + /// The destination honors natively. + public bool DelayedDelivery { get; init; } + + /// + /// The longest delivery delay honored natively when is true (e.g. SQS caps + /// DelaySeconds at 15 minutes); null means unbounded. A send scheduled further out is routed through the + /// runtime-store fallback instead of being silently truncated to the broker's ceiling. + /// + public TimeSpan? MaxDeliveryDelay { get; init; } + + /// The destination honors . + public bool Priority { get; init; } + + /// The destination honors per-message expiration (). + public bool Expiration { get; init; } + + public OrderingGuarantee Ordering { get; init; } = OrderingGuarantee.None; + + /// Maximum messages per call; null means unbounded. The core chunks larger sends. + public int? MaxBatchSize { get; init; } + + /// Maximum message body size in bytes; null means unbounded. The core rejects oversized messages up front. + public long? MaxMessageBytes { get; init; } +} + public interface ITransportInfo { DeliveryGuarantee DeliveryGuarantee { get; } - OrderingGuarantee Ordering { get; } IReadOnlySet SupportedRoles { get; } - int? MaxBatchSize { get; } - long? MaxMessageBytes { get; } + + /// + /// The capabilities and limits this transport honors for destinations of the given role. Must be side-effect free + /// and cheap; the core consults it on every send-path decision (native delay vs. runtime-store fallback, + /// priority/expiration validation, size and batch limits). + /// + TransportCapabilities GetCapabilities(DestinationRole role); } public interface IMessageTransport : IAsyncDisposable @@ -216,18 +257,6 @@ public interface ISupportsStats : IMessageTransport Task GetStatsAsync(string destination, CancellationToken ct = default); } -public interface ISupportsPriority : IMessageTransport { } - -public interface ISupportsDelayedDelivery : IMessageTransport -{ - // The longest delivery delay the transport can honor natively (e.g. SQS caps DelaySeconds at 15 minutes). - // Null means unbounded. A send scheduled further out than this is routed through the runtime-store fallback - // instead of being silently truncated to the broker's maximum. - TimeSpan? MaxDeliveryDelay { get; } -} - -public interface ISupportsExpiration : IMessageTransport { } - public interface ISupportsProvisioning : IMessageTransport { Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct = default); diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index 6b5166c5d..145a4a5cf 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -12,7 +12,7 @@ namespace Foundatio.Redis.Tests; /// /// End-to-end tests that wire the real messaging core / CRON scheduler on top of the Redis /// and exercise the two paths the store exists to support but that the primitive-level conformance suite does not cover: -/// (1) a delayed send whose delay exceeds the transport's being +/// (1) a delayed send whose delay exceeds the transport's being /// durably stored in Redis and drained by the dispatch pump when due, and (2) CRON occurrences being materialized, run, /// retried/dead-lettered, and stale-reclaimed through Redis. /// @@ -242,7 +242,7 @@ private sealed class PreviewWorkItem // Minimal pull transport with a configurable native delayed-delivery ceiling, so a delay beyond the cap is forced // through the runtime store (mirrors the fixture used by the in-memory MessageBus tests). - private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ISupportsDelayedDelivery + private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ITransportInfo { private readonly Queue _entries = new(); @@ -252,6 +252,12 @@ private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, IS public int SendCount { get; private set; } public TransportSendOptions? LastSendOptions { get; private set; } + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; + + public TransportCapabilities GetCapabilities(DestinationRole role) => + new() { DelayedDelivery = true, MaxDeliveryDelay = MaxDeliveryDelay }; + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendCount += messages.Count; diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index ab54d5bd7..89b954a6c 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -348,6 +348,33 @@ await pubSub.PublishBatchAsync(new object[] } + [Fact] + public async Task PublishAsync_WithDelay_OnTopicWithoutNativeDelay_RoutesThroughRuntimeStoreAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + + // The AWS SQS/SNS shape: queues honor a native delay (15-minute cap) but topics have none. A delayed publish + // within the QUEUE ceiling must still route through the runtime store — deciding by transport-wide capability + // would take the native path and the broker would silently drop the delay. + var store = new InMemoryJobRuntimeStore(); + await using var transport = new RoleSplitDelayTransport(queueMaxDelay: TimeSpan.FromMinutes(15)); + await using var pubSub = new MessageBus(transport, new MessageBusOptions { RuntimeStore = store }); + var processor = CreateDispatchProcessor(store, transport); + + await pubSub.PublishAsync(new PreviewEvent { Data = "later" }, new MessagePublishOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + + Assert.Equal(0, transport.SendCount); + Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(10), cancellationToken: cancellationToken)); + Assert.Equal(1, transport.SendCount); + Assert.Equal(DestinationRole.Topic, transport.LastSendOptions?.DestinationRole); + Assert.Null(transport.LastSendOptions?.DeliverAt); // the store dispatches it as due; the delay is spent, not forwarded + + // A delayed QUEUE send within the same transport's queue ceiling still uses the native path. + await pubSub.SendAsync(new PreviewEvent { Data = "soon" }, new MessageSendOptions { Delay = TimeSpan.FromMinutes(5) }, cancellationToken); + Assert.Equal(2, transport.SendCount); + Assert.NotNull(transport.LastSendOptions?.DeliverAt); + } + private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, string destination, long expected, CancellationToken cancellationToken) { var deadline = DateTimeOffset.UtcNow.AddSeconds(2); @@ -371,6 +398,47 @@ private static JobScheduleProcessor CreateDispatchProcessor(IJobRuntimeStore sto return new JobScheduleProcessor(new InMemoryJobScheduler(), store, worker, nodeId: "node-a", transport: transport); } + // Mirrors AWS SQS/SNS: native delayed delivery on queues only. Topic sends with a future DeliverAt throw, so a + // silent delay drop cannot hide. + private sealed class RoleSplitDelayTransport : IMessageTransport, ISupportsPull, ITransportInfo + { + private readonly TimeSpan _queueMaxDelay; + + public RoleSplitDelayTransport(TimeSpan queueMaxDelay) => _queueMaxDelay = queueMaxDelay; + + public int SendCount { get; private set; } + public TransportSendOptions? LastSendOptions { get; private set; } + + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public IReadOnlySet SupportedRoles => + new HashSet { DestinationRole.Queue, DestinationRole.Topic, DestinationRole.Subscription }; + + public TransportCapabilities GetCapabilities(DestinationRole role) => role == DestinationRole.Topic + ? TransportCapabilities.None + : new TransportCapabilities { DelayedDelivery = true, MaxDeliveryDelay = _queueMaxDelay }; + + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + if (options.DestinationRole == DestinationRole.Topic && options.DeliverAt is { } deliverAt && deliverAt > DateTimeOffset.UtcNow) + throw new NotSupportedException("Topics have no native delayed delivery."); + + SendCount += messages.Count; + LastSendOptions = options; + var items = new SendItemResult[messages.Count]; + for (int i = 0; i < messages.Count; i++) + items[i] = new SendItemResult { MessageId = messages[i].MessageId ?? Guid.NewGuid().ToString("N") }; + + return Task.FromResult(new SendResult { Items = items }); + } + + public Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + => Task.FromResult>([]); + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + private interface IGroupedEvent { } diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index c0479a53f..23898f440 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -983,11 +983,13 @@ public BatchLimitTransport(int maxBatchSize, long? maxMessageBytes = null) public List SendBatchSizes { get; } = new(); public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; - public OrderingGuarantee Ordering => OrderingGuarantee.Fifo; public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; public int? MaxBatchSize { get; } public long? MaxMessageBytes { get; } + public TransportCapabilities GetCapabilities(DestinationRole role) => + new() { Ordering = OrderingGuarantee.Fifo, MaxBatchSize = MaxBatchSize, MaxMessageBytes = MaxMessageBytes }; + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendBatchSizes.Add(messages.Count); @@ -1003,7 +1005,7 @@ public Task SendAsync(string destination, IReadOnlyList ValueTask.CompletedTask; } - private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ISupportsDelayedDelivery + private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, ITransportInfo { private readonly Queue _entries = new(); @@ -1016,6 +1018,12 @@ public CappedDelayTransport(TimeSpan? maxDeliveryDelay) public int SendCount { get; private set; } public TransportSendOptions? LastSendOptions { get; private set; } + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; + + public TransportCapabilities GetCapabilities(DestinationRole role) => + new() { DelayedDelivery = true, MaxDeliveryDelay = MaxDeliveryDelay }; + public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendCount += messages.Count; From e9e2c0acf32ae288b6c272d3a22f3ec8b58dcff4 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:22:43 -0500 Subject: [PATCH 49/57] One canonical DestinationAddress across the whole transport contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Destination identity was spread across bare strings, a separate DestinationRole on the send options, DestinationDeclaration Name/Role/Source triples, and the formatted "{topic}/{subscription}" SubscriptionAddress convention that transports re-parsed at receive/settle/delete time. The same subscription was even declared under two different names — the routing topology declared the bare subscription name while the runtime subscribe path declared the formatted string, so Redis consumer groups created by IMessageTopology.EnsureAsync and by SubscribeAsync would not have agreed. DestinationAddress (Name + Role + owning Topic for subscriptions) is now the one identity on every path: send, receive, subscribe, settlement entries, stats, dead-letter reads, and provisioning (declare/exists/delete). The role rides with the address, so TransportSendOptions.DestinationRole is gone, the SubscriptionAddress parse-and-prefix convention is deleted, transports derive physical names structurally (Redis groups are named by the bare subscription name scoped to the topic stream; AWS keeps EncodeResourceName(address.Key), so no physical resources change), and topology and runtime declarations are equal by construction. ScheduledDispatchState splits the overloaded Destination field into a typed address for message dispatches and JobName for CRON occurrences. New ProvisioningLifecycle conformance fact covers ensure/exists/idempotent-re-ensure/delete. (Review feedback #3.) Co-Authored-By: Claude Fable 5 --- src/Foundatio.Aws/AwsMessageTransport.cs | 88 ++++---- .../Messaging/RedisStreamsMessageTransport.cs | 133 ++++++------ src/Foundatio.Redis/RedisJobRuntimeStore.cs | 9 +- .../Jobs/JobRuntimeStoreConformanceTests.cs | 17 +- .../MessageTransportConformanceTests.cs | 199 ++++++++++++------ .../RecordingMessageTransport.cs | 40 ++-- src/Foundatio/Jobs/JobRuntime.cs | 8 +- src/Foundatio/Jobs/JobScheduler.cs | 7 +- .../Messaging/InMemoryMessageTransport.cs | 120 +++++------ src/Foundatio/Messaging/MessageBus.cs | 46 ++-- src/Foundatio/Messaging/MessageClientCore.cs | 106 +++++----- src/Foundatio/Messaging/MessageRouting.cs | 19 +- src/Foundatio/Messaging/MessageTopology.cs | 11 +- src/Foundatio/Messaging/MessageTransport.cs | 78 +++++-- .../Messaging/SubscriptionAddress.cs | 43 ---- .../AwsMessageTransportTests.cs | 12 +- .../RedisJobStoreIntegrationTests.cs | 6 +- .../RedisStreamsTransportIntegrationTests.cs | 30 +-- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 4 +- .../Foundatio.Tests/Jobs/JobSchedulerTests.cs | 8 +- .../Messaging/FailureHandlingTests.cs | 9 +- .../InMemoryMessageTransportTests.cs | 22 +- .../Foundatio.Tests/Messaging/PubSubTests.cs | 24 ++- .../Queue/BasicQueueTransport.cs | 18 +- .../Queue/MessageQueueTests.cs | 55 ++--- 25 files changed, 583 insertions(+), 529 deletions(-) delete mode 100644 src/Foundatio/Messaging/SubscriptionAddress.cs diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index ba48fedff..c9e6279dc 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -47,7 +47,6 @@ public sealed class AwsMessageTransport : IMessageTransport, ISupportsPull, ISup private readonly Lazy _sns; private readonly ConcurrentDictionary _queueUrls = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _topicArns = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary _roles = new(StringComparer.Ordinal); private int _isDisposed; public AwsMessageTransport(AwsMessageTransportOptions options) @@ -83,17 +82,17 @@ public TransportCapabilities GetCapabilities(DestinationRole role) => public TimeSpan? MaxRedeliveryDelay => TimeSpan.FromHours(12); // SQS ChangeMessageVisibility maximum public TimeSpan? MaxVisibilityTimeout => TimeSpan.FromHours(12); // SQS visibility maximum - public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public async Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); ArgumentNullException.ThrowIfNull(messages); var items = new List(messages.Count); - // The caller states the destination role, so route without inferring: a topic publishes to SNS, anything else + // The address states the destination role, so route without inferring: a topic publishes to SNS, anything else // sends to an SQS queue. - if (options.DestinationRole == DestinationRole.Topic) + if (destination.Role == DestinationRole.Topic) { // SNS has no native delayed publish. The core routes delayed topic publishes through the runtime-store // fallback (topic capabilities advertise no DelayedDelivery), so a DeliverAt reaching here is a contract @@ -101,7 +100,7 @@ public async Task SendAsync(string destination, IReadOnlyList DateTimeOffset.UtcNow) throw new NotSupportedException($"Transport \"{nameof(AwsMessageTransport)}\" does not support delayed delivery for Topic destinations (SNS has no native delay). Register a job runtime store so delayed publishes use the scheduled-dispatch fallback."); - string topicArn = await ResolveTopicArnAsync(destination, ct).ConfigureAwait(false); + string topicArn = await ResolveTopicArnAsync(destination.Name, ct).ConfigureAwait(false); foreach (var message in messages) { var (body, encoding) = EncodeBody(message); @@ -139,15 +138,15 @@ public async Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) { return ReceiveAsync(source, request, _options.DefaultVisibilityTimeout, ct); } - public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + public async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(source); ArgumentNullException.ThrowIfNull(request); string queueUrl = await ResolveQueueUrlAsync(source, ct).ConfigureAwait(false); @@ -219,51 +218,50 @@ public async Task EnsureAsync(IReadOnlyList declarations foreach (var declaration in declarations) { - switch (declaration.Role) + switch (declaration.Address.Role) { case DestinationRole.Topic: - await ResolveTopicArnAsync(declaration.Name, ct).ConfigureAwait(false); + await ResolveTopicArnAsync(declaration.Address.Name, ct).ConfigureAwait(false); break; case DestinationRole.Subscription: case DestinationRole.Binding: - await EnsureSubscriptionAsync(declaration.Name, declaration.Source, ct).ConfigureAwait(false); + await EnsureSubscriptionAsync(declaration.Address, ct).ConfigureAwait(false); break; default: - await ResolveQueueUrlAsync(declaration.Name, ct).ConfigureAwait(false); + await ResolveQueueUrlAsync(declaration.Address, ct).ConfigureAwait(false); break; } } } - public async Task DeleteAsync(string name, CancellationToken ct) + public async Task DeleteAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - if (_roles.TryGetValue(name, out var role) && role == DestinationRole.Topic) + if (destination.Role == DestinationRole.Topic) { - if (_topicArns.TryRemove(name, out string? arn)) + if (_topicArns.TryRemove(destination.Name, out string? arn)) await _sns.Value.DeleteTopicAsync(arn, ct).ConfigureAwait(false); - } - else if (_queueUrls.TryRemove(name, out string? url)) - { - await _sqs.Value.DeleteQueueAsync(url, ct).ConfigureAwait(false); + return; } - _roles.TryRemove(name, out _); + // Queue and subscription destinations are both backed by an SQS queue named from the address key. + if (_queueUrls.TryRemove(destination.Key, out string? url)) + await _sqs.Value.DeleteQueueAsync(url, ct).ConfigureAwait(false); } - public async Task ExistsAsync(string name, CancellationToken ct) + public async Task ExistsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - if (_roles.TryGetValue(name, out var role) && role == DestinationRole.Topic) - return _topicArns.ContainsKey(name); + if (destination.Role == DestinationRole.Topic) + return _topicArns.ContainsKey(destination.Name); try { - await _sqs.Value.GetQueueUrlAsync(ResourceName(name), ct).ConfigureAwait(false); + await _sqs.Value.GetQueueUrlAsync(ResourceName(destination.Key), ct).ConfigureAwait(false); return true; } catch (QueueDoesNotExistException) @@ -272,7 +270,7 @@ public async Task ExistsAsync(string name, CancellationToken ct) } } - public async Task GetStatsAsync(string destination, CancellationToken ct) + public async Task GetStatsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); string queueUrl = await ResolveQueueUrlAsync(destination, ct).ConfigureAwait(false); @@ -302,15 +300,14 @@ public async ValueTask DisposeAsync() await ValueTask.CompletedTask.ConfigureAwait(false); } - private async Task EnsureSubscriptionAsync(string subscriptionName, string? topicName, CancellationToken ct) + private async Task EnsureSubscriptionAsync(DestinationAddress address, CancellationToken ct) { - string queueUrl = await ResolveQueueUrlAsync(subscriptionName, ct).ConfigureAwait(false); - _roles[subscriptionName] = DestinationRole.Subscription; + string queueUrl = await ResolveQueueUrlAsync(address, ct).ConfigureAwait(false); - if (String.IsNullOrEmpty(topicName)) + if (String.IsNullOrEmpty(address.Topic)) return; - string topicArn = await ResolveTopicArnAsync(topicName, ct).ConfigureAwait(false); + string topicArn = await ResolveTopicArnAsync(address.Topic, ct).ConfigureAwait(false); string queueArn = await GetQueueArnAsync(queueUrl, ct).ConfigureAwait(false); // Allow the topic to deliver to the queue, then subscribe with raw delivery so the SQS body/attributes match a @@ -331,24 +328,26 @@ await _sns.Value.SubscribeAsync(new SubscribeRequest }, ct).ConfigureAwait(false); } - private async Task ResolveQueueUrlAsync(string name, CancellationToken ct) + // Queue and subscription destinations are both backed by an SQS queue whose logical name is the address key + // (Name for queues, "topic/subscription" for subscriptions), so provisioning and every runtime path resolve the + // same physical queue from the same address. + private async Task ResolveQueueUrlAsync(DestinationAddress address, CancellationToken ct) { - if (_queueUrls.TryGetValue(name, out string? cached)) + string key = address.Key; + if (_queueUrls.TryGetValue(key, out string? cached)) return cached; - string resourceName = ResourceName(name); + string resourceName = ResourceName(key); try { var response = await _sqs.Value.GetQueueUrlAsync(resourceName, ct).ConfigureAwait(false); - _queueUrls[name] = response.QueueUrl; - _roles.TryAdd(name, DestinationRole.Queue); + _queueUrls[key] = response.QueueUrl; return response.QueueUrl; } catch (QueueDoesNotExistException) when (_options.AutoCreateDestinations) { var response = await _sqs.Value.CreateQueueAsync(new CreateQueueRequest { QueueName = resourceName }, ct).ConfigureAwait(false); - _queueUrls[name] = response.QueueUrl; - _roles.TryAdd(name, DestinationRole.Queue); + _queueUrls[key] = response.QueueUrl; return response.QueueUrl; } } @@ -361,15 +360,14 @@ private async Task ResolveTopicArnAsync(string name, CancellationToken c // CreateTopic is idempotent and returns the ARN of an existing topic with the same name. var response = await _sns.Value.CreateTopicAsync(new CreateTopicRequest { Name = ResourceName(name) }, ct).ConfigureAwait(false); _topicArns[name] = response.TopicArn; - _roles[name] = DestinationRole.Topic; return response.TopicArn; } // SQS queue / SNS topic names allow only [A-Za-z0-9_-] (max 80 chars). Most logical names already conform, but a - // pub/sub subscription's destination is the opaque "topic/subscription" key (see SubscriptionAddress) which - // contains '/'. Encode any illegal name deterministically and collision-free — sanitize, then append a short - // stable hash of the original — so EnsureAsync/ReceiveAsync/CompleteAsync all resolve the same queue from the same - // logical name. Legal names are returned unchanged (no behavior change for plain queues/topics). + // subscription's key (see DestinationAddress.Key) is the opaque "topic/subscription" form which contains '/'. + // Encode any illegal name deterministically and collision-free — sanitize, then append a short stable hash of the + // original — so EnsureAsync/ReceiveAsync/CompleteAsync all resolve the same queue from the same logical name. + // Legal names are returned unchanged (no behavior change for plain queues/topics). private string ResourceName(string logicalName) => EncodeResourceName(_options.ResourcePrefix, logicalName); private static string EncodeResourceName(string prefix, string logicalName) diff --git a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs index b5fd141ea..72df233b5 100644 --- a/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs +++ b/src/Foundatio.Redis/Messaging/RedisStreamsMessageTransport.cs @@ -36,8 +36,6 @@ public sealed class RedisStreamsMessageTransport : IMessageTransport, ISupportsP private readonly TimeProvider _timeProvider; private readonly string _prefix; private readonly string _consumer; - // Logical destination name -> resolved (stream key, consumer group, group-create position). Populated by EnsureAsync. - private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _ensuredGroups = new(StringComparer.Ordinal); private int _isDisposed; @@ -61,10 +59,10 @@ public RedisStreamsMessageTransport(RedisStreamsMessageTransportOptions options) public TimeSpan? MaxRedeliveryDelay => null; // lease is tracked in Redis, so any delay is honored public TimeSpan? MaxVisibilityTimeout => null; - public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public async Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); ArgumentNullException.ThrowIfNull(messages); // Streams have no native delayed delivery; the core routes delayed sends through the runtime-store fallback @@ -73,9 +71,9 @@ public async Task SendAsync(string destination, IReadOnlyList _timeProvider.GetUtcNow()) throw new NotSupportedException($"Transport \"{nameof(RedisStreamsMessageTransport)}\" does not support native delayed delivery. Register a job runtime store so delayed sends use the scheduled-dispatch fallback."); - // The stream IS the queue/topic; subscriptions read it through their own group. The caller-stated role picks - // the stream namespace so a queue and a topic sharing a route name never cross-deliver. - RedisKey streamKey = options.DestinationRole == DestinationRole.Topic ? TopicStreamKey(destination) : QueueStreamKey(destination); + // The stream IS the queue/topic; subscriptions read it through their own group. The address role picks the + // stream namespace so a queue and a topic sharing a route name never cross-deliver. + RedisKey streamKey = destination.Role == DestinationRole.Topic ? TopicStreamKey(destination.Name) : QueueStreamKey(destination.Name); var items = new List(messages.Count); foreach (var message in messages) { @@ -87,13 +85,13 @@ public async Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) => ReceiveAsync(source, request, _options.DefaultVisibilityTimeout, ct); - public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + public async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(source); ArgumentNullException.ThrowIfNull(request); var resolved = Resolve(source); @@ -118,7 +116,7 @@ public async Task> ReceiveAsync(string source, Rec } } - private async Task> PollOnceAsync(string source, ResolvedSource resolved, int max, long visibilityMs, CancellationToken ct) + private async Task> PollOnceAsync(DestinationAddress source, ResolvedSource resolved, int max, long visibilityMs, CancellationToken ct) { var result = new List(max); long nowMs = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); @@ -164,7 +162,7 @@ private async Task> PollOnceAsync(string source, ResolvedSo // Records the lease (sorted set) + owner token & delivery count (hash) for a just-delivered entry and projects it // into a TransportEntry whose Receipt carries everything needed to settle it. - private async Task TrackAsync(string source, ResolvedSource resolved, StreamEntry entry, int deliveries, long nowMs, long visibilityMs) + private async Task TrackAsync(DestinationAddress source, ResolvedSource resolved, StreamEntry entry, int deliveries, long nowMs, long visibilityMs) { string token = Guid.NewGuid().ToString("N"); await _db.HashSetAsync(MetaKey(resolved), entry.Id, $"{token}|{deliveries}").ConfigureAwait(false); @@ -230,10 +228,10 @@ await _db.StreamAddAsync(DeadKey(r.StreamKey), BuildFields(entry.Id, entry.Body, await ClearTrackingAsync(r).ConfigureAwait(false); } - public async Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + public async Task> ReceiveDeadLetteredAsync(DestinationAddress destination, ReceiveRequest request, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); ArgumentNullException.ThrowIfNull(request); RedisKey deadKey = DeadKey(Resolve(destination).StreamKey); @@ -261,86 +259,80 @@ public async Task EnsureAsync(IReadOnlyList declarations foreach (var declaration in declarations) { - switch (declaration.Role) + switch (declaration.Address.Role) { case DestinationRole.Topic: - // Topics are read through subscription groups; nothing to create until a subscription appears. The - // name must NOT be registered in _sources: a queue can share the route name, and receive-side - // resolution of the bare name must keep meaning the queue stream. Exists/delete are role-aware by - // probing both namespaces instead. - break; - case DestinationRole.Subscription: - case DestinationRole.Binding: - string topic = declaration.Source ?? declaration.Name; - var sub = new ResolvedSource(TopicStreamKey(topic), declaration.Name, "$"); - _sources[declaration.Name] = sub; - await EnsureGroupAsync(sub).ConfigureAwait(false); + // Topics are read through subscription groups; nothing to create until a subscription appears. break; default: - var queue = new ResolvedSource(QueueStreamKey(declaration.Name), _options.DefaultConsumerGroup, "0"); - _sources[declaration.Name] = queue; - await EnsureGroupAsync(queue).ConfigureAwait(false); + // Queue, subscription, and binding declarations all materialize as a consumer group on the stream + // the address resolves to. + await EnsureGroupAsync(Resolve(declaration.Address)).ConfigureAwait(false); break; } } } - public async Task DeleteAsync(string name, CancellationToken ct) + public async Task DeleteAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - // A subscription address deletes only that group's state (never the shared topic stream); a bare name deletes - // the name in both role namespaces, mirroring the in-memory transport. - if (SubscriptionAddress.TryParse(name, out string topic, out string subscription)) + // A subscription deletes only that group's state (never the shared topic stream — other subscriptions still + // read it); a queue or topic deletes its own stream and everything scoped to it. + if (destination.Role is DestinationRole.Subscription or DestinationRole.Binding) { - var sub = new ResolvedSource(TopicStreamKey(topic), subscription, "$"); + var sub = Resolve(destination); await _db.StreamDeleteConsumerGroupAsync(sub.StreamKey, sub.Group).ConfigureAwait(false); await _db.KeyDeleteAsync([LockKey(sub), MetaKey(sub)]).ConfigureAwait(false); - _sources.TryRemove(name, out _); _ensuredGroups.TryRemove(GroupKey(sub), out _); return; } - foreach (var resolved in (ResolvedSource[]) - [ - new ResolvedSource(QueueStreamKey(name), _options.DefaultConsumerGroup, "0"), - new ResolvedSource(TopicStreamKey(name), _options.DefaultConsumerGroup, "$") - ]) + var resolved = Resolve(destination); + + // Drop each consumer group's lease/meta state before the stream itself (topic streams can carry several). + if (await _db.KeyExistsAsync(resolved.StreamKey).ConfigureAwait(false)) { - // Drop each consumer group's lease/meta state before the stream itself (topic streams can carry several). - if (await _db.KeyExistsAsync(resolved.StreamKey).ConfigureAwait(false)) + foreach (var group in await _db.StreamGroupInfoAsync(resolved.StreamKey).ConfigureAwait(false)) { - foreach (var group in await _db.StreamGroupInfoAsync(resolved.StreamKey).ConfigureAwait(false)) - { - var groupSource = resolved with { Group = group.Name }; - await _db.KeyDeleteAsync([LockKey(groupSource), MetaKey(groupSource)]).ConfigureAwait(false); - _ensuredGroups.TryRemove(GroupKey(groupSource), out _); - } + var groupSource = resolved with { Group = group.Name }; + await _db.KeyDeleteAsync([LockKey(groupSource), MetaKey(groupSource)]).ConfigureAwait(false); + _ensuredGroups.TryRemove(GroupKey(groupSource), out _); } - - await _db.KeyDeleteAsync([resolved.StreamKey, DeadKey(resolved.StreamKey)]).ConfigureAwait(false); - _ensuredGroups.TryRemove(GroupKey(resolved), out _); } - _sources.TryRemove(name, out _); + await _db.KeyDeleteAsync([resolved.StreamKey, DeadKey(resolved.StreamKey)]).ConfigureAwait(false); + _ensuredGroups.TryRemove(GroupKey(resolved), out _); } - public async Task ExistsAsync(string name, CancellationToken ct) + public async Task ExistsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - if (SubscriptionAddress.TryParse(name, out string topic, out _)) - return await _db.KeyExistsAsync(TopicStreamKey(topic)).ConfigureAwait(false); + var resolved = Resolve(destination); + if (!await _db.KeyExistsAsync(resolved.StreamKey).ConfigureAwait(false)) + return false; - return await _db.KeyExistsAsync(QueueStreamKey(name)).ConfigureAwait(false) - || await _db.KeyExistsAsync(TopicStreamKey(name)).ConfigureAwait(false); + // A subscription exists when its consumer group exists on the topic stream; a queue/topic exists when its + // stream key does. + if (destination.Role is not (DestinationRole.Subscription or DestinationRole.Binding)) + return true; + + foreach (var group in await _db.StreamGroupInfoAsync(resolved.StreamKey).ConfigureAwait(false)) + { + if (String.Equals(group.Name, resolved.Group, StringComparison.Ordinal)) + return true; + } + + return false; } - public async Task GetStatsAsync(string destination, CancellationToken ct) + public async Task GetStatsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(destination); var resolved = Resolve(destination); // Probing stats must not create phantom streams/groups; a destination that doesn't exist yet is simply empty. @@ -403,19 +395,18 @@ private async Task EnsureGroupAsync(ResolvedSource resolved) } } - private ResolvedSource Resolve(string source) + // The address is structural, so the physical mapping is derived from it directly — topology declarations and the + // runtime resolve the SAME address to the SAME stream/group, with no registration cache to drift. A subscription is + // a consumer group on its owning topic's stream: the group is named by the bare subscription name and scoped by the + // topic stream key. A queue is its own stream read through the shared default group (competing consumers). + private ResolvedSource Resolve(DestinationAddress address) => address.Role switch { - if (_sources.TryGetValue(source, out var registered)) - return registered; - - // PubSub facade sources are "topic/subscription" (a consumer group on the topic stream); a bare name is a queue - // on the default group. Parse via the shared convention rather than re-deriving the split. - return SubscriptionAddress.TryParse(source, out string topic, out string subscription) - ? new ResolvedSource(TopicStreamKey(topic), subscription, "$") - : new ResolvedSource(QueueStreamKey(source), _options.DefaultConsumerGroup, "0"); - } + DestinationRole.Topic => new ResolvedSource(TopicStreamKey(address.Name), _options.DefaultConsumerGroup, "$"), + DestinationRole.Subscription or DestinationRole.Binding => new ResolvedSource(TopicStreamKey(address.Topic ?? address.Name), address.Name, "$"), + _ => new ResolvedSource(QueueStreamKey(address.Name), _options.DefaultConsumerGroup, "0") + }; - private TransportEntry ToEntry(string destination, ResolvedSource? resolved, StreamEntry entry, int deliveries, string token) + private TransportEntry ToEntry(DestinationAddress destination, ResolvedSource? resolved, StreamEntry entry, int deliveries, string token) { string? messageId = GetField(entry, "id"); var headers = MessageHeaders.DeserializeFromJson(GetField(entry, "h")); diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.cs index ab1589cae..3bddc6c9e 100644 --- a/src/Foundatio.Redis/RedisJobRuntimeStore.cs +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.cs @@ -457,7 +457,6 @@ private static HashEntry[] ToHash(ScheduledDispatchState dispatch) { new("dispatchId", dispatch.DispatchId), new("kind", dispatch.Kind.ToString()), - new("destination", dispatch.Destination), new("body", Convert.ToBase64String(dispatch.Body.Span)), new("headers", JsonSerializer.Serialize(headers)), new("options", JsonSerializer.Serialize(dispatch.Options)), @@ -465,6 +464,10 @@ private static HashEntry[] ToHash(ScheduledDispatchState dispatch) new("attempts", dispatch.Attempts) }; + // Destination (message dispatches) and JobName (job occurrences) are mutually exclusive; only the populated + // side is written so the read side can distinguish them by field presence. + if (dispatch.Destination is not null) entries.Add(new("destination", JsonSerializer.Serialize(dispatch.Destination))); + if (dispatch.JobName is not null) entries.Add(new("jobName", dispatch.JobName)); if (dispatch.ClaimOwner is not null) entries.Add(new("claimOwner", dispatch.ClaimOwner)); if (dispatch.ClaimExpiresUtc is { } claimExpires) entries.Add(new("claimExpiresUtc", Ticks(claimExpires))); if (dispatch.JobId is not null) entries.Add(new("jobId", dispatch.JobId)); @@ -480,12 +483,14 @@ private static ScheduledDispatchState DispatchFromHash(HashEntry[] entries) string headersJson = (string?)Get("headers") ?? "{}"; var headerMap = JsonSerializer.Deserialize>(headersJson) ?? []; var options = JsonSerializer.Deserialize((string?)Get("options") ?? "{}") ?? new TransportSendOptions(); + var destination = Get("destination"); return new ScheduledDispatchState { DispatchId = (string)Get("dispatchId")!, Kind = Enum.Parse((string)Get("kind")!), - Destination = (string)Get("destination")!, + Destination = destination.IsNullOrEmpty ? null : JsonSerializer.Deserialize((string)destination!), + JobName = ToStringOrNull(Get("jobName")), Body = Get("body").IsNullOrEmpty ? ReadOnlyMemory.Empty : Convert.FromBase64String((string)Get("body")!), Headers = MessageHeaders.Create(headerMap), Options = options, diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs index 1869836d8..7f6243624 100644 --- a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -274,14 +274,14 @@ public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() var t = time.GetUtcNow(); var headers = MessageHeaders.Create(new Dictionary { ["message.type"] = "order.created", ["tenant"] = "acme" }); - var options = new TransportSendOptions { DestinationRole = DestinationRole.Topic, Priority = MessagePriority.High }; + var options = new TransportSendOptions { Priority = MessagePriority.High }; byte[] body = [0x01, 0x02, 0xFF, 0x00, 0x10]; var due = new ScheduledDispatchState { DispatchId = "d1", Kind = ScheduledDispatchKind.JobOccurrence, - Destination = "jobs", + JobName = "jobs", Body = body, Headers = headers, Options = options, @@ -292,25 +292,24 @@ public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() { DispatchId = "d2", Kind = ScheduledDispatchKind.QueueMessage, - Destination = "later", + Destination = DestinationAddress.ForQueue("later"), Body = body, DueUtc = t.AddHours(1) }; await store.ScheduleDispatchAsync(due, ct); await store.ScheduleDispatchAsync(future, ct); - // Re-scheduling the same id is a no-op (must not overwrite the destination). - await store.ScheduleDispatchAsync(due with { Destination = "overwritten" }, ct); + // Re-scheduling the same id is a no-op (must not overwrite the dispatch). + await store.ScheduleDispatchAsync(due with { JobName = "overwritten" }, ct); // Only the due dispatch is claimed; the full payload round-trips and the attempt counter increments. var claimed = await store.ClaimDueDispatchesAsync(t, 100, "node-a", TimeSpan.FromMinutes(5), ct); var d = Assert.Single(claimed); Assert.Equal("d1", d.DispatchId); Assert.Equal(ScheduledDispatchKind.JobOccurrence, d.Kind); - Assert.Equal("jobs", d.Destination); + Assert.Equal("jobs", d.JobName); Assert.Equal(body, d.Body.ToArray()); Assert.Equal("acme", d.Headers["tenant"]); Assert.Equal("order.created", d.Headers["message.type"]); - Assert.Equal(DestinationRole.Topic, d.Options.DestinationRole); Assert.Equal(MessagePriority.High, d.Options.Priority); Assert.Equal("node-a", d.ClaimOwner); Assert.Equal(1, d.Attempts); @@ -333,7 +332,7 @@ public virtual async Task ScheduledDispatches_ClaimCompleteAndRescheduleAsync() { DispatchId = "d3", Kind = ScheduledDispatchKind.JobOccurrence, - Destination = "cron", + JobName = "cron", Body = body, DueUtc = t.AddMinutes(20) }; @@ -382,7 +381,7 @@ public virtual async Task Concurrency_OptimisticControlElectsSingleWinnerAsync() await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = "dispatch-race", - Destination = "q", + Destination = DestinationAddress.ForQueue("q"), Body = new byte[] { 1 }, DueUtc = time.GetUtcNow().AddMinutes(-1) }, ct); diff --git a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs index 01dc9e081..f2f3be2b3 100644 --- a/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs +++ b/src/Foundatio.TestHarness/Messaging/MessageTransportConformanceTests.cs @@ -36,9 +36,10 @@ public virtual async Task CanSendAndReceiveBatchAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "orders", Role = DestinationRole.Queue }); + var queue = DestinationAddress.ForQueue("orders"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); - var result = await transport.SendAsync("orders", [ + var result = await transport.SendAsync(queue, [ CreateMessage("one", ("tenant", "acme")), CreateMessage("two", ("tenant", "acme")) ], new TransportSendOptions(), TestCancellationToken); @@ -46,7 +47,7 @@ public virtual async Task CanSendAndReceiveBatchAsync() // Send is throw-on-failure, so reaching here means both messages were accepted; assert the accepted ids. Assert.Equal(2, result.Items.Count); - var entries = await pull.ReceiveAsync("orders", new ReceiveRequest + var entries = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxMessages = 2, MaxWaitTime = TimeSpan.FromSeconds(1) @@ -76,7 +77,7 @@ public virtual async Task CanSendAndReceiveBatchAsync() // Assert only the point-in-time gauges every broker can report, and tolerate eventual consistency // (e.g. SQS ApproximateNumberOf* lag). Lifetime counters such as Completed are not universally // available across transports, so they are not part of the shared contract. - await AssertQueueDrainedAsync(stats, "orders", TestCancellationToken); + await AssertQueueDrainedAsync(stats, queue, TestCancellationToken); } } finally @@ -97,15 +98,16 @@ public virtual async Task AbandonAsync_RedeliversWithIncrementedDeliveryCountAsy try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "retry", Role = DestinationRole.Queue }); - await transport.SendAsync("retry", [CreateMessage("retry-me")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("retry"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("retry-me")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("retry", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); await transport.AbandonAsync(first, TestCancellationToken); - var second = Assert.Single(await pull.ReceiveAsync("retry", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var second = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.DeliveryCount); Assert.Equal("retry-me", ReadBody(second)); @@ -130,10 +132,11 @@ public virtual async Task CompleteAsync_WithExpiredReceipt_ThrowsReceiptExpiredE try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "receipts", Role = DestinationRole.Queue }); - await transport.SendAsync("receipts", [CreateMessage("done")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("receipts"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("done")], new TransportSendOptions(), TestCancellationToken); - var entry = Assert.Single(await pull.ReceiveAsync("receipts", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var entry = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); await transport.CompleteAsync(entry, TestCancellationToken); await Assert.ThrowsAsync(async () => @@ -157,21 +160,22 @@ public virtual async Task SubscribeAsync_DeliversPushMessagesAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "push", Role = DestinationRole.Queue }); + var queue = DestinationAddress.ForQueue("push"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - await using var subscription = await push.SubscribeAsync("push", async (entry, ct) => + await using var subscription = await push.SubscribeAsync(queue, async (entry, ct) => { await transport.CompleteAsync(entry, ct); received.TrySetResult(entry); }, new PushOptions(), TestCancellationToken); - await transport.SendAsync("push", [CreateMessage("pushed")], new TransportSendOptions(), TestCancellationToken); + await transport.SendAsync(queue, [CreateMessage("pushed")], new TransportSendOptions(), TestCancellationToken); var completed = await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(3), TestCancellationToken)); Assert.Equal(received.Task, completed); Assert.Equal("pushed", ReadBody(await received.Task)); - Assert.Equal("push", subscription.Source); + Assert.Equal(queue, subscription.Source); } finally { @@ -191,16 +195,19 @@ public virtual async Task SendAsync_ToTopic_FansOutToSubscriptionsAsync() try { + var topic = DestinationAddress.ForTopic("orders-topic"); + var subscriptionA = DestinationAddress.ForSubscription("orders-topic", "orders-subscription-a"); + var subscriptionB = DestinationAddress.ForSubscription("orders-topic", "orders-subscription-b"); await EnsureAsync(transport, - new DestinationDeclaration { Name = "orders-topic", Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = "orders-subscription-a", Role = DestinationRole.Subscription, Source = "orders-topic" }, - new DestinationDeclaration { Name = "orders-subscription-b", Role = DestinationRole.Subscription, Source = "orders-topic" }); + new DestinationDeclaration { Address = topic }, + new DestinationDeclaration { Address = subscriptionA }, + new DestinationDeclaration { Address = subscriptionB }); - // The caller states the destination role; publishing to a topic must set DestinationRole.Topic. - await transport.SendAsync("orders-topic", [CreateMessage("fanout")], new TransportSendOptions { DestinationRole = DestinationRole.Topic }, TestCancellationToken); + // The address states the destination role; publishing to a topic must use a topic-role address. + await transport.SendAsync(topic, [CreateMessage("fanout")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("orders-subscription-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); - var second = Assert.Single(await pull.ReceiveAsync("orders-subscription-b", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync(subscriptionA, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + var second = Assert.Single(await pull.ReceiveAsync(subscriptionB, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); Assert.Equal("fanout", ReadBody(first)); Assert.Equal("fanout", ReadBody(second)); @@ -214,6 +221,54 @@ await EnsureAsync(transport, } } + [Fact] + public virtual async Task ProvisioningLifecycle_EnsureExistsDeleteAsync() + { + var transport = CreateTransport(); + if (transport is not ISupportsProvisioning provisioning) + { + Assert.Skip("Transport does not support provisioning (ISupportsProvisioning)."); + return; + } + + try + { + var queue = DestinationAddress.ForQueue("prov-queue"); + var topic = DestinationAddress.ForTopic("prov-topic"); + var subscription = DestinationAddress.ForSubscription("prov-topic", "prov-sub"); + + Assert.False(await provisioning.ExistsAsync(queue, TestCancellationToken)); + Assert.False(await provisioning.ExistsAsync(topic, TestCancellationToken)); + Assert.False(await provisioning.ExistsAsync(subscription, TestCancellationToken)); + + DestinationDeclaration[] declarations = [ + new DestinationDeclaration { Address = queue }, + new DestinationDeclaration { Address = topic }, + new DestinationDeclaration { Address = subscription } + ]; + await provisioning.EnsureAsync(declarations, TestCancellationToken); + + Assert.True(await provisioning.ExistsAsync(queue, TestCancellationToken)); + Assert.True(await provisioning.ExistsAsync(topic, TestCancellationToken)); + Assert.True(await provisioning.ExistsAsync(subscription, TestCancellationToken)); + + // Ensure is idempotent: re-declaring destinations that already exist must not throw. + await provisioning.EnsureAsync(declarations, TestCancellationToken); + + await provisioning.DeleteAsync(subscription, TestCancellationToken); + await provisioning.DeleteAsync(topic, TestCancellationToken); + await provisioning.DeleteAsync(queue, TestCancellationToken); + + Assert.False(await provisioning.ExistsAsync(queue, TestCancellationToken)); + Assert.False(await provisioning.ExistsAsync(topic, TestCancellationToken)); + Assert.False(await provisioning.ExistsAsync(subscription, TestCancellationToken)); + } + finally + { + await CleanupTransportIfNotNullAsync(transport); + } + } + [Fact] public virtual async Task SendAsync_ToTopic_WithDeliverAt_WithoutNativeDelay_ThrowsAsync() { @@ -241,9 +296,9 @@ public virtual async Task SendAsync_ToTopic_WithDeliverAt_WithoutNativeDelay_Thr // A transport that cannot honor DeliverAt for a role must refuse it, never publish immediately and // silently drop the delay — the core only routes a delayed send here when the role advertises the // capability, so acceptance would mean a lost delay (the AWS SNS delayed-publish bug shape). - await Assert.ThrowsAsync(() => transport.SendAsync("delayed-topic", + await Assert.ThrowsAsync(() => transport.SendAsync(DestinationAddress.ForTopic("delayed-topic"), [CreateMessage("later")], - new TransportSendOptions { DestinationRole = DestinationRole.Topic, DeliverAt = DateTimeOffset.UtcNow.AddMinutes(5) }, + new TransportSendOptions { DeliverAt = DateTimeOffset.UtcNow.AddMinutes(5) }, TestCancellationToken)); } finally @@ -264,12 +319,13 @@ public virtual async Task ReceiveAsync_RespectsPriorityAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "priority", Role = DestinationRole.Queue }); - await transport.SendAsync("priority", [CreateMessage("low")], new TransportSendOptions { Priority = MessagePriority.Low }, TestCancellationToken); - await transport.SendAsync("priority", [CreateMessage("high")], new TransportSendOptions { Priority = MessagePriority.High }, TestCancellationToken); - await transport.SendAsync("priority", [CreateMessage("normal")], new TransportSendOptions { Priority = MessagePriority.Normal }, TestCancellationToken); + var queue = DestinationAddress.ForQueue("priority"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("low")], new TransportSendOptions { Priority = MessagePriority.Low }, TestCancellationToken); + await transport.SendAsync(queue, [CreateMessage("high")], new TransportSendOptions { Priority = MessagePriority.High }, TestCancellationToken); + await transport.SendAsync(queue, [CreateMessage("normal")], new TransportSendOptions { Priority = MessagePriority.Normal }, TestCancellationToken); - var entries = await pull.ReceiveAsync("priority", new ReceiveRequest + var entries = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxMessages = 3, MaxWaitTime = TimeSpan.FromSeconds(1) @@ -301,16 +357,17 @@ public virtual async Task SendAsync_WithDeliverAt_DelaysVisibilityAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "delayed", Role = DestinationRole.Queue }); - await transport.SendAsync("delayed", [CreateMessage("later")], new TransportSendOptions + var queue = DestinationAddress.ForQueue("delayed"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("later")], new TransportSendOptions { DeliverAt = DateTimeOffset.UtcNow.AddMilliseconds(250) }, TestCancellationToken); - var immediate = await pull.ReceiveAsync("delayed", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + var immediate = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); Assert.Empty(immediate); - var delayed = Assert.Single(await pull.ReceiveAsync("delayed", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + var delayed = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); Assert.Equal("later", ReadBody(delayed)); await transport.CompleteAsync(delayed, TestCancellationToken); } @@ -332,13 +389,14 @@ public virtual async Task DeadLetterAsync_MovesEntryToDeadletterStatsAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "deadletter", Role = DestinationRole.Queue }); - await transport.SendAsync("deadletter", [CreateMessage("poison")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("deadletter"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("poison")], new TransportSendOptions(), TestCancellationToken); - var entry = Assert.Single(await pull.ReceiveAsync("deadletter", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var entry = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); await ((ISupportsDeadLetter)transport).DeadLetterAsync(entry, "bad-payload", TestCancellationToken); - MessageDestinationStats queueStats = await stats.GetStatsAsync("deadletter", TestCancellationToken); + MessageDestinationStats queueStats = await stats.GetStatsAsync(queue, TestCancellationToken); Assert.Equal(0, queueStats.Working); Assert.Equal(1, queueStats.Deadletter); } @@ -360,7 +418,8 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "expiration", Role = DestinationRole.Queue }); + var queue = DestinationAddress.ForQueue("expiration"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); var expired = new TransportMessage { Body = Encoding.UTF8.GetBytes("expired"), @@ -369,12 +428,12 @@ public virtual async Task ReceiveAsync_WithExpiredMessage_DeadlettersAndSkipsAsy ]) }; - await transport.SendAsync("expiration", [expired], new TransportSendOptions(), TestCancellationToken); + await transport.SendAsync(queue, [expired], new TransportSendOptions(), TestCancellationToken); - var entries = await pull.ReceiveAsync("expiration", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); + var entries = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(50) }, TestCancellationToken); Assert.Empty(entries); - MessageDestinationStats queueStats = await stats.GetStatsAsync("expiration", TestCancellationToken); + MessageDestinationStats queueStats = await stats.GetStatsAsync(queue, TestCancellationToken); Assert.Equal(0, queueStats.Queued); Assert.Equal(1, queueStats.Deadletter); } @@ -395,22 +454,23 @@ public virtual async Task ReceiveAsync_AfterVisibilityTimeout_RedeliversAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "visibility", Role = DestinationRole.Queue }); - await transport.SendAsync("visibility", [CreateMessage("lease")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("visibility"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("lease")], new TransportSendOptions(), TestCancellationToken); // Whole-second visibility window: real brokers (e.g. SQS) only support second-resolution visibility timeouts. var visibilityWindow = TimeSpan.FromSeconds(2); - var first = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibilityWindow, TestCancellationToken)); + var first = Assert.Single(await visibility.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibilityWindow, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); // Still within the visibility window: a competing receive must not see the in-flight message. - var hidden = await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibilityWindow, TestCancellationToken); + var hidden = await visibility.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibilityWindow, TestCancellationToken); Assert.Empty(hidden); // After the visibility window lapses without settlement the message must be redelivered (at-least-once). A // long poll observes the lapse — a transport wakes a blocked receive when a visibility window expires — so // this is robust to coarse/variable redelivery latency without a fixed sleep. - var second = Assert.Single(await visibility.ReceiveAsync("visibility", new ReceiveRequest { MaxWaitTime = visibilityWindow + TimeSpan.FromSeconds(5) }, visibilityWindow, TestCancellationToken)); + var second = Assert.Single(await visibility.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = visibilityWindow + TimeSpan.FromSeconds(5) }, visibilityWindow, TestCancellationToken)); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.DeliveryCount); @@ -434,10 +494,11 @@ public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayA try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "redelivery-delay", Role = DestinationRole.Queue }); - await transport.SendAsync("redelivery-delay", [CreateMessage("delay-me")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("redelivery-delay"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("delay-me")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); // Whole-second redelivery delay: SQS serves this via ChangeMessageVisibility, which is second-resolution. @@ -445,11 +506,11 @@ public virtual async Task AbandonAsync_WithRedeliveryDelay_RedeliversAfterDelayA await redelivery.AbandonAsync(first, redeliveryDelay, TestCancellationToken); // Within the delay window the message must not be visible again. - var early = await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); + var early = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); Assert.Empty(early); // After the delay lapses it is redelivered with an incremented delivery count. Long poll for robustness. - var second = Assert.Single(await pull.ReceiveAsync("redelivery-delay", new ReceiveRequest { MaxWaitTime = redeliveryDelay + TimeSpan.FromSeconds(5) }, TestCancellationToken)); + var second = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = redeliveryDelay + TimeSpan.FromSeconds(5) }, TestCancellationToken)); Assert.Equal(first.Id, second.Id); Assert.Equal(2, second.DeliveryCount); Assert.Equal("delay-me", ReadBody(second)); @@ -474,13 +535,14 @@ public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "lock-renewal", Role = DestinationRole.Queue }); - await transport.SendAsync("lock-renewal", [CreateMessage("hold")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("lock-renewal"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("hold")], new TransportSendOptions(), TestCancellationToken); // Whole-second windows so the test maps onto second-resolution brokers (e.g. SQS). var originalWindow = TimeSpan.FromSeconds(2); var renewedWindow = TimeSpan.FromSeconds(8); - var first = Assert.Single(await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, originalWindow, TestCancellationToken)); + var first = Assert.Single(await visibility.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, originalWindow, TestCancellationToken)); Assert.Equal(1, first.DeliveryCount); // Renew before the original window lapses, extending it well past the original expiry. @@ -490,7 +552,7 @@ public virtual async Task RenewLockAsync_ExtendsVisibilityWindowAsync() // Past the original window but inside the renewed window: the message must still be held, so a competing // receive sees nothing rather than a premature redelivery. await Task.Delay(originalWindow, TestCancellationToken); - var held = await visibility.ReceiveAsync("lock-renewal", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, originalWindow, TestCancellationToken); + var held = await visibility.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, originalWindow, TestCancellationToken); Assert.Empty(held); await transport.CompleteAsync(first, TestCancellationToken); @@ -513,13 +575,14 @@ public virtual async Task CompetingConsumers_DoNotReceiveTheSameInFlightMessageA try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "competing", Role = DestinationRole.Queue }); - await transport.SendAsync("competing", [CreateMessage("once")], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("competing"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("once")], new TransportSendOptions(), TestCancellationToken); - var first = Assert.Single(await pull.ReceiveAsync("competing", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var first = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); // A competing consumer must not receive the same message while it is in flight. - var second = await pull.ReceiveAsync("competing", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); + var second = await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, TestCancellationToken); Assert.Empty(second); await transport.CompleteAsync(first, TestCancellationToken); @@ -542,14 +605,15 @@ public virtual async Task ReceiveDeadLetteredAsync_ReturnsPoisonPayloadAndReason try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "dlq-read", Role = DestinationRole.Queue }); - await transport.SendAsync("dlq-read", [CreateMessage("poison", ("tenant", "acme"))], new TransportSendOptions(), TestCancellationToken); + var queue = DestinationAddress.ForQueue("dlq-read"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); + await transport.SendAsync(queue, [CreateMessage("poison", ("tenant", "acme"))], new TransportSendOptions(), TestCancellationToken); - var entry = Assert.Single(await pull.ReceiveAsync("dlq-read", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); + var entry = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(1) }, TestCancellationToken)); await deadLetter.DeadLetterAsync(entry, "bad-payload", TestCancellationToken); // The raw (un-deserialized) payload and the dead-letter reason must be inspectable. - var deadLettered = Assert.Single(await deadLetter.ReceiveDeadLetteredAsync("dlq-read", new ReceiveRequest { MaxMessages = 10 }, TestCancellationToken)); + var deadLettered = Assert.Single(await deadLetter.ReceiveDeadLetteredAsync(queue, new ReceiveRequest { MaxMessages = 10 }, TestCancellationToken)); Assert.Equal("poison", ReadBody(deadLettered)); Assert.Equal("acme", deadLettered.Headers["tenant"]); Assert.Equal("bad-payload", deadLettered.Headers[KnownHeaders.DeadLetterReason]); @@ -572,12 +636,13 @@ public virtual async Task SendAsync_PreservesBinaryBodyAndCaseInsensitiveHeaders try { - await EnsureAsync(transport, new DestinationDeclaration { Name = "binary", Role = DestinationRole.Queue }); + var queue = DestinationAddress.ForQueue("binary"); + await EnsureAsync(transport, new DestinationDeclaration { Address = queue }); // Arbitrary, non-UTF-8 bytes with no content type must round-trip exactly (catches body-encoding bugs — a // provider must not assume text), and header keys must round-trip case-insensitively across the wire. byte[] payload = [0x00, 0x01, 0xFF, 0xFE, 0x10, 0x80, 0x7F]; - await transport.SendAsync("binary", [new TransportMessage + await transport.SendAsync(queue, [new TransportMessage { Body = payload, Headers = MessageHeaders.Create([ @@ -586,7 +651,7 @@ await transport.SendAsync("binary", [new TransportMessage ]) }], new TransportSendOptions(), TestCancellationToken); - var entry = Assert.Single(await pull.ReceiveAsync("binary", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); + var entry = Assert.Single(await pull.ReceiveAsync(queue, new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, TestCancellationToken)); Assert.Equal(payload, entry.Body.ToArray()); Assert.Equal("acme", entry.Headers["tenant"]); Assert.Equal("x", entry.Headers["MIXED.CASE"]); @@ -607,7 +672,7 @@ private async ValueTask CleanupTransportIfNotNullAsync(IMessageTransport? transp // Polls until the destination reports no queued or in-flight messages (the point-in-time gauges every broker can // report), tolerating transports whose stats are only eventually consistent (e.g. SQS ApproximateNumberOf*). - private async Task AssertQueueDrainedAsync(ISupportsStats stats, string destination, CancellationToken cancellationToken) + private async Task AssertQueueDrainedAsync(ISupportsStats stats, DestinationAddress destination, CancellationToken cancellationToken) { var current = await stats.GetStatsAsync(destination, cancellationToken); for (int attempt = 0; attempt < 50 && (current.Queued != 0 || current.Working != 0); attempt++) diff --git a/src/Foundatio.Testing/RecordingMessageTransport.cs b/src/Foundatio.Testing/RecordingMessageTransport.cs index 007ef1b16..196c95a04 100644 --- a/src/Foundatio.Testing/RecordingMessageTransport.cs +++ b/src/Foundatio.Testing/RecordingMessageTransport.cs @@ -29,8 +29,8 @@ internal sealed class RecordingMessageTransport : IMessageTransport, ISupportsPu private readonly ConcurrentQueue _handled = new(); private readonly ConcurrentQueue _abandoned = new(); private readonly ConcurrentQueue _deadLettered = new(); - private readonly ConcurrentDictionary _knownNames = new(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary _pendingRedeliveries = new(); + private readonly ConcurrentDictionary _knownNames = new(); + private readonly ConcurrentDictionary _pendingRedeliveries = new(); public RecordingMessageTransport(TimeProvider? timeProvider = null) { @@ -50,18 +50,18 @@ public RecordingMessageTransport(TimeProvider? timeProvider = null) public TimeSpan? MaxVisibilityTimeout => _inner.MaxVisibilityTimeout; public TimeSpan? MaxRedeliveryDelay => _inner.MaxRedeliveryDelay; - public async Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public async Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { var result = await _inner.SendAsync(destination, messages, options, ct).ConfigureAwait(false); _knownNames.TryAdd(destination, 0); - var recordings = options.DestinationRole == DestinationRole.Topic ? _published : _sent; + var recordings = destination.Role == DestinationRole.Topic ? _published : _sent; foreach (var message in messages) { recordings.Enqueue(new RecordedMessage { - Destination = destination, - Role = options.DestinationRole, + Destination = destination.Key, + Role = destination.Role, MessageType = message.Headers.GetValueOrDefault(KnownHeaders.MessageType), Body = message.Body, Headers = message.Headers @@ -71,19 +71,19 @@ public async Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct = default) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct = default) { _knownNames.TryAdd(source, 0); return _inner.ReceiveAsync(source, request, ct); } - public Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default) { _knownNames.TryAdd(source, 0); return _inner.ReceiveAsync(source, request, visibility, ct); } - public Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct = default) + public Task SubscribeAsync(DestinationAddress source, Func onMessage, PushOptions options, CancellationToken ct = default) { _knownNames.TryAdd(source, 0); return _inner.SubscribeAsync(source, onMessage, options, ct); @@ -126,25 +126,25 @@ public async Task DeadLetterAsync(TransportEntry entry, string? reason, Cancella _deadLettered.Enqueue(Record(entry) with { Reason = reason }); } - public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct = default) + public Task> ReceiveDeadLetteredAsync(DestinationAddress destination, ReceiveRequest request, CancellationToken ct = default) => _inner.ReceiveDeadLetteredAsync(destination, request, ct); public Task RenewLockAsync(TransportEntry entry, TimeSpan? duration, CancellationToken ct = default) => _inner.RenewLockAsync(entry, duration, ct); - public Task GetStatsAsync(string destination, CancellationToken ct = default) + public Task GetStatsAsync(DestinationAddress destination, CancellationToken ct = default) => _inner.GetStatsAsync(destination, ct); public Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct = default) { foreach (var declaration in declarations) - _knownNames.TryAdd(declaration.Name, 0); + _knownNames.TryAdd(declaration.Address, 0); return _inner.EnsureAsync(declarations, ct); } - public Task DeleteAsync(string name, CancellationToken ct = default) => _inner.DeleteAsync(name, ct); + public Task DeleteAsync(DestinationAddress destination, CancellationToken ct = default) => _inner.DeleteAsync(destination, ct); - public Task ExistsAsync(string name, CancellationToken ct = default) => _inner.ExistsAsync(name, ct); + public Task ExistsAsync(DestinationAddress destination, CancellationToken ct = default) => _inner.ExistsAsync(destination, ct); public ValueTask DisposeAsync() => _inner.DisposeAsync(); @@ -153,7 +153,7 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc public async Task> GetPendingAsync(CancellationToken ct = default) { var now = _timeProvider.GetUtcNow(); - var scheduled = new Dictionary(StringComparer.OrdinalIgnoreCase); + var scheduled = new Dictionary(); foreach (var redelivery in _pendingRedeliveries) { if (now >= redelivery.Value.DueAt + _redeliveryGrace) @@ -163,12 +163,12 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc } var pending = new List<(string, long, long)>(); - foreach (string name in _knownNames.Keys.OrderBy(n => n, StringComparer.Ordinal)) + foreach (var address in _knownNames.Keys.OrderBy(a => a.Key, StringComparer.Ordinal)) { - var stats = await _inner.GetStatsAsync(name, ct).ConfigureAwait(false); - long queued = stats.Queued + scheduled.GetValueOrDefault(name); + var stats = await _inner.GetStatsAsync(address, ct).ConfigureAwait(false); + long queued = stats.Queued + scheduled.GetValueOrDefault(address); if (queued > 0 || stats.Working > 0) - pending.Add((name, queued, stats.Working)); + pending.Add((address.Key, queued, stats.Working)); } return pending; @@ -176,7 +176,7 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc private static RecordedMessage Record(TransportEntry entry) => new() { - Destination = entry.Destination, + Destination = entry.Destination.Key, Role = DestinationRole.Queue, MessageType = entry.Headers.GetValueOrDefault(KnownHeaders.MessageType), Body = entry.Body, diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 2bcedf7bf..bf067fe9a 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -97,7 +97,13 @@ public sealed record ScheduledDispatchState { public required string DispatchId { get; init; } public ScheduledDispatchKind Kind { get; init; } - public required string Destination { get; init; } + + /// The transport destination for queue/pub-sub message dispatches; null for job occurrences. + public DestinationAddress? Destination { get; init; } + + /// The scheduled job definition name for dispatches; null for message dispatches. + public string? JobName { get; init; } + public required ReadOnlyMemory Body { get; init; } public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; public TransportSendOptions Options { get; init; } = new(); diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index dfabeffbc..10f7a76a2 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -201,7 +201,7 @@ await _store.CreateIfAbsentAsync(new JobState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, - Destination = definition.Name, + JobName = definition.Name, Body = Array.Empty(), Headers = CreateOccurrenceHeaders(definition, occurrence, scopeKey), DueUtc = utcNow, @@ -246,7 +246,7 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = continue; } - if (!definitions.TryGetValue(dispatch.Destination, out var definition) || !definition.Enabled || definition.JobType is null) + if (dispatch.JobName is null || !definitions.TryGetValue(dispatch.JobName, out var definition) || !definition.Enabled || definition.JobType is null) { await _store.ReleaseDispatchAsync(dispatch.DispatchId, _nodeId, utcNow.AddMinutes(1), cancellationToken).ConfigureAwait(false); continue; @@ -312,6 +312,9 @@ private async Task MaterializeMessageDispatchAsync(ScheduledDispatchState dispat if (_transport is null) throw new InvalidOperationException("A message transport is required to materialize scheduled queue and pub/sub dispatches."); + if (dispatch.Destination is null) + throw new InvalidOperationException($"Scheduled {dispatch.Kind} dispatch \"{dispatch.DispatchId}\" has no destination address."); + await _transport.SendAsync(dispatch.Destination, [ new TransportMessage { diff --git a/src/Foundatio/Messaging/InMemoryMessageTransport.cs b/src/Foundatio/Messaging/InMemoryMessageTransport.cs index eac68c589..16d2b28cd 100644 --- a/src/Foundatio/Messaging/InMemoryMessageTransport.cs +++ b/src/Foundatio/Messaging/InMemoryMessageTransport.cs @@ -53,19 +53,19 @@ public InMemoryMessageTransport(TimeProvider? timeProvider = null) public TimeSpan? MaxVisibilityTimeout => null; public TimeSpan? MaxRedeliveryDelay => null; - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); ArgumentNullException.ThrowIfNull(messages); if (options.DeliverAt is { } deliverAt && deliverAt > _timeProvider.GetUtcNow()) throw new NotSupportedException($"Transport \"{GetType().Name}\" does not support native delayed delivery. Use the runtime-store scheduled dispatch fallback."); - // The caller-stated role picks the physical namespace, so a queue and a topic can share a route name (a message + // The address role picks the physical namespace, so a queue and a topic can share a route name (a message // type that is both sent and published) without colliding or cross-delivering. - string key = options.DestinationRole == DestinationRole.Topic ? TopicKey(destination) : SourceKey(destination); + string key = StorageKey(destination); var results = new SendItemResult[messages.Count]; for (int index = 0; index < messages.Count; index++) @@ -82,24 +82,24 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) { return ReceiveAsync(source, request, visibility: null, ct); } - public async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) + public async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct) { return await ReceiveAsync(source, request, (TimeSpan?)visibility, ct).AnyContext(); } - private async Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan? visibility, CancellationToken ct) + private async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan? visibility, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(source); int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; - var state = GetOrAddDestination(SourceKey(source)); + var state = GetOrAddDestination(ReceivableKey(source)); var entries = new List(maxMessages); DateTimeOffset? waitUntil = request.MaxWaitTime is { } waitTime && waitTime > TimeSpan.Zero ? _timeProvider.GetUtcNow().Add(waitTime) @@ -242,14 +242,14 @@ public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationTo return Task.CompletedTask; } - public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveDeadLetteredAsync(DestinationAddress destination, ReceiveRequest request, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); ArgumentNullException.ThrowIfNull(request); - if (!_destinations.TryGetValue(SourceKey(destination), out var state)) + if (!_destinations.TryGetValue(ReceivableKey(destination), out var state)) return Task.FromResult>([]); int maxMessages = request.MaxMessages <= 0 ? 1 : request.MaxMessages; @@ -271,11 +271,11 @@ public Task> ReceiveDeadLetteredAsync(string desti return Task.FromResult>(entries); } - public Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct) + public Task SubscribeAsync(DestinationAddress source, Func onMessage, PushOptions options, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(source); + ArgumentNullException.ThrowIfNull(source); ArgumentNullException.ThrowIfNull(onMessage); ArgumentNullException.ThrowIfNull(options); @@ -284,13 +284,13 @@ public Task SubscribeAsync(string source, Func(subscription); } - public Task GetStatsAsync(string destination, CancellationToken ct) + public Task GetStatsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(destination); + ArgumentNullException.ThrowIfNull(destination); - if (!_destinations.TryGetValue(SourceKey(destination), out var state)) + if (!_destinations.TryGetValue(ReceivableKey(destination), out var state)) return Task.FromResult(new MessageDestinationStats()); return Task.FromResult(new MessageDestinationStats @@ -313,64 +313,63 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc foreach (var declaration in declarations) { - ArgumentException.ThrowIfNullOrEmpty(declaration.Name); + var address = declaration.Address; + ArgumentNullException.ThrowIfNull(address); - switch (declaration.Role) + switch (address.Role) { case DestinationRole.Queue: - GetOrAddDestination(QueueKey(declaration.Name)); + GetOrAddDestination(StorageKey(address)); break; case DestinationRole.Topic: - _roles.TryAdd(TopicKey(declaration.Name), DestinationRole.Topic); - _topicSubscriptions.GetOrAdd(TopicKey(declaration.Name), static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + _roles.TryAdd(StorageKey(address), DestinationRole.Topic); + _topicSubscriptions.GetOrAdd(StorageKey(address), static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); break; case DestinationRole.Subscription: - GetOrAddDestination(QueueKey(declaration.Name)); - if (!String.IsNullOrEmpty(declaration.Source)) - AddTopicSubscription(declaration.Source, declaration.Name); + if (String.IsNullOrEmpty(address.Topic)) + throw new ArgumentException("A subscription declaration must specify its owning topic.", nameof(declarations)); + + AddTopicSubscription(address.Topic, StorageKey(address)); break; case DestinationRole.Binding: - if (String.IsNullOrEmpty(declaration.Source)) + if (String.IsNullOrEmpty(address.Topic)) throw new ArgumentException("A binding declaration must specify a source topic.", nameof(declarations)); - GetOrAddDestination(QueueKey(declaration.Name)); - AddTopicSubscription(declaration.Source, declaration.Name); + AddTopicSubscription(address.Topic, StorageKey(address)); break; default: - throw new ArgumentOutOfRangeException(nameof(declarations), declaration.Role, "Unsupported destination role."); + throw new ArgumentOutOfRangeException(nameof(declarations), address.Role, "Unsupported destination role."); } } return Task.CompletedTask; } - public Task DeleteAsync(string name, CancellationToken ct) + public Task DeleteAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - foreach (string key in (string[])[QueueKey(name), TopicKey(name)]) - { - _roles.TryRemove(key, out _); - if (_destinations.TryRemove(key, out var removed)) - removed.Complete(); - _topicSubscriptions.TryRemove(key, out _); + string key = StorageKey(destination); + _roles.TryRemove(key, out _); + if (_destinations.TryRemove(key, out var removed)) + removed.Complete(); + _topicSubscriptions.TryRemove(key, out _); - foreach (var subscriptions in _topicSubscriptions.Values) - subscriptions.TryRemove(key, out _); - } + foreach (var subscriptions in _topicSubscriptions.Values) + subscriptions.TryRemove(key, out _); return Task.CompletedTask; } - public Task ExistsAsync(string name, CancellationToken ct) + public Task ExistsAsync(DestinationAddress destination, CancellationToken ct) { ThrowIfDisposed(); ct.ThrowIfCancellationRequested(); - ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(destination); - return Task.FromResult(_roles.ContainsKey(QueueKey(name)) || _roles.ContainsKey(TopicKey(name))); + return Task.FromResult(_roles.ContainsKey(StorageKey(destination))); } public ValueTask DisposeAsync() @@ -393,7 +392,7 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } - private async Task RunPushSubscriptionAsync(string source, Func onMessage, PushOptions options, CancellationToken subscriptionCancellationToken) + private async Task RunPushSubscriptionAsync(DestinationAddress source, Func onMessage, PushOptions options, CancellationToken subscriptionCancellationToken) { using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(subscriptionCancellationToken, _disposeCancellationTokenSource.Token); var token = linkedCancellationTokenSource.Token; @@ -530,7 +529,7 @@ private void ScheduleReclaim(DestinationState state, TimeSpan delay) timer.Dispose(); } - private bool TryReceive(string source, DestinationState state, TimeSpan? visibility, out TransportEntry entry) + private bool TryReceive(DestinationAddress source, DestinationState state, TimeSpan? visibility, out TransportEntry entry) { while (state.TryDequeue(out var message)) { @@ -541,8 +540,8 @@ private bool TryReceive(string source, DestinationState state, TimeSpan? visibil } // The receipt carries the internal (role-qualified) key so settlement resolves the same state; the entry's - // Destination stays the caller-facing source name. - var receipt = new InMemoryReceipt(SourceKey(source), Guid.NewGuid().ToString("N")); + // Destination stays the caller-facing source address. + var receipt = new InMemoryReceipt(ReceivableKey(source), Guid.NewGuid().ToString("N")); DateTimeOffset? visibilityExpiresUtc = visibility is { } window ? _timeProvider.GetUtcNow().Add(window) : null; state.InFlight[receipt.LockToken] = new InFlightMessage(message, receipt, visibilityExpiresUtc); Interlocked.Increment(ref state.Dequeued); @@ -606,12 +605,15 @@ private StoredMessage CreateStoredMessage(string destination, string messageId, EnqueuedUtc: _timeProvider.GetUtcNow()); } - // Internal state is keyed by role-qualified names: "t:" for topics, "q:" for every receivable destination (queues - // AND subscriptions — a subscription is a queue-shaped destination a topic fans into, exactly like an SNS-bound SQS - // queue). This gives a queue/subscription and a topic sharing a route name distinct namespaces, as real brokers do. - private static string QueueKey(string name) => "q:" + name; - private static string TopicKey(string name) => "t:" + name; - private static string SourceKey(string name) => QueueKey(name); + // Internal state is keyed by role-qualified names derived from the canonical address: "t:" for topics, "q:" for + // every receivable destination (queues AND subscriptions — a subscription is a queue-shaped destination a topic + // fans into, exactly like an SNS-bound SQS queue, keyed by its topic-qualified address key). This gives a + // queue/subscription and a topic sharing a route name distinct namespaces, as real brokers do. + private static string StorageKey(DestinationAddress address) => + address.Role == DestinationRole.Topic ? "t:" + address.Name : "q:" + address.Key; + + // Receive-path keys are always queue-shaped; receive/stats/dead-letter reads never target a topic. + private static string ReceivableKey(DestinationAddress address) => "q:" + address.Key; private static DestinationRole RoleForKey(string key) => key[0] == 't' ? DestinationRole.Topic : DestinationRole.Queue; @@ -629,13 +631,13 @@ private DestinationState GetExistingDestination(string key) throw new ReceiptExpiredException($"The destination \"{key}\" no longer exists."); } - private void AddTopicSubscription(string topic, string subscription) + private void AddTopicSubscription(string topic, string subscriptionStorageKey) { - string topicKey = TopicKey(topic); + string topicKey = "t:" + topic; _roles.TryAdd(topicKey, DestinationRole.Topic); - GetOrAddDestination(QueueKey(subscription)); + GetOrAddDestination(subscriptionStorageKey); var subscriptions = _topicSubscriptions.GetOrAdd(topicKey, static _ => new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); - subscriptions[QueueKey(subscription)] = 0; + subscriptions[subscriptionStorageKey] = 0; } private static MessagePriority NormalizePriority(MessagePriority priority) @@ -795,12 +797,12 @@ private sealed class PushSubscription : IPushSubscription private readonly CancellationTokenSource _cancellationTokenSource = new(); private Task? _worker; - public PushSubscription(string source) + public PushSubscription(DestinationAddress source) { Source = source; } - public string Source { get; } + public DestinationAddress Source { get; } public CancellationToken CancellationToken => _cancellationTokenSource.Token; public void Start(Task worker) diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index 8433602b5..dec4a7c3f 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -319,11 +319,11 @@ public ValueTask DisposeAsync() // its own subscriber group for published ones. An explicit Key opts subscriptions into one shared group. string uniqueKey = Guid.NewGuid().ToString("N"); - string destination = GetDestination(routeType, options.Destination); + var destination = GetDestination(routeType, options.Destination); var send = new ListenerConfig { Source = destination, - Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{destination}:{uniqueKey}", + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{destination.Key}:{uniqueKey}", MessageType = routeType, AckMode = options.AckMode, MaxConcurrency = options.MaxConcurrency, @@ -332,18 +332,16 @@ public ValueTask DisposeAsync() DeadLetterWhen = options.DeadLetterWhen }; - string topic = GetTopic(routeType, options.Topic); + var topic = GetTopic(routeType, options.Topic); string subscription = options.PerInstance ? $"{Environment.MachineName}-{Guid.NewGuid():N}" - : options.Subscription ?? QualifySubscription(GetSubscription(routeType, topic, null), options.SubscriptionQualifier); + : options.Subscription ?? QualifySubscription(GetSubscription(routeType, topic.Name, null), options.SubscriptionQualifier); var publish = new ListenerConfig { - Topic = topic, - Subscription = subscription, - // The transport source is the topic-qualified subscription destination, not the bare subscription name, so - // the same subscription identity used on two topics resolves to two distinct sources (and isolates). - Source = SubscriptionAddress.Format(topic, subscription), - Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic}:{subscription}:{uniqueKey}", + // The source is the topic-qualified subscription address, not the bare subscription name, so the same + // subscription identity used on two topics resolves to two distinct sources (and isolates). + Source = DestinationAddress.ForSubscription(topic.Name, subscription), + Key = !String.IsNullOrEmpty(options.Key) ? options.Key : $"{topic.Name}:{subscription}:{uniqueKey}", MessageType = routeType, AckMode = options.AckMode, MaxConcurrency = options.MaxConcurrency, @@ -365,41 +363,41 @@ private static string QualifySubscription(string identity, string? qualifier) private void LogSubscription(ListenerConfig send, ListenerConfig publish) { _logger.LogInformation( - "Subscribed {MessageType}: send={Destination}, publish={Topic}/{Subscription}, concurrency={MaxConcurrency}, attempts={MaxAttempts}, ack={AckMode}", - send.MessageType.Name, send.Source, publish.Topic, publish.Subscription, Math.Max(1, send.MaxConcurrency), send.MaxAttempts?.ToString() ?? "default", send.AckMode); + "Subscribed {MessageType}: send={Destination}, publish={Subscription}, concurrency={MaxConcurrency}, attempts={MaxAttempts}, ack={AckMode}", + send.MessageType.Name, send.Source.Key, publish.Source.Key, Math.Max(1, send.MaxConcurrency), send.MaxAttempts?.ToString() ?? "default", send.AckMode); } - private Task EnsureTopicAsync(string topic, CancellationToken cancellationToken) + private Task EnsureTopicAsync(DestinationAddress topic, CancellationToken cancellationToken) { - return _core.EnsureAsync([new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }], cancellationToken); + return _core.EnsureAsync([new DestinationDeclaration { Address = topic }], cancellationToken); } private Task EnsureSubscriptionAsync(ListenerConfig config, CancellationToken cancellationToken) { return _core.EnsureAsync([ - new DestinationDeclaration { Name = config.Topic, Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = config.Source, Role = DestinationRole.Subscription, Source = config.Topic } + new DestinationDeclaration { Address = DestinationAddress.ForTopic(config.Source.Topic!) }, + new DestinationDeclaration { Address = config.Source } ], cancellationToken); } - private string GetDestination(Type messageType, string? destination) + private DestinationAddress GetDestination(Type messageType, string? destination) { - return _core.Router.ResolveRoute(new MessageRouteContext + return DestinationAddress.ForQueue(_core.Router.ResolveRoute(new MessageRouteContext { MessageType = messageType, Role = MessageRouteRole.QueueDestination, OperationOverride = destination - }); + })); } - private string GetTopic(Type messageType, string? topic) + private DestinationAddress GetTopic(Type messageType, string? topic) { - return _core.Router.ResolveRoute(new MessageRouteContext + return DestinationAddress.ForTopic(_core.Router.ResolveRoute(new MessageRouteContext { MessageType = messageType, Role = MessageRouteRole.PubSubTopic, OperationOverride = topic - }); + })); } private string GetSubscription(Type messageType, string topic, string? subscription) @@ -450,10 +448,10 @@ public MessageSubscription(MessageListenerHandle sent, MessageListenerHandle pub } public string Key => _sent.Key; - public string Destination => _sent.Source; + public string Destination => _sent.Source.Key; public string Topic => _published.Topic; public string Subscription => _published.Subscription; - public string Source => _published.Source; + public string Source => _published.Source.Key; public async ValueTask DisposeAsync() { diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 46fcb4328..dbd35b69b 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -48,11 +48,9 @@ internal sealed record MessageEnvelopeOptions /// internal sealed record ListenerConfig { - public required string Source { get; init; } + public required DestinationAddress Source { get; init; } public required string Key { get; init; } public required Type MessageType { get; init; } - public string Topic { get; init; } = ""; - public string Subscription { get; init; } = ""; public AckMode AckMode { get; init; } = AckMode.Auto; public int MaxConcurrency { get; init; } = 1; // Null falls back to the client's default RetryPolicy. @@ -79,7 +77,7 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly IMessageTypeRegistry _typeRegistry; private readonly string? _contentType; private readonly bool _ownsTransport; - private readonly ConcurrentDictionary _sources = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _sources = new(); private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, @@ -107,12 +105,12 @@ public Task EnsureAsync(IReadOnlyList declarations, Canc : Task.CompletedTask; } - public async Task SendAsync(ScheduledDispatchKind kind, Type messageType, object message, MessageEnvelopeOptions options, string destination, Func? ensureDestination, CancellationToken cancellationToken) + public async Task SendAsync(ScheduledDispatchKind kind, Type messageType, object message, MessageEnvelopeOptions options, DestinationAddress destination, Func? ensureDestination, CancellationToken cancellationToken) { ThrowIfDisposed(); - ValidateCapabilities(RoleFor(kind), options.Priority, options.TimeToLive); + ValidateCapabilities(destination.Role, options.Priority, options.TimeToLive); - var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; + var sendOptions = BuildSendOptions(options); string messageId = Guid.NewGuid().ToString("N"); var transportMessage = CreateTransportMessage(message, messageType, options, messageId); @@ -128,19 +126,19 @@ public async Task SendAsync(ScheduledDispatchKind kind, Type messageType return (items.Count > 0 ? items[0].MessageId : null) ?? messageId; } - public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable messages, Type? declaredType, MessageEnvelopeOptions options, Func resolveDestination, Func? ensureDestination, CancellationToken cancellationToken) + public async Task SendBatchAsync(ScheduledDispatchKind kind, IEnumerable messages, Type? declaredType, MessageEnvelopeOptions options, Func resolveDestination, Func? ensureDestination, CancellationToken cancellationToken) { ThrowIfDisposed(); ValidateCapabilities(RoleFor(kind), options.Priority, options.TimeToLive); - var sendOptions = BuildSendOptions(options) with { DestinationRole = RoleFor(kind) }; - var grouped = new Dictionary>(StringComparer.Ordinal); + var sendOptions = BuildSendOptions(options); + var grouped = new Dictionary>(); foreach (var message in messages) { ArgumentNullException.ThrowIfNull(message); Type messageType = declaredType ?? message.GetType(); - string destination = resolveDestination(messageType); + var destination = resolveDestination(messageType); if (!grouped.TryGetValue(destination, out var transportMessages)) { @@ -241,7 +239,7 @@ private async Task RegisterConsumerAsync(ListenerConfig c } // The listener was disposing as its last consumer detached; drop our stale reference and retry. - _sources.TryRemove(new KeyValuePair(config.Source, listener)); + _sources.TryRemove(new KeyValuePair(config.Source, listener)); } } @@ -252,7 +250,7 @@ private static bool IsCatchAll(Type messageType) return messageType == typeof(object) || messageType.IsInterface || messageType.IsAbstract; } - private async Task HandleUnmatchedAsync(TransportEntry entry, string source, CancellationToken cancellationToken) + private async Task HandleUnmatchedAsync(TransportEntry entry, DestinationAddress source, CancellationToken cancellationToken) { MessagingInstruments.Unhandled.Add(1, new KeyValuePair("source", source)); @@ -270,7 +268,7 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, string source, Can // Surface to direct callers. The throw is caught (and not re-logged) by the loop's per-message handling // (SafeProcessAsync), so it never tears down the receive loop or the other type handlers sharing this source. - throw new UnhandledMessageTypeException(message.MessageType, source); + throw new UnhandledMessageTypeException(message.MessageType, source.Key); } // MaxConcurrency bounds the number of in-flight messages. A slot is held from receive until the message settles @@ -278,7 +276,7 @@ private async Task HandleUnmatchedAsync(TransportEntry entry, string source, Can // (no head-of-line blocking) and steady-state utilization stays at the configured concurrency. A failure while // receiving or while processing a single entry (including a poison message that was already dead-lettered) must // never tear down the loop, otherwise one bad message or a transient transport blip silently stops consumption. - private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func onMessage, int maxConcurrency, CancellationToken cancellationToken) + private async Task RunPullLoopAsync(DestinationAddress source, ISupportsPull pull, Func onMessage, int maxConcurrency, CancellationToken cancellationToken) { maxConcurrency = Math.Max(1, maxConcurrency); var slots = new SemaphoreSlim(maxConcurrency, maxConcurrency); @@ -363,7 +361,7 @@ private async Task RunPullLoopAsync(string source, ISupportsPull pull, Func onMessage, string source, SemaphoreSlim slots, CancellationToken cancellationToken) + private async Task ProcessAndReleaseSlotAsync(TransportEntry entry, Func onMessage, DestinationAddress source, SemaphoreSlim slots, CancellationToken cancellationToken) { try { @@ -381,7 +379,7 @@ private static void ReleaseSlots(SemaphoreSlim slots, int count) slots.Release(count); } - private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, string source, CancellationToken cancellationToken) + private async Task SafeProcessAsync(TransportEntry entry, Func onMessage, DestinationAddress source, CancellationToken cancellationToken) { try { @@ -453,7 +451,7 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig } finally { - MessagingInstruments.HandlerTime.Record(Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds, new KeyValuePair("source", config.Source)); + MessagingInstruments.HandlerTime.Record(Stopwatch.GetElapsedTime(startTimestamp).TotalMilliseconds, new KeyValuePair("source", config.Source.Key)); } } @@ -472,7 +470,7 @@ private async Task HandleMessageAsync(TMessage message, ListenerConfig if (activity.IsAllDataRequested) { - activity.SetTag("messaging.source", config.Source); + activity.SetTag("messaging.source", config.Source.Key); activity.SetTag("messaging.message.id", message.Id); } @@ -494,13 +492,13 @@ private static Task SettleFailedMessageAsync(IMessageContext message, bool unrec private MessageContext CreateMessageContext(TransportEntry entry, CancellationToken cancellationToken) { - MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); + MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination.Key)); return new MessageContext(_transport, entry, cancellationToken, _runtimeStore, _timeProvider, _retryPolicy.DeadLetterDestination, _logger); } private async Task> CreateMessageContextAsync(TransportEntry entry, CancellationToken cancellationToken) where T : class { - MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination)); + MessagingInstruments.Received.Add(1, new KeyValuePair("source", entry.Destination.Key)); // For an interface/base route the body cannot be deserialized as T directly. Resolve the concrete payload type // from the message-type header via the registry and deserialize that, then hand it back as T (the concrete @@ -541,7 +539,7 @@ private async Task> CreateMessageContextAsync(TransportEnt private Task DeadLetterPoisonMessageAsync(TransportEntry entry, string reason, Exception? exception, CancellationToken cancellationToken) { - MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination)); + MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", entry.Destination.Key)); var enriched = entry with { Headers = MessageContext.BuildDeadLetterHeaders(entry, entry.DeliveryCount, exception, _timeProvider) }; return MessageContext.DeadLetterOrDropAsync(_transport, enriched, reason, _retryPolicy.DeadLetterDestination, _logger, cancellationToken); } @@ -613,9 +611,9 @@ private void ValidateCapabilities(DestinationRole role, MessagePriority priority throw new NotSupportedException($"Transport \"{_transport.GetType().Name}\" does not support message expiration for {role} destinations."); } - private async Task TryScheduleAsync(ScheduledDispatchKind kind, string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + private async Task TryScheduleAsync(ScheduledDispatchKind kind, DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { - if (!ShouldScheduleThroughRuntimeStore(options, out var dueUtc)) + if (!ShouldScheduleThroughRuntimeStore(destination.Role, options, out var dueUtc)) return false; foreach (var message in messages) @@ -636,7 +634,7 @@ private async Task TryScheduleAsync(ScheduledDispatchKind kind, string des return true; } - private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out DateTimeOffset dueUtc) + private bool ShouldScheduleThroughRuntimeStore(DestinationRole role, TransportSendOptions options, out DateTimeOffset dueUtc) { dueUtc = options.DeliverAt.GetValueOrDefault(); var now = _timeProvider.GetUtcNow(); @@ -647,19 +645,19 @@ private bool ShouldScheduleThroughRuntimeStore(TransportSendOptions options, out // (e.g. SQS caps DelaySeconds at 15 minutes) must route through the durable runtime store rather than be // silently truncated to the broker's ceiling. The check is per destination role: a transport whose queues take // a native delay may still have topics that cannot (SQS vs. SNS), and those publishes must fall back too. - var capabilities = CapabilitiesFor(options.DestinationRole); + var capabilities = CapabilitiesFor(role); if (capabilities.DelayedDelivery && (capabilities.MaxDeliveryDelay is not { } max || dueUtc - now <= max)) return false; if (_runtimeStore is null) - throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" for {options.DestinationRole} destinations (within its supported maximum) or a registered job runtime store.", null); + throw _exceptionFactory($"Delayed delivery requires either native delayed-delivery support from transport \"{_transport.GetType().Name}\" for {role} destinations (within its supported maximum) or a registered job runtime store.", null); return true; } - private async Task> SendChunkedAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) + private async Task> SendChunkedAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken cancellationToken) { - var capabilities = CapabilitiesFor(options.DestinationRole); + var capabilities = CapabilitiesFor(destination.Role); // Enforce a transport-declared maximum message size up front with a clear error, rather than letting an opaque // broker rejection surface mid-send (the limit is advertised, so honor it). @@ -693,11 +691,11 @@ private async Task> SendChunkedAsync(string destin return items; } - private static void RecordSent(string destination, IReadOnlyList items) + private static void RecordSent(DestinationAddress destination, IReadOnlyList items) { // Every returned item was accepted (send is throw-on-failure). if (items.Count > 0) - MessagingInstruments.Sent.Add(items.Count, new KeyValuePair("destination", destination)); + MessagingInstruments.Sent.Add(items.Count, new KeyValuePair("destination", destination.Key)); } private ISupportsPull RequirePull() @@ -706,9 +704,9 @@ private ISupportsPull RequirePull() ?? throw _exceptionFactory($"Transport \"{_transport.GetType().Name}\" does not support pull receive.", null); } - private void RemoveSource(string source, SourceListener listener) + private void RemoveSource(DestinationAddress source, SourceListener listener) { - _sources.TryRemove(new KeyValuePair(source, listener)); + _sources.TryRemove(new KeyValuePair(source, listener)); } private void ThrowIfDisposed() @@ -733,7 +731,7 @@ private sealed class ConsumerRegistration private sealed class SourceListener { private readonly MessageClientCore _core; - private readonly string _source; + private readonly DestinationAddress _source; private readonly object _lock = new(); private readonly CancellationTokenSource _cancellationTokenSource = new(); private readonly ConcurrentDictionary _consumers = new(StringComparer.Ordinal); @@ -744,7 +742,7 @@ private sealed class SourceListener private Task? _loop; private bool _isDisposed; - public SourceListener(MessageClientCore core, string source) + public SourceListener(MessageClientCore core, DestinationAddress source) { _core = core; _source = source; @@ -780,7 +778,7 @@ public bool TryAddConsumer(ConsumerRegistration registration, out MessageListene throw new InvalidOperationException($"Source \"{_source}\" is already consumed with MaxConcurrency {_maxConcurrency}; a conflicting MaxConcurrency {desired} was requested. Consumers sharing a destination must use the same MaxConcurrency."); } - handle = new MessageListenerHandle(registration.Config.Topic, registration.Config.Subscription, _source, registration.Key, () => RemoveConsumerAsync(registration.Key)); + handle = new MessageListenerHandle(_source, registration.Key, () => RemoveConsumerAsync(registration.Key)); _consumers[registration.Key] = new Registered(registration, handle); GroupFor(registration).Add(registration); @@ -986,7 +984,7 @@ public Task CompleteAsync(CancellationToken cancellationToken = default) if (!TryMarkHandled()) return Task.CompletedTask; - MessagingInstruments.Completed.Add(1, new KeyValuePair("source", _entry.Destination)); + MessagingInstruments.Completed.Add(1, new KeyValuePair("source", _entry.Destination.Key)); return _transport.CompleteAsync(_entry, cancellationToken); } @@ -999,13 +997,13 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c if (options.Terminal) { - MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination)); + MessagingInstruments.DeadLettered.Add(1, new KeyValuePair("source", _entry.Destination.Key)); var enriched = _entry with { Headers = BuildDeadLetterHeaders(_entry, Attempts, options.Exception, _timeProvider) }; await DeadLetterOrDropAsync(_transport, enriched, options.Reason, _deadLetterDestination, _logger, cancellationToken).AnyContext(); return; } - MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination)); + MessagingInstruments.Abandoned.Add(1, new KeyValuePair("source", _entry.Destination.Key)); if (options.RedeliveryDelay is not { } redeliveryDelay || redeliveryDelay <= TimeSpan.Zero) { @@ -1023,9 +1021,9 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c } // The runtime-store fallback re-sends the message as a plain queue send, which only makes sense for a - // queue-channel entry: a subscription-channel entry's Destination is the opaque topic-qualified address, and a - // queue send to that name would land where no subscription group reads. - bool isSubscriptionSource = SubscriptionAddress.TryParse(_entry.Destination, out _, out _); + // queue-channel entry: a subscription-channel entry would need to be re-sent into its subscription group, and + // a queue send to that address would land where no subscription group reads. + bool isSubscriptionSource = _entry.Destination.Role == DestinationRole.Subscription; if (_runtimeStore is null || isSubscriptionSource) { // A best-effort delay (the core retry policy) degrades to immediate redelivery; an explicit caller delay @@ -1036,7 +1034,7 @@ public async Task RejectAsync(RejectOptions? options = null, CancellationToken c return; } - throw new MessageBusException($"Delayed redelivery of \"{_entry.Destination}\" requires native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum){(isSubscriptionSource ? "" : " or a registered job runtime store")}."); + throw new MessageBusException($"Delayed redelivery of \"{_entry.Destination.Key}\" requires native redelivery-delay support from transport \"{_transport.GetType().Name}\" (within its supported maximum){(isSubscriptionSource ? "" : " or a registered job runtime store")}."); } // Advance from the reconciled attempt count, not the raw transport DeliveryCount: the re-send produces a new @@ -1081,7 +1079,9 @@ internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, Tr return; } - string destination = !String.IsNullOrEmpty(deadLetterDestination) ? deadLetterDestination : $"{entry.Destination}.deadletter"; + var destination = !String.IsNullOrEmpty(deadLetterDestination) + ? DestinationAddress.ForQueue(deadLetterDestination) + : DestinationAddress.ForQueue($"{entry.Destination.Key}.deadletter"); var headers = String.IsNullOrEmpty(reason) ? entry.Headers : entry.Headers.ToBuilder().Set(KnownHeaders.DeadLetterReason, reason).Build(); @@ -1092,7 +1092,7 @@ internal static async Task DeadLetterOrDropAsync(IMessageTransport transport, Tr } catch (Exception ex) { - logger.LogError(ex, "Failed to park dead-lettered message \"{MessageId}\" at \"{Destination}\"; dropping it: {Message}", entry.Id, destination, ex.Message); + logger.LogError(ex, "Failed to park dead-lettered message \"{MessageId}\" at \"{Destination}\"; dropping it: {Message}", entry.Id, destination.Key, ex.Message); } await transport.CompleteAsync(entry, cancellationToken).AnyContext(); @@ -1106,7 +1106,7 @@ internal static MessageHeaders BuildDeadLetterHeaders(TransportEntry entry, int var headers = entry.Headers.ToBuilder() .Set(KnownHeaders.DeadLetterAttempts, attempts.ToString(CultureInfo.InvariantCulture)) .Set(KnownHeaders.DeadLetterFailedAt, timeProvider.GetUtcNow().ToString("O", CultureInfo.InvariantCulture)) - .Set(KnownHeaders.DeadLetterOriginalDestination, entry.Destination); + .Set(KnownHeaders.DeadLetterOriginalDestination, entry.Destination.Key); if (exception is not null) { @@ -1193,18 +1193,16 @@ internal sealed class MessageListenerHandle : IAsyncDisposable private readonly Func _dispose; private int _isDisposed; - public MessageListenerHandle(string topic, string subscription, string source, string key, Func dispose) + public MessageListenerHandle(DestinationAddress source, string key, Func dispose) { - Topic = topic; - Subscription = subscription; Source = source; Key = key; _dispose = dispose; } - public string Topic { get; } - public string Subscription { get; } - public string Source { get; } + public DestinationAddress Source { get; } + public string Topic => Source.Topic ?? ""; + public string Subscription => Source.Role == DestinationRole.Subscription ? Source.Name : ""; public string Key { get; } // Disposing a single consumer handle detaches just that consumer from its source listener; the underlying receive @@ -1221,7 +1219,7 @@ public async ValueTask DisposeAsync() internal sealed record MessageListenerRegistration { public required Type MessageType { get; init; } - public required string Source { get; init; } + public required DestinationAddress Source { get; init; } public required Delegate Handler { get; init; } public required AckMode AckMode { get; init; } public required int MaxConcurrency { get; init; } @@ -1247,7 +1245,7 @@ public static MessageListenerRegistration Create(Delegate handler, ListenerConfi public bool Matches(MessageListenerRegistration other) { return MessageType == other.MessageType - && String.Equals(Source, other.Source, StringComparison.Ordinal) + && Source == other.Source && Handler == other.Handler && AckMode == other.AckMode && MaxConcurrency == other.MaxConcurrency diff --git a/src/Foundatio/Messaging/MessageRouting.cs b/src/Foundatio/Messaging/MessageRouting.cs index 1b1245cea..6b7edf1b3 100644 --- a/src/Foundatio/Messaging/MessageRouting.cs +++ b/src/Foundatio/Messaging/MessageRouting.cs @@ -57,14 +57,8 @@ public IReadOnlyList GetTopologyDeclarations() internal void Declare(DestinationDeclaration declaration) { ArgumentNullException.ThrowIfNull(declaration); - ArgumentException.ThrowIfNullOrEmpty(declaration.Name); - bool exists = TopologyDeclarations.Any(d => - String.Equals(d.Name, declaration.Name, StringComparison.Ordinal) - && d.Role == declaration.Role - && String.Equals(d.Source, declaration.Source, StringComparison.Ordinal)); - - if (!exists) + if (!TopologyDeclarations.Any(d => d.Address == declaration.Address)) TopologyDeclarations.Add(declaration); } @@ -191,18 +185,18 @@ private MessageRoutingOptionsBuilder Map(MessageRouteRole role, string route, pa private void DeclareQueue(string destination) { - _options.Declare(new DestinationDeclaration { Name = destination, Role = DestinationRole.Queue }); + _options.Declare(new DestinationDeclaration { Address = DestinationAddress.ForQueue(destination) }); } private void DeclareTopic(string topic) { - _options.Declare(new DestinationDeclaration { Name = topic, Role = DestinationRole.Topic }); + _options.Declare(new DestinationDeclaration { Address = DestinationAddress.ForTopic(topic) }); DeclareSubscription(topic); } private void RebuildSubscriptionDeclarations() { - _options.RemoveDeclarations(d => d.Role == DestinationRole.Subscription); + _options.RemoveDeclarations(d => d.Address.Role == DestinationRole.Subscription); if (!String.IsNullOrEmpty(_options.DefaultPubSubTopic)) DeclareSubscription(_options.DefaultPubSubTopic); @@ -227,7 +221,10 @@ private void DeclareSubscription(string topic) private void DeclareSubscription(string topic, string subscription) { - _options.Declare(new DestinationDeclaration { Name = subscription, Role = DestinationRole.Subscription, Source = topic }); + // The SAME canonical address the runtime subscribe path ensures and receives from — declaring the bare + // subscription name here while the runtime used a topic-qualified string is exactly the topology-vs-runtime + // identity mismatch DestinationAddress exists to prevent. + _options.Declare(new DestinationDeclaration { Address = DestinationAddress.ForSubscription(topic, subscription) }); } } diff --git a/src/Foundatio/Messaging/MessageTopology.cs b/src/Foundatio/Messaging/MessageTopology.cs index 1d3af8b34..b5ad04702 100644 --- a/src/Foundatio/Messaging/MessageTopology.cs +++ b/src/Foundatio/Messaging/MessageTopology.cs @@ -54,18 +54,11 @@ public async Task ValidateAsync(CancellationToken cancellationToken = default) var missing = new List(); foreach (var declaration in declarations) { - if (!await provisioning.ExistsAsync(declaration.Name, cancellationToken).AnyContext()) + if (!await provisioning.ExistsAsync(declaration.Address, cancellationToken).AnyContext()) missing.Add(declaration); } if (missing.Count > 0) - throw new InvalidOperationException($"Message topology is missing: {String.Join(", ", missing.Select(FormatDeclaration))}."); - } - - private static string FormatDeclaration(DestinationDeclaration declaration) - { - return String.IsNullOrEmpty(declaration.Source) - ? $"{declaration.Role} '{declaration.Name}'" - : $"{declaration.Role} '{declaration.Name}' from '{declaration.Source}'"; + throw new InvalidOperationException($"Message topology is missing: {String.Join(", ", missing.Select(d => d.Address.ToString()))}."); } } diff --git a/src/Foundatio/Messaging/MessageTransport.cs b/src/Foundatio/Messaging/MessageTransport.cs index 11f53997c..19bac2841 100644 --- a/src/Foundatio/Messaging/MessageTransport.cs +++ b/src/Foundatio/Messaging/MessageTransport.cs @@ -34,6 +34,51 @@ public enum DestinationRole Binding } +/// +/// The canonical identity of a transport destination: a name, the role that names the physical namespace it lives in, +/// and — for subscriptions — the owning topic. Every transport API (send, receive, subscribe, stats, settlement, +/// provisioning) uses this one value, so the same logical destination can never be spelled two ways on two paths. +/// +/// +/// is the destination's opaque string form ("{topic}/{name}" for subscriptions, Name +/// otherwise) for logging, metrics tags, and dictionary keys. Because a subscription key contains '/', a +/// transport must NOT assume it is a legal broker resource name (e.g. an SQS queue name) — map it to native resources +/// during and treat it as a lookup key thereafter. Topic and +/// subscription names must not contain '/'. +/// +public sealed record DestinationAddress +{ + public required string Name { get; init; } + public DestinationRole Role { get; init; } = DestinationRole.Queue; + + /// The owning topic when is ; null otherwise. + public string? Topic { get; init; } + + /// The canonical opaque string form: "{topic}/{name}" for subscriptions, Name otherwise. + public string Key => Topic is { Length: > 0 } topic ? $"{topic}/{Name}" : Name; + + public override string ToString() => $"{Role}:{Key}"; + + public static DestinationAddress ForQueue(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + return new DestinationAddress { Name = name, Role = DestinationRole.Queue }; + } + + public static DestinationAddress ForTopic(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + return new DestinationAddress { Name = name, Role = DestinationRole.Topic }; + } + + public static DestinationAddress ForSubscription(string topic, string subscription) + { + ArgumentException.ThrowIfNullOrEmpty(topic); + ArgumentException.ThrowIfNullOrEmpty(subscription); + return new DestinationAddress { Name = subscription, Role = DestinationRole.Subscription, Topic = topic }; + } +} + public sealed record TransportMessage { public required ReadOnlyMemory Body { get; init; } @@ -52,19 +97,12 @@ public sealed record TransportSendOptions { public MessagePriority Priority { get; init; } = MessagePriority.Normal; public DateTimeOffset? DeliverAt { get; init; } - - /// - /// The role of the destination being sent to. Lets a transport route the send without inferring (for example, a - /// queue send to SQS vs. a topic publish to SNS) — the caller always knows whether it is sending to a queue or a - /// topic, so it states it rather than relying on prior provisioning. - /// - public DestinationRole DestinationRole { get; init; } = DestinationRole.Queue; } public sealed record TransportEntry { public required string Id { get; init; } - public required string Destination { get; init; } + public required DestinationAddress Destination { get; init; } public required ReadOnlyMemory Body { get; init; } public MessageHeaders Headers { get; init; } = MessageHeaders.Empty; public int DeliveryCount { get; init; } = 1; @@ -136,9 +174,9 @@ public ReceiptExpiredException(string message, Exception innerException) : base( public sealed record DestinationDeclaration { - public required string Name { get; init; } - public DestinationRole Role { get; init; } = DestinationRole.Queue; - public string? Source { get; init; } + /// The canonical identity of the destination to provision — the SAME address the runtime later sends to, + /// receives from, and asks stats for, so provisioning and runtime can never disagree on a destination's identity. + public required DestinationAddress Address { get; init; } // Provider-specific creation arguments for transports that provision destinations (e.g. RabbitMQ queue arguments). // Retry and dead-letter behavior is owned by the core RetryPolicy, not declared here, so destinations stay simple. @@ -203,19 +241,19 @@ public interface ITransportInfo public interface IMessageTransport : IAsyncDisposable { - Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default); + Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default); Task CompleteAsync(TransportEntry entry, CancellationToken ct = default); Task AbandonAsync(TransportEntry entry, CancellationToken ct = default); } public interface ISupportsPull : IMessageTransport { - Task> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct = default); + Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct = default); } public interface ISupportsPush : IMessageTransport { - Task SubscribeAsync(string source, Func onMessage, PushOptions options, CancellationToken ct = default); + Task SubscribeAsync(DestinationAddress source, Func onMessage, PushOptions options, CancellationToken ct = default); } public interface ISupportsRedeliveryDelay : IMessageTransport @@ -234,7 +272,7 @@ public interface ISupportsDeadLetter : IMessageTransport // Reads dead-lettered entries for a destination so callers can inspect raw payloads (including poison messages // that never deserialized) and the dead-letter reason header. Read entries are removed from the dead-letter store. - Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct = default); + Task> ReceiveDeadLetteredAsync(DestinationAddress destination, ReceiveRequest request, CancellationToken ct = default); } public interface ISupportsLockRenewal : IMessageTransport @@ -249,22 +287,22 @@ public interface ISupportsVisibilityTimeout : IMessageTransport // unsatisfiable rather than relying on a silently clamped value. TimeSpan? MaxVisibilityTimeout { get; } - Task> ReceiveAsync(string source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default); + Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, TimeSpan visibility, CancellationToken ct = default); } public interface ISupportsStats : IMessageTransport { - Task GetStatsAsync(string destination, CancellationToken ct = default); + Task GetStatsAsync(DestinationAddress destination, CancellationToken ct = default); } public interface ISupportsProvisioning : IMessageTransport { Task EnsureAsync(IReadOnlyList declarations, CancellationToken ct = default); - Task DeleteAsync(string name, CancellationToken ct = default); - Task ExistsAsync(string name, CancellationToken ct = default); + Task DeleteAsync(DestinationAddress destination, CancellationToken ct = default); + Task ExistsAsync(DestinationAddress destination, CancellationToken ct = default); } public interface IPushSubscription : IAsyncDisposable { - string Source { get; } + DestinationAddress Source { get; } } diff --git a/src/Foundatio/Messaging/SubscriptionAddress.cs b/src/Foundatio/Messaging/SubscriptionAddress.cs deleted file mode 100644 index 07d923b35..000000000 --- a/src/Foundatio/Messaging/SubscriptionAddress.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; - -namespace Foundatio.Messaging; - -/// -/// The single, shared convention for addressing a pub/sub subscription as one transport destination string: -/// "{topic}/{subscription}". The topic is part of the identity so the same subscription name used on two topics -/// resolves to two distinct sources. -/// -/// -/// The resulting string (the source passed to receive/subscribe and carried as ) -/// is an opaque provider-agnostic key: because it contains '/' a transport must NOT assume it is a legal -/// broker resource name (e.g. an SQS queue name). Map it to native resources during -/// — which also supplies the structured topic via -/// — and treat it as a dictionary key thereafter, or parse it with -/// . Topic and subscription names must not contain '/'. Centralizing the convention here -/// (rather than each provider re-deriving it) keeps providers interoperable. -/// -public static class SubscriptionAddress -{ - /// Formats the topic-qualified subscription destination key. - public static string Format(string topic, string subscription) => $"{topic}/{subscription}"; - - /// - /// Splits a destination produced by into its topic and subscription. Returns false for a bare - /// (non-subscription) destination, leaving = the whole input and empty. - /// - public static bool TryParse(string destination, out string topic, out string subscription) - { - ArgumentNullException.ThrowIfNull(destination); - int slash = destination.IndexOf('/'); - if (slash <= 0 || slash >= destination.Length - 1) - { - topic = destination; - subscription = ""; - return false; - } - - topic = destination[..slash]; - subscription = destination[(slash + 1)..]; - return true; - } -} diff --git a/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs b/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs index 68cc82f21..99ca3f20d 100644 --- a/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs +++ b/tests/Foundatio.Aws.Tests/AwsMessageTransportTests.cs @@ -32,13 +32,13 @@ public async Task TextContentBody_RoundTripsThroughSqsAsync() // Non-ASCII JSON exercises UTF-8 round-trip through the SQS string body (the text-content path that avoids base64). string json = "{\"greeting\":\"héllo wörld\",\"n\":42}"; - await transport.EnsureAsync([new DestinationDeclaration { Name = "text-body", Role = DestinationRole.Queue }], cancellationToken); + await transport.EnsureAsync([new DestinationDeclaration { Address = DestinationAddress.ForQueue("text-body") }], cancellationToken); - await transport.SendAsync("text-body", + await transport.SendAsync(DestinationAddress.ForQueue("text-body"), [new TransportMessage { Body = Encoding.UTF8.GetBytes(json), ContentType = "application/json" }], new TransportSendOptions(), cancellationToken); - var entries = await transport.ReceiveAsync("text-body", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var entries = await transport.ReceiveAsync(DestinationAddress.ForQueue("text-body"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); var entry = Assert.Single(entries); Assert.Equal(json, Encoding.UTF8.GetString(entry.Body.Span)); await transport.CompleteAsync(entry, cancellationToken); @@ -57,13 +57,13 @@ public async Task BinaryContentBody_RoundTripsThroughSqsAsync() // Non-UTF-8 bytes must still round-trip (via base64) when no text content type is declared. byte[] payload = [0x00, 0x01, 0xFF, 0xFE, 0x10, 0x80]; - await transport.EnsureAsync([new DestinationDeclaration { Name = "binary-body", Role = DestinationRole.Queue }], cancellationToken); + await transport.EnsureAsync([new DestinationDeclaration { Address = DestinationAddress.ForQueue("binary-body") }], cancellationToken); - await transport.SendAsync("binary-body", + await transport.SendAsync(DestinationAddress.ForQueue("binary-body"), [new TransportMessage { Body = payload }], new TransportSendOptions(), cancellationToken); - var entries = await transport.ReceiveAsync("binary-body", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); + var entries = await transport.ReceiveAsync(DestinationAddress.ForQueue("binary-body"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, cancellationToken); var entry = Assert.Single(entries); Assert.Equal(payload, entry.Body.ToArray()); await transport.CompleteAsync(entry, cancellationToken); diff --git a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs index 145a4a5cf..5d691efea 100644 --- a/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisJobStoreIntegrationTests.cs @@ -184,7 +184,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, - Destination = "nightly", + JobName = "nightly", Body = Array.Empty(), DueUtc = now, JobId = jobId @@ -258,7 +258,7 @@ private sealed class CappedDelayTransport : IMessageTransport, ISupportsPull, IT public TransportCapabilities GetCapabilities(DestinationRole role) => new() { DelayedDelivery = true, MaxDeliveryDelay = MaxDeliveryDelay }; - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendCount += messages.Count; LastSendOptions = options; @@ -273,7 +273,7 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) => Task.FromResult>(_entries.Count > 0 ? [_entries.Dequeue()] : []); public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; diff --git a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs index d3e7f39cc..e35bfab4c 100644 --- a/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs +++ b/tests/Foundatio.Redis.Tests/RedisStreamsTransportIntegrationTests.cs @@ -41,24 +41,24 @@ public async Task CrashedConsumer_LeaseLapses_AnotherInstanceReclaimsAndComplete await using var nodeA = CreateTransport(connection, prefix, "node-a"); await using var nodeB = CreateTransport(connection, prefix, "node-b"); - await nodeA.EnsureAsync([new DestinationDeclaration { Name = "work", Role = DestinationRole.Queue }], ct); - await nodeA.SendAsync("work", [Message("survive-me")], new TransportSendOptions(), ct); + await nodeA.EnsureAsync([new DestinationDeclaration { Address = DestinationAddress.ForQueue("work") }], ct); + await nodeA.SendAsync(DestinationAddress.ForQueue("work"), [Message("survive-me")], new TransportSendOptions(), ct); // node-a receives and then "crashes" — it never settles the message. - var heldByA = Assert.Single(await nodeA.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibility, ct)); + var heldByA = Assert.Single(await nodeA.ReceiveAsync(DestinationAddress.ForQueue("work"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, visibility, ct)); Assert.Equal(1, heldByA.DeliveryCount); // While node-a's lease is live, node-b must not see it. - Assert.Empty(await nodeB.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibility, ct)); + Assert.Empty(await nodeB.ReceiveAsync(DestinationAddress.ForQueue("work"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromMilliseconds(100) }, visibility, ct)); // After the lease lapses, node-b reclaims the in-flight message (lease state lives in Redis) and completes it. - var reclaimedByB = Assert.Single(await nodeB.ReceiveAsync("work", new ReceiveRequest { MaxWaitTime = visibility + TimeSpan.FromSeconds(5) }, visibility, ct)); + var reclaimedByB = Assert.Single(await nodeB.ReceiveAsync(DestinationAddress.ForQueue("work"), new ReceiveRequest { MaxWaitTime = visibility + TimeSpan.FromSeconds(5) }, visibility, ct)); Assert.Equal(heldByA.Id, reclaimedByB.Id); Assert.Equal(2, reclaimedByB.DeliveryCount); Assert.Equal("survive-me", System.Text.Encoding.UTF8.GetString(reclaimedByB.Body.Span)); await nodeB.CompleteAsync(reclaimedByB, ct); - var stats = await nodeB.GetStatsAsync("work", ct); + var stats = await nodeB.GetStatsAsync(DestinationAddress.ForQueue("work"), ct); Assert.Equal(0, stats.Queued); Assert.Equal(0, stats.Working); } @@ -103,18 +103,18 @@ public async Task Core_RetriesFailedHandler_ThenDeadLettersAfterMaxAttemptsAsync Assert.Equal(2, Volatile.Read(ref retryAttempts)); // The poison message lands in the dead-letter stream after exhausting its 2 attempts. - MessageDestinationStats stats = await transport.GetStatsAsync("streams-poison", ct); + MessageDestinationStats stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("streams-poison"), ct); for (int i = 0; i < 100 && stats.Deadletter == 0; i++) { await Task.Delay(100, ct); - stats = await transport.GetStatsAsync("streams-poison", ct); + stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("streams-poison"), ct); } Assert.Equal(1, stats.Deadletter); Assert.Equal(0, stats.Working); // The poison payload is inspectable in the dead-letter stream with a reason recorded by the core. - var deadLettered = Assert.Single(await transport.ReceiveDeadLetteredAsync("streams-poison", new ReceiveRequest { MaxMessages = 10 }, ct)); + var deadLettered = Assert.Single(await transport.ReceiveDeadLetteredAsync(DestinationAddress.ForQueue("streams-poison"), new ReceiveRequest { MaxMessages = 10 }, ct)); Assert.NotEmpty(deadLettered.Headers[KnownHeaders.DeadLetterReason]); } @@ -212,19 +212,19 @@ public async Task Publish_CompletedByOneGroup_StillDeliveredToSlowerGroupAsync() await transport.EnsureAsync( [ - new DestinationDeclaration { Name = "iso-topic", Role = DestinationRole.Topic }, - new DestinationDeclaration { Name = "iso-topic/sub-a", Role = DestinationRole.Subscription, Source = "iso-topic" }, - new DestinationDeclaration { Name = "iso-topic/sub-b", Role = DestinationRole.Subscription, Source = "iso-topic" } + new DestinationDeclaration { Address = DestinationAddress.ForTopic("iso-topic") }, + new DestinationDeclaration { Address = DestinationAddress.ForSubscription("iso-topic", "sub-a") }, + new DestinationDeclaration { Address = DestinationAddress.ForSubscription("iso-topic", "sub-b") } ], ct); - await transport.SendAsync("iso-topic", [Message("retained")], new TransportSendOptions { DestinationRole = DestinationRole.Topic }, ct); + await transport.SendAsync(DestinationAddress.ForTopic("iso-topic"), [Message("retained")], new TransportSendOptions(), ct); // Group A reads and completes FIRST; the entry must remain on the topic stream for group B (completing must // not delete a shared topic entry other groups haven't read yet). - var byA = Assert.Single(await transport.ReceiveAsync("iso-topic/sub-a", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); + var byA = Assert.Single(await transport.ReceiveAsync(DestinationAddress.ForSubscription("iso-topic", "sub-a"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); await transport.CompleteAsync(byA, ct); - var byB = Assert.Single(await transport.ReceiveAsync("iso-topic/sub-b", new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); + var byB = Assert.Single(await transport.ReceiveAsync(DestinationAddress.ForSubscription("iso-topic", "sub-b"), new ReceiveRequest { MaxWaitTime = TimeSpan.FromSeconds(2) }, ct)); Assert.Equal("retained", System.Text.Encoding.UTF8.GetString(byB.Body.Span)); await transport.CompleteAsync(byB, ct); } diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index 0df723d3f..ddf060a45 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -181,7 +181,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = "dispatch-1", Kind = ScheduledDispatchKind.QueueMessage, - Destination = "work", + Destination = DestinationAddress.ForQueue("work"), Body = "hello"u8.ToArray(), DueUtc = now.AddSeconds(-1) }, cancellationToken); @@ -190,7 +190,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = "dispatch-2", Kind = ScheduledDispatchKind.QueueMessage, - Destination = "work", + Destination = DestinationAddress.ForQueue("work"), Body = "later"u8.ToArray(), DueUtc = now.AddHours(1) }, cancellationToken); diff --git a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs index ddbcc18e5..4b8fdafec 100644 --- a/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobSchedulerTests.cs @@ -207,7 +207,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = "delayed-message", Kind = ScheduledDispatchKind.QueueMessage, - Destination = "work", + Destination = DestinationAddress.ForQueue("work"), Body = "hello"u8.ToArray(), DueUtc = now }, cancellationToken); @@ -216,7 +216,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState Assert.Equal(1, completed); var pull = Assert.IsAssignableFrom(transport); - var entries = await pull.ReceiveAsync("work", new ReceiveRequest { MaxMessages = 1, MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); + var entries = await pull.ReceiveAsync(DestinationAddress.ForQueue("work"), new ReceiveRequest { MaxMessages = 1, MaxWaitTime = TimeSpan.FromMilliseconds(50) }, cancellationToken); var entry = Assert.Single(entries); Assert.Equal("delayed-message", entry.Id); Assert.Equal("hello"u8.ToArray(), entry.Body.ToArray()); @@ -296,7 +296,7 @@ await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, - Destination = "nightly", + JobName = "nightly", Body = Array.Empty(), DueUtc = now, JobId = jobId @@ -349,7 +349,7 @@ public async Task RunDueOccurrencesAsync_WhenOccurrenceIsTerminal_RetiresDispatc await scheduler.ScheduleAsync(new ScheduledJobDefinition { Name = "nightly", Cron = "* * * * *", JobType = typeof(ScheduledProbeJob) }, cancellationToken); // A worker completed the occurrence but crashed before retiring its dispatch: a terminal job with a live dispatch. await store.CreateIfAbsentAsync(new JobState { JobId = jobId, Name = "nightly", Status = JobStatus.Completed, ScheduledForUtc = now.AddSeconds(-30) }, cancellationToken); - await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, Destination = "nightly", Body = Array.Empty(), DueUtc = now, JobId = jobId }, cancellationToken); + await store.ScheduleDispatchAsync(new ScheduledDispatchState { DispatchId = jobId, Kind = ScheduledDispatchKind.JobOccurrence, JobName = "nightly", Body = Array.Empty(), DueUtc = now, JobId = jobId }, cancellationToken); await processor.RunDueOccurrencesAsync(now, cancellationToken: cancellationToken); diff --git a/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs index 409146645..4ea2857cb 100644 --- a/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs +++ b/tests/Foundatio.Tests/Messaging/FailureHandlingTests.cs @@ -29,7 +29,7 @@ public async Task DeadLetterOn_MatchingException_DeadLettersOnFirstAttemptAsync( Assert.Equal(1, stats.Deadletter); Assert.Equal(1, Volatile.Read(ref attempts)); // never retried - var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync("failing-item", new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); + var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync(DestinationAddress.ForQueue("failing-item"), new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); Assert.Equal("unrecoverable:ArgumentException", dead.Headers[KnownHeaders.DeadLetterReason]); } @@ -71,7 +71,7 @@ public async Task DeadLetter_StampsForensicsHeadersAsync() await bus.SendAsync(new FailingItem { Data = "doomed" }, cancellationToken: cancellationToken); await WaitForDeadLetterAsync(transport, "failing-item", cancellationToken); - var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync("failing-item", new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); + var dead = Assert.Single(await transport.ReceiveDeadLetteredAsync(DestinationAddress.ForQueue("failing-item"), new ReceiveRequest { MaxMessages = 10 }, cancellationToken)); Assert.Equal(typeof(InvalidOperationException).FullName, dead.Headers[KnownHeaders.DeadLetterExceptionType]); Assert.Equal("the failure detail", dead.Headers[KnownHeaders.DeadLetterExceptionMessage]); @@ -101,12 +101,13 @@ public void DefaultBackoff_MatchesTheConvergedCurve() private static async Task WaitForDeadLetterAsync(InMemoryMessageTransport transport, string destination, CancellationToken cancellationToken) { - var stats = await transport.GetStatsAsync(destination, cancellationToken); + var address = DestinationAddress.ForQueue(destination); + var stats = await transport.GetStatsAsync(address, cancellationToken); long deadline = Environment.TickCount64 + 10_000; while (stats.Deadletter == 0 && Environment.TickCount64 < deadline) { await Task.Delay(25, cancellationToken); - stats = await transport.GetStatsAsync(destination, cancellationToken); + stats = await transport.GetStatsAsync(address, cancellationToken); } return stats; diff --git a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs index e63a064e1..0a4821e37 100644 --- a/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs +++ b/tests/Foundatio.Tests/Messaging/InMemoryMessageTransportTests.cs @@ -16,19 +16,19 @@ protected override IMessageTransport CreateTransport() } [Fact] - public void SubscriptionAddress_FormatsAndParsesTopicAndSubscription() + public void DestinationAddress_KeyEncodesTopicAndSubscription() { - string destination = SubscriptionAddress.Format("orders", "sub-a"); - Assert.Equal("orders/sub-a", destination); + var destination = DestinationAddress.ForSubscription("orders", "sub-a"); + Assert.Equal("orders/sub-a", destination.Key); + Assert.Equal("orders", destination.Topic); + Assert.Equal("sub-a", destination.Name); + Assert.Equal(DestinationRole.Subscription, destination.Role); - Assert.True(SubscriptionAddress.TryParse(destination, out string topic, out string subscription)); - Assert.Equal("orders", topic); - Assert.Equal("sub-a", subscription); - - // A bare (non-subscription) destination is not a subscription address. - Assert.False(SubscriptionAddress.TryParse("orders", out string bareTopic, out string bareSubscription)); - Assert.Equal("orders", bareTopic); - Assert.Equal("", bareSubscription); + // A bare (non-subscription) destination has no topic and a bare key. + var bare = DestinationAddress.ForQueue("orders"); + Assert.Null(bare.Topic); + Assert.Equal("orders", bare.Key); + Assert.NotEqual(destination, bare); } [Fact] diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 89b954a6c..2d4acd221 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -44,8 +44,8 @@ public async Task PublishAsync_FansOutToMultipleSubscriptionsAsync() await firstReceived.WaitAsync(TimeSpan.FromSeconds(2)); await secondReceived.WaitAsync(TimeSpan.FromSeconds(2)); - var firstStats = await transport.GetStatsAsync(first.Source, cancellationToken); - var secondStats = await transport.GetStatsAsync(second.Source, cancellationToken); + var firstStats = await transport.GetStatsAsync(DestinationAddress.ForSubscription(first.Topic, first.Subscription), cancellationToken); + var secondStats = await transport.GetStatsAsync(DestinationAddress.ForSubscription(second.Topic, second.Subscription), cancellationToken); Assert.Equal(1, firstStats.Completed); Assert.Equal(1, secondStats.Completed); } @@ -85,7 +85,7 @@ await pubSub.PublishBatchAsync([ ], cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - await WaitForCompletedAsync(transport, first.Source, 2, cancellationToken); + await WaitForCompletedAsync(transport, DestinationAddress.ForSubscription(first.Topic, first.Subscription), 2, cancellationToken); Assert.Equal(first.Topic, second.Topic); Assert.Equal(first.Subscription, second.Subscription); @@ -167,7 +167,7 @@ await pubSub.PublishBatchAsync([ ], cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForSubscription(subscription.Topic, subscription.Subscription), cancellationToken); Assert.Equal(2, stats.Completed); } @@ -259,7 +259,7 @@ public async Task SubscribeAsync_WhenHandlerFails_RedeliversAsync() await pubSub.PublishAsync(new PreviewEvent { Data = "retry" }, cancellationToken: cancellationToken); await received.WaitAsync(TimeSpan.FromSeconds(2)); - var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForSubscription(subscription.Topic, subscription.Subscription), cancellationToken); Assert.Equal(1, stats.Completed); Assert.Equal(1, stats.Abandoned); } @@ -343,7 +343,7 @@ await pubSub.PublishBatchAsync(new object[] Assert.Contains(typeof(PreviewEvent).FullName!, messageTypes); Assert.Contains(typeof(OtherEvent).FullName!, messageTypes); - var stats = await transport.GetStatsAsync(subscription.Source, cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForSubscription(subscription.Topic, subscription.Subscription), cancellationToken); Assert.Equal(2, stats.Completed); } @@ -366,7 +366,7 @@ public async Task PublishAsync_WithDelay_OnTopicWithoutNativeDelay_RoutesThrough Assert.Equal(0, transport.SendCount); Assert.Equal(1, await processor.RunDueOccurrencesAsync(DateTimeOffset.UtcNow.AddMinutes(10), cancellationToken: cancellationToken)); Assert.Equal(1, transport.SendCount); - Assert.Equal(DestinationRole.Topic, transport.LastSendOptions?.DestinationRole); + Assert.Equal(DestinationRole.Topic, transport.LastDestination?.Role); Assert.Null(transport.LastSendOptions?.DeliverAt); // the store dispatches it as due; the delay is spent, not forwarded // A delayed QUEUE send within the same transport's queue ceiling still uses the native path. @@ -375,7 +375,7 @@ public async Task PublishAsync_WithDelay_OnTopicWithoutNativeDelay_RoutesThrough Assert.NotNull(transport.LastSendOptions?.DeliverAt); } - private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, string destination, long expected, CancellationToken cancellationToken) + private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, DestinationAddress destination, long expected, CancellationToken cancellationToken) { var deadline = DateTimeOffset.UtcNow.AddSeconds(2); while (DateTimeOffset.UtcNow < deadline) @@ -408,6 +408,7 @@ private sealed class RoleSplitDelayTransport : IMessageTransport, ISupportsPull, public int SendCount { get; private set; } public TransportSendOptions? LastSendOptions { get; private set; } + public DestinationAddress? LastDestination { get; private set; } public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; public IReadOnlySet SupportedRoles => @@ -417,13 +418,14 @@ public TransportCapabilities GetCapabilities(DestinationRole role) => role == De ? TransportCapabilities.None : new TransportCapabilities { DelayedDelivery = true, MaxDeliveryDelay = _queueMaxDelay }; - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { - if (options.DestinationRole == DestinationRole.Topic && options.DeliverAt is { } deliverAt && deliverAt > DateTimeOffset.UtcNow) + if (destination.Role == DestinationRole.Topic && options.DeliverAt is { } deliverAt && deliverAt > DateTimeOffset.UtcNow) throw new NotSupportedException("Topics have no native delayed delivery."); SendCount += messages.Count; LastSendOptions = options; + LastDestination = destination; var items = new SendItemResult[messages.Count]; for (int i = 0; i < messages.Count; i++) items[i] = new SendItemResult { MessageId = messages[i].MessageId ?? Guid.NewGuid().ToString("N") }; @@ -431,7 +433,7 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) => Task.FromResult>([]); public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; diff --git a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs index 320f10266..af0f7a4a8 100644 --- a/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs +++ b/tests/Foundatio.Tests/Queue/BasicQueueTransport.cs @@ -19,9 +19,9 @@ internal sealed class BasicQueueTransport : IMessageTransport, ISupportsPull, IS { private readonly ConcurrentDictionary _destinations = new(StringComparer.OrdinalIgnoreCase); - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { - var dest = _destinations.GetOrAdd(destination, static _ => new Destination()); + var dest = _destinations.GetOrAdd(destination.Key, static _ => new Destination()); var results = new SendItemResult[messages.Count]; for (int index = 0; index < messages.Count; index++) { @@ -35,9 +35,9 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) { - var dest = _destinations.GetOrAdd(source, static _ => new Destination()); + var dest = _destinations.GetOrAdd(source.Key, static _ => new Destination()); int max = request.MaxMessages <= 0 ? 1 : request.MaxMessages; DateTimeOffset? deadline = request.MaxWaitTime is { } wait && wait > TimeSpan.Zero ? DateTimeOffset.UtcNow.Add(wait) : null; var entries = new List(max); @@ -56,7 +56,7 @@ public async Task> ReceiveAsync(string source, Rec Body = stored.Body, Headers = stored.Headers, DeliveryCount = stored.DeliveryCount, - Receipt = new Receipt { TransportState = new BasicReceipt(source, token) } + Receipt = new Receipt { TransportState = new BasicReceipt(source.Key, token) } }); } @@ -99,10 +99,10 @@ public Task DeadLetterAsync(TransportEntry entry, string? reason, CancellationTo return Task.CompletedTask; } - public Task> ReceiveDeadLetteredAsync(string destination, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveDeadLetteredAsync(DestinationAddress destination, ReceiveRequest request, CancellationToken ct) { var entries = new List(); - if (_destinations.TryGetValue(destination, out var dest)) + if (_destinations.TryGetValue(destination.Key, out var dest)) { int max = request.MaxMessages <= 0 ? 1 : request.MaxMessages; while (entries.Count < max && dest.Dead.TryDequeue(out var stored)) @@ -122,9 +122,9 @@ public Task> ReceiveDeadLetteredAsync(string desti return Task.FromResult>(entries); } - public Task GetStatsAsync(string destination, CancellationToken ct) + public Task GetStatsAsync(DestinationAddress destination, CancellationToken ct) { - if (!_destinations.TryGetValue(destination, out var dest)) + if (!_destinations.TryGetValue(destination.Key, out var dest)) return Task.FromResult(new MessageDestinationStats()); return Task.FromResult(new MessageDestinationStats diff --git a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs index 23898f440..aa6213ec0 100644 --- a/tests/Foundatio.Tests/Queue/MessageQueueTests.cs +++ b/tests/Foundatio.Tests/Queue/MessageQueueTests.cs @@ -46,7 +46,7 @@ public async Task EnqueueAsync_WithOptions_CanReceiveAndCompleteAsync() await received.CompleteAsync(cancellationToken); - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cancellationToken); Assert.Equal(1, stats.Completed); Assert.Equal(0, stats.Working); } @@ -128,7 +128,7 @@ public async Task RejectAsync_Terminal_DeadLettersAsync() await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }, cancellationToken); - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cancellationToken); Assert.Equal(1, stats.Deadletter); Assert.Equal(0, stats.Working); } @@ -176,7 +176,7 @@ public async Task StartConsumerAsync_WithManualAck_DoesNotAutoCompleteAsync() await Task.Delay(200, cts.Token); // Manual ack: the handler ran but did not settle, so the message stays in flight and is not auto-completed. - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cancellationToken); Assert.Equal(0, stats.Completed); Assert.Equal(1, stats.Working); } @@ -200,7 +200,7 @@ public async Task StartConsumerAsync_WithPoisonMessage_DeadLettersAndKeepsConsum // A poison (undeserializable) payload must be dead-lettered without tearing down the consumer loop, so the // subsequent valid message is still delivered. - await transport.SendAsync("preview-work-item", [ + await transport.SendAsync(DestinationAddress.ForQueue("preview-work-item"), [ new TransportMessage { Body = System.Text.Encoding.UTF8.GetBytes("}{ not json"), Headers = MessageHeaders.Empty } ], new TransportSendOptions(), cts.Token); await queue.SendAsync(new PreviewWorkItem { Data = "good" }, cancellationToken: cts.Token); @@ -406,7 +406,7 @@ public async Task SendAsync_WithExpiredMessage_IsDeadLetteredNotDeliveredAsync() var received = await collector.NextAsync(TimeSpan.FromMilliseconds(500), cancellationToken); Assert.Null(received); - var stats = await transport.GetStatsAsync("preview-work-item", cancellationToken); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cancellationToken); Assert.Equal(1, stats.Deadletter); } @@ -459,9 +459,9 @@ public async Task AddFoundatio_WithRouting_RegistersRouterAndTopologyAsync() var topology = provider.GetRequiredService(); var declarations = topology.GetDeclarations(); - Assert.Contains(declarations, d => d.Role == DestinationRole.Queue && d.Name == "all-work"); - Assert.Contains(declarations, d => d.Role == DestinationRole.Topic && d.Name == "grouped-events"); - Assert.Contains(declarations, d => d.Role == DestinationRole.Subscription && d.Name == "billing-service" && d.Source == "grouped-events"); + Assert.Contains(declarations, d => d.Address.Role == DestinationRole.Queue && d.Address.Name == "all-work"); + Assert.Contains(declarations, d => d.Address.Role == DestinationRole.Topic && d.Address.Name == "grouped-events"); + Assert.Contains(declarations, d => d.Address.Role == DestinationRole.Subscription && d.Address.Name == "billing-service" && d.Address.Topic == "grouped-events"); await Assert.ThrowsAsync(async () => await topology.ValidateAsync(cancellationToken)); await topology.EnsureAsync(cancellationToken); @@ -617,17 +617,18 @@ public async Task ReceiveAsync_WithDefaultQueueRoute_ReturnsRawMessageAsync() private static async Task WaitForCompletedAsync(InMemoryMessageTransport transport, string destination, CancellationToken cancellationToken) { + var address = DestinationAddress.ForQueue(destination); var deadline = DateTimeOffset.UtcNow.AddSeconds(2); while (DateTimeOffset.UtcNow < deadline) { - var stats = await transport.GetStatsAsync(destination, cancellationToken); + var stats = await transport.GetStatsAsync(address, cancellationToken); if (stats.Completed == 1) return; await Task.Delay(TimeSpan.FromMilliseconds(10), cancellationToken); } - var finalStats = await transport.GetStatsAsync(destination, cancellationToken); + var finalStats = await transport.GetStatsAsync(address, cancellationToken); Assert.Equal(1, finalStats.Completed); } @@ -779,12 +780,12 @@ public async Task StartConsumerAsync_UnmatchedType_DeadLettersAndKeepsConsumingA // It is retried and finally dead-lettered as "no-handler" once the configured unmatched budget is exhausted. for (int i = 0; i < 400; i++) { - if ((await transport.GetStatsAsync("shared-demux", cts.Token)).Deadletter == 1) + if ((await transport.GetStatsAsync(DestinationAddress.ForQueue("shared-demux"), cts.Token)).Deadletter == 1) break; await Task.Delay(TimeSpan.FromMilliseconds(25), cts.Token); } - Assert.Equal(1, (await transport.GetStatsAsync("shared-demux", cts.Token)).Deadletter); + Assert.Equal(1, (await transport.GetStatsAsync(DestinationAddress.ForQueue("shared-demux"), cts.Token)).Deadletter); // The loop survived the unmatched message and keeps consuming the type it does handle. await queue.SendAsync(new SharedAWorkItem { Data = "ok" }, cancellationToken: cts.Token); @@ -857,12 +858,12 @@ public async Task RetryPolicy_BackoffOnTransportWithoutDelaySupport_DegradesToIm await queue.SendAsync(new PreviewWorkItem { Data = "doomed" }, cancellationToken: cts.Token); // All three attempts happen without a 10s stall, ending in the transport's native dead-letter sink. - var stats = await transport.GetStatsAsync("preview-work-item", cts.Token); + var stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cts.Token); long deadline = Environment.TickCount64 + 10_000; while (stats.Deadletter == 0 && Environment.TickCount64 < deadline) { await Task.Delay(25, cts.Token); - stats = await transport.GetStatsAsync("preview-work-item", cts.Token); + stats = await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cts.Token); } Assert.Equal(1, stats.Deadletter); @@ -889,12 +890,12 @@ public async Task StartConsumerAsync_UsesDefaultRetryPolicyMaxAttempts_WhenConsu for (int i = 0; i < 400; i++) { - if ((await transport.GetStatsAsync("preview-work-item", cts.Token)).Deadletter == 1) + if ((await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cts.Token)).Deadletter == 1) break; await Task.Delay(TimeSpan.FromMilliseconds(25), cts.Token); } - Assert.Equal(1, (await transport.GetStatsAsync("preview-work-item", cts.Token)).Deadletter); + Assert.Equal(1, (await transport.GetStatsAsync(DestinationAddress.ForQueue("preview-work-item"), cts.Token)).Deadletter); Assert.Equal(2, attempts); } @@ -907,7 +908,7 @@ public async Task DisposeAsync_RespectsTransportOwnershipAsync() var shared = new InMemoryMessageTransport(); var nonOwning = new MessageBus(shared, new MessageBusOptions { OwnsTransport = false }); await nonOwning.DisposeAsync(); - await shared.SendAsync("still-alive", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken); + await shared.SendAsync(DestinationAddress.ForQueue("still-alive"), [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken); await shared.DisposeAsync(); // Owning client (default): disposing the client disposes the transport. @@ -915,7 +916,7 @@ public async Task DisposeAsync_RespectsTransportOwnershipAsync() var owning = new MessageBus(owned); await owning.DisposeAsync(); await Assert.ThrowsAsync(async () => - await owned.SendAsync("dead", [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken)); + await owned.SendAsync(DestinationAddress.ForQueue("dead"), [new TransportMessage { Body = ReadOnlyMemory.Empty }], new TransportSendOptions(), cancellationToken)); } [Fact] @@ -990,7 +991,7 @@ public BatchLimitTransport(int maxBatchSize, long? maxMessageBytes = null) public TransportCapabilities GetCapabilities(DestinationRole role) => new() { Ordering = OrderingGuarantee.Fifo, MaxBatchSize = MaxBatchSize, MaxMessageBytes = MaxMessageBytes }; - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendBatchSizes.Add(messages.Count); var items = new SendItemResult[messages.Count]; @@ -1024,7 +1025,7 @@ public CappedDelayTransport(TimeSpan? maxDeliveryDelay) public TransportCapabilities GetCapabilities(DestinationRole role) => new() { DelayedDelivery = true, MaxDeliveryDelay = MaxDeliveryDelay }; - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { SendCount += messages.Count; LastSendOptions = options; @@ -1039,7 +1040,7 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) { return Task.FromResult>(_entries.Count > 0 ? [_entries.Dequeue()] : []); } @@ -1055,9 +1056,9 @@ private sealed class NoDeadLetterTransport : IMessageTransport, ISupportsPull { private readonly ConcurrentDictionary> _queues = new(StringComparer.Ordinal); - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) { - var queue = _queues.GetOrAdd(destination, _ => new ConcurrentQueue()); + var queue = _queues.GetOrAdd(destination.Key, _ => new ConcurrentQueue()); var items = new SendItemResult[messages.Count]; for (int i = 0; i < messages.Count; i++) { @@ -1069,9 +1070,9 @@ public Task SendAsync(string destination, IReadOnlyList> ReceiveAsync(string source, ReceiveRequest request, CancellationToken ct) + public Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) { - if (_queues.TryGetValue(source, out var queue) && queue.TryDequeue(out var entry)) + if (_queues.TryGetValue(source.Key, out var queue) && queue.TryDequeue(out var entry)) return Task.FromResult>([entry]); return Task.FromResult>([]); @@ -1081,7 +1082,7 @@ public Task> ReceiveAsync(string source, ReceiveRe public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) { - _queues.GetOrAdd(entry.Destination, _ => new ConcurrentQueue()).Enqueue(entry with { DeliveryCount = entry.DeliveryCount + 1 }); + _queues.GetOrAdd(entry.Destination.Key, _ => new ConcurrentQueue()).Enqueue(entry with { DeliveryCount = entry.DeliveryCount + 1 }); return Task.CompletedTask; } @@ -1093,7 +1094,7 @@ private sealed class DisposeCountingTransport : IMessageTransport { public int DisposeCount { get; private set; } - public Task SendAsync(string destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) => Task.FromResult(new SendResult { Items = Array.Empty() }); public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) => Task.CompletedTask; From 12baa624bbf58690b5f22781c3567dd55a41a39e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:28:24 -0500 Subject: [PATCH 50/57] Handler delivery intent: subscriptions declare Sent, Published, or Both MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every subscription used to wire both delivery channels unconditionally — a command-only handler still got a subscriber group provisioned and an idle topic listener, and the core never consulted ITransportInfo.SupportedRoles, so a queue-only transport silently wired listeners on channels it could never feed. MessageSubscriptionOptions.Deliveries (flags: Sent | Published, default Both) states the intent: the default narrows to whatever channels the transport's SupportedRoles can serve (skipped channel logged at debug), an explicit single-channel request the transport cannot serve throws NotSupportedException, and a transport that can serve neither always throws. IMessageSubscription's channel properties are empty for unwired channels. (Review feedback #1.) Co-Authored-By: Claude Fable 5 --- src/Foundatio/Messaging/MessageBus.cs | 133 ++++++++---- src/Foundatio/Messaging/MessageClientCore.cs | 7 + .../Messaging/DeliveryIntentTests.cs | 197 ++++++++++++++++++ 3 files changed, 300 insertions(+), 37 deletions(-) create mode 100644 tests/Foundatio.Tests/Messaging/DeliveryIntentTests.cs diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index dec4a7c3f..e93412403 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -35,14 +35,38 @@ public sealed record MessagePublishOptions public MessageHeaders? Headers { get; init; } } +/// +/// The delivery channels a subscription consumes. By default a handler listens on both of its type's channels; a +/// handler that only ever consumes commands (or only events) states that intent so no idle listener is wired — and so +/// a queue-only or topic-only transport can serve it. +/// +[Flags] +public enum MessageDeliveries +{ + /// Sent commands from the type's queue-role destination (competing consumers). + Sent = 1, + + /// Published events, delivered via this subscriber's group on the type's topic. + Published = 2, + + Both = Sent | Published +} + /// /// Options for attaching a handler to a message type — via AddFoundatio().Messaging.AddHandler<T, THandler>(o => ...) -/// or programmatically via . A subscription listens on the type's two -/// delivery channels: sent messages (one handler instance across the fleet processes each) and published messages -/// (delivered per the subscription identity below). +/// or programmatically via . By default a subscription listens on the +/// type's two delivery channels — sent messages (one handler instance across the fleet processes each) and published +/// messages (delivered per the subscription identity below) — narrowed by . /// public sealed class MessageSubscriptionOptions { + /// + /// Which delivery channels this subscription consumes. Default : on a + /// transport that supports only one channel's roles, the unsupported channel is skipped (logged at debug). + /// Explicitly requesting a single channel the transport cannot serve throws . + /// + public MessageDeliveries Deliveries { get; set; } = MessageDeliveries.Both; + /// /// When true, published messages are received by EVERY running instance (each takes a unique subscription), /// instead of once per service. For per-instance local state — cache invalidation, config reload. Mutually @@ -120,7 +144,10 @@ public MessageSubscriptionOptions DeadLetterOn() where TException : } } -/// A started subscription; disposing detaches the handler from the message type's delivery channels. +/// +/// A started subscription; disposing detaches the handler from the message type's delivery channels. Channel-specific +/// properties are empty when that channel was not wired (see ). +/// public interface IMessageSubscription : IAsyncDisposable { /// Consumer identity; subscriptions sharing a key on a channel form one competing group. @@ -260,40 +287,70 @@ public Task PublishBatchAsync(IEnumerable messages, MessagePublishOption return _core.SendBatchAsync(ScheduledDispatchKind.PubSubMessage, messages, null, ToEnvelope(options), type => GetTopic(type, options.Topic), EnsureTopicAsync, cancellationToken); } - public async Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class + public Task SubscribeAsync(Func, CancellationToken, Task> handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) where T : class { ArgumentNullException.ThrowIfNull(handler); - var channels = BuildChannels(options, typeof(T)); - var sent = await _core.StartListenerAsync(channels.Send, handler, cancellationToken).AnyContext(); - try + return SubscribeCoreAsync(options, typeof(T), (config, token) => _core.StartListenerAsync(config, handler, token), cancellationToken); + } + + public Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(handler); + return SubscribeCoreAsync(options, typeof(object), (config, token) => _core.StartListenerAsync(config, handler, token), cancellationToken); + } + + private async Task SubscribeCoreAsync(MessageSubscriptionOptions? options, Type fallbackType, Func> start, CancellationToken cancellationToken) + { + var deliveries = options?.Deliveries ?? MessageDeliveries.Both; + if ((deliveries & MessageDeliveries.Both) == 0) + throw new ArgumentException("Deliveries must include at least one delivery channel.", nameof(options)); + + bool wantSent = deliveries.HasFlag(MessageDeliveries.Sent); + bool wantPublished = deliveries.HasFlag(MessageDeliveries.Published); + bool canSend = _core.SupportsRole(DestinationRole.Queue); + bool canPublish = _core.SupportsRole(DestinationRole.Topic) && _core.SupportsRole(DestinationRole.Subscription); + + // An explicit single-channel request the transport cannot serve is a configuration error and must fail loudly; + // the default Both narrows to whatever the transport supports (a queue-only transport still serves commands) + // but a transport that can serve neither channel is always an error. + if (deliveries != MessageDeliveries.Both) { - await EnsureSubscriptionAsync(channels.Publish, cancellationToken).AnyContext(); - var published = await _core.StartListenerAsync(channels.Publish, handler, cancellationToken).AnyContext(); - LogSubscription(channels.Send, channels.Publish); - return new MessageSubscription(sent, published); + if (wantSent && !canSend) + throw new NotSupportedException($"Subscription requests {nameof(MessageDeliveries.Sent)} deliveries, but the transport does not support {DestinationRole.Queue} destinations."); + if (wantPublished && !canPublish) + throw new NotSupportedException($"Subscription requests {nameof(MessageDeliveries.Published)} deliveries, but the transport does not support {DestinationRole.Topic} and {DestinationRole.Subscription} destinations."); } - catch + else if (!canSend && !canPublish) { - await sent.DisposeAsync().AnyContext(); - throw; + throw new NotSupportedException("The transport supports neither queue nor topic/subscription destinations; no delivery channel can be wired."); } - } - public async Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(handler); - var channels = BuildChannels(options, typeof(object)); - var sent = await _core.StartListenerAsync(channels.Send, handler, cancellationToken).AnyContext(); + bool wireSent = wantSent && canSend; + bool wirePublished = wantPublished && canPublish; + + if (wantSent && !wireSent) + _logger.LogDebug("Skipping the sent-message channel for {MessageType}: the transport does not support {Role} destinations", fallbackType.Name, DestinationRole.Queue); + if (wantPublished && !wirePublished) + _logger.LogDebug("Skipping the published-message channel for {MessageType}: the transport does not support {TopicRole}/{SubscriptionRole} destinations", fallbackType.Name, DestinationRole.Topic, DestinationRole.Subscription); + + var channels = BuildChannels(options, fallbackType); + MessageListenerHandle? sent = wireSent ? await start(channels.Send, cancellationToken).AnyContext() : null; try { - await EnsureSubscriptionAsync(channels.Publish, cancellationToken).AnyContext(); - var published = await _core.StartListenerAsync(channels.Publish, handler, cancellationToken).AnyContext(); - LogSubscription(channels.Send, channels.Publish); + MessageListenerHandle? published = null; + if (wirePublished) + { + await EnsureSubscriptionAsync(channels.Publish, cancellationToken).AnyContext(); + published = await start(channels.Publish, cancellationToken).AnyContext(); + } + + LogSubscription(channels.Send, channels.Publish, wireSent, wirePublished); return new MessageSubscription(sent, published); } catch { - await sent.DisposeAsync().AnyContext(); + if (sent is not null) + await sent.DisposeAsync().AnyContext(); throw; } } @@ -360,11 +417,11 @@ private static string QualifySubscription(string identity, string? qualifier) // Delivery semantics must never be invisible: log each subscription's effective topology (which destination it // consumes, which subscriber group it joins, and its retry posture) once at subscribe time. - private void LogSubscription(ListenerConfig send, ListenerConfig publish) + private void LogSubscription(ListenerConfig send, ListenerConfig publish, bool sentWired, bool publishedWired) { _logger.LogInformation( "Subscribed {MessageType}: send={Destination}, publish={Subscription}, concurrency={MaxConcurrency}, attempts={MaxAttempts}, ack={AckMode}", - send.MessageType.Name, send.Source.Key, publish.Source.Key, Math.Max(1, send.MaxConcurrency), send.MaxAttempts?.ToString() ?? "default", send.AckMode); + send.MessageType.Name, sentWired ? send.Source.Key : "(none)", publishedWired ? publish.Source.Key : "(none)", Math.Max(1, send.MaxConcurrency), send.MaxAttempts?.ToString() ?? "default", send.AckMode); } private Task EnsureTopicAsync(DestinationAddress topic, CancellationToken cancellationToken) @@ -438,25 +495,27 @@ private static MessageEnvelopeOptions ToEnvelope(MessagePublishOptions options) private sealed class MessageSubscription : IMessageSubscription { - private readonly MessageListenerHandle _sent; - private readonly MessageListenerHandle _published; + private readonly MessageListenerHandle? _sent; + private readonly MessageListenerHandle? _published; - public MessageSubscription(MessageListenerHandle sent, MessageListenerHandle published) + public MessageSubscription(MessageListenerHandle? sent, MessageListenerHandle? published) { _sent = sent; _published = published; } - public string Key => _sent.Key; - public string Destination => _sent.Source.Key; - public string Topic => _published.Topic; - public string Subscription => _published.Subscription; - public string Source => _published.Source.Key; + public string Key => (_sent ?? _published)!.Key; + public string Destination => _sent?.Source.Key ?? ""; + public string Topic => _published?.Topic ?? ""; + public string Subscription => _published?.Subscription ?? ""; + public string Source => _published?.Source.Key ?? ""; public async ValueTask DisposeAsync() { - await _sent.DisposeAsync().AnyContext(); - await _published.DisposeAsync().AnyContext(); + if (_sent is not null) + await _sent.DisposeAsync().AnyContext(); + if (_published is not null) + await _published.DisposeAsync().AnyContext(); } } } diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index dbd35b69b..598566ca9 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -98,6 +98,13 @@ public MessageClientCore(IMessageTransport transport, ISerializer serializer, IM public IMessageRouter Router => _router; + // A transport that does not advertise ITransportInfo is assumed to support every role (test doubles, minimal + // providers); one that does advertise is held to its declaration. + public bool SupportsRole(DestinationRole role) + { + return _transport is not ITransportInfo info || info.SupportedRoles.Contains(role); + } + public Task EnsureAsync(IReadOnlyList declarations, CancellationToken cancellationToken) { return _transport is ISupportsProvisioning provisioning diff --git a/tests/Foundatio.Tests/Messaging/DeliveryIntentTests.cs b/tests/Foundatio.Tests/Messaging/DeliveryIntentTests.cs new file mode 100644 index 000000000..16412f92d --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/DeliveryIntentTests.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Tests.Extensions; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class DeliveryIntentTests +{ + [Fact] + public async Task SubscribeAsync_SentOnly_IgnoresPublishedMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var received = new ConcurrentQueue(); + var sentSignal = new AsyncCountdownEvent(1); + await using var subscription = await bus.SubscribeAsync((message, _) => + { + received.Enqueue(message.Message.Data); + sentSignal.Signal(); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { Deliveries = MessageDeliveries.Sent }, cts.Token); + + Assert.Equal("", subscription.Source); // no publish channel was wired + Assert.NotEqual("", subscription.Destination); + + // A published event must not reach a sent-only handler (its group does not exist), and the command must. + await bus.PublishAsync(new IntentEvent { Data = "event" }, cancellationToken: cancellationToken); + await bus.SendAsync(new IntentEvent { Data = "command" }, cancellationToken: cancellationToken); + + await sentSignal.WaitAsync(TimeSpan.FromSeconds(2)); + await Task.Delay(TimeSpan.FromMilliseconds(150), cancellationToken); // let any (incorrect) event delivery arrive + + Assert.Equal(["command"], received); + } + + [Fact] + public async Task SubscribeAsync_PublishedOnly_IgnoresSentMessagesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var received = new ConcurrentQueue(); + var publishedSignal = new AsyncCountdownEvent(1); + await using var subscription = await bus.SubscribeAsync((message, _) => + { + received.Enqueue(message.Message.Data); + publishedSignal.Signal(); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { Deliveries = MessageDeliveries.Published }, cts.Token); + + Assert.Equal("", subscription.Destination); // no send channel was wired + Assert.NotEqual("", subscription.Source); + + // The command sits unconsumed on its queue (this handler never attached to it); the event must arrive. + await bus.SendAsync(new IntentEvent { Data = "command" }, cancellationToken: cancellationToken); + await bus.PublishAsync(new IntentEvent { Data = "event" }, cancellationToken: cancellationToken); + + await publishedSignal.WaitAsync(TimeSpan.FromSeconds(2)); + await Task.Delay(TimeSpan.FromMilliseconds(150), cancellationToken); // let any (incorrect) command delivery arrive + + Assert.Equal(["event"], received); + + var queueStats = await transport.GetStatsAsync(DestinationAddress.ForQueue("intent-event"), cancellationToken); + Assert.Equal(1, queueStats.Queued); // the command is still parked on the routed queue, untouched + } + + [Fact] + public async Task SubscribeAsync_ExplicitPublished_OnQueueOnlyTransport_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var bus = new MessageBus(new QueueOnlyTransport()); + + await Assert.ThrowsAsync(() => bus.SubscribeAsync( + (_, _) => Task.CompletedTask, + new MessageSubscriptionOptions { Deliveries = MessageDeliveries.Published }, + cancellationToken)); + } + + [Fact] + public async Task SubscribeAsync_DefaultBoth_OnQueueOnlyTransport_WiresSendChannelOnlyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new QueueOnlyTransport(); + await using var bus = new MessageBus(transport); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var received = new AsyncCountdownEvent(1); + await using var subscription = await bus.SubscribeAsync((message, _) => + { + Assert.Equal("command", message.Message.Data); + received.Signal(); + return Task.CompletedTask; + }, cancellationToken: cts.Token); + + Assert.NotEqual("", subscription.Destination); + Assert.Equal("", subscription.Source); // the publish channel was skipped, not faked + + await bus.SendAsync(new IntentEvent { Data = "command" }, cancellationToken: cancellationToken); + await received.WaitAsync(TimeSpan.FromSeconds(2)); + } + + private sealed class IntentEvent + { + public string? Data { get; set; } + } + + // A transport that truly has no topic/subscription support, so the bus must not wire (or fake) a publish channel. + private sealed class QueueOnlyTransport : IMessageTransport, ISupportsPull, ITransportInfo + { + private readonly ConcurrentDictionary> _queues = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal); + + public DeliveryGuarantee DeliveryGuarantee => DeliveryGuarantee.AtLeastOnce; + public IReadOnlySet SupportedRoles => new HashSet { DestinationRole.Queue }; + public TransportCapabilities GetCapabilities(DestinationRole role) => TransportCapabilities.None; + + public Task SendAsync(DestinationAddress destination, IReadOnlyList messages, TransportSendOptions options, CancellationToken ct = default) + { + if (destination.Role != DestinationRole.Queue) + throw new NotSupportedException("Queues only."); + + var queue = _queues.GetOrAdd(destination.Key, static _ => new ConcurrentQueue()); + var items = new List(messages.Count); + foreach (var message in messages) + { + string id = message.MessageId ?? Guid.NewGuid().ToString("N"); + queue.Enqueue(message with { MessageId = id }); + items.Add(new SendItemResult { MessageId = id }); + } + + return Task.FromResult(new SendResult { Items = items }); + } + + public async Task> ReceiveAsync(DestinationAddress source, ReceiveRequest request, CancellationToken ct) + { + var queue = _queues.GetOrAdd(source.Key, static _ => new ConcurrentQueue()); + var deadline = request.MaxWaitTime is { } wait && wait > TimeSpan.Zero ? DateTimeOffset.UtcNow.Add(wait) : DateTimeOffset.UtcNow; + var entries = new List(); + int max = Math.Max(1, request.MaxMessages); + + while (true) + { + while (entries.Count < max && queue.TryDequeue(out var message)) + { + string token = Guid.NewGuid().ToString("N"); + _inFlight[token] = message; + entries.Add(new TransportEntry + { + Id = message.MessageId!, + Destination = source, + Body = message.Body, + Headers = message.Headers, + Receipt = new Receipt { TransportState = token } + }); + } + + if (entries.Count > 0 || DateTimeOffset.UtcNow >= deadline) + return entries; + + await Task.Delay(TimeSpan.FromMilliseconds(15), ct).ConfigureAwait(false); + } + } + + public Task CompleteAsync(TransportEntry entry, CancellationToken ct = default) + { + if (entry.Receipt.TransportState is not string token || !_inFlight.TryRemove(token, out _)) + throw new ReceiptExpiredException(); + + return Task.CompletedTask; + } + + public Task AbandonAsync(TransportEntry entry, CancellationToken ct = default) + { + if (entry.Receipt.TransportState is not string token || !_inFlight.TryRemove(token, out var message)) + throw new ReceiptExpiredException(); + + _queues.GetOrAdd(entry.Destination.Key, static _ => new ConcurrentQueue()).Enqueue(message); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} From c982247ce45b0a60675aff5e0ca5dc32f557b7ce Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:36:57 -0500 Subject: [PATCH 51/57] Explicit topology modes: Ensure, Validate, or None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing and subscribing implicitly granted themselves topology administration — every publish ensured its topic, every subscribe created its subscription, IMessageTopology.ValidateAsync was registered but consumed nowhere, and an app on a locked-down broker had no supported way to run. MessageBusOptions.Topology (and Messaging.ConfigureTopology(...) in DI) now selects the policy: Ensure keeps today's create-on-use behavior and also ensures the declared topology at handler-host startup; Validate never creates — each destination is existence-checked once (cached) and a missing one fails loudly, with the declared topology validated at startup so a misprovisioned app stops at boot instead of erroring per send; None makes no topology calls at all. AWS's AutoCreateDestinations now also guards SNS topic creation (lookup + loud failure instead of CreateTopic when disabled); the explicit EnsureAsync provisioning path always creates, since that call is the administrative intent the option withholds from the data paths. (Review feedback #4.) Co-Authored-By: Claude Fable 5 --- src/Foundatio.Aws/AwsMessageTransport.cs | 42 ++++++--- src/Foundatio/FoundatioServicesExtensions.cs | 15 +++ src/Foundatio/Messaging/MessageBus.cs | 27 +++++- src/Foundatio/Messaging/MessageClientCore.cs | 36 ++++++- .../Messaging/MessageHandlerHostedService.cs | 36 +++++++ .../Messaging/TopologyModeTests.cs | 93 +++++++++++++++++++ 6 files changed, 233 insertions(+), 16 deletions(-) create mode 100644 tests/Foundatio.Tests/Messaging/TopologyModeTests.cs diff --git a/src/Foundatio.Aws/AwsMessageTransport.cs b/src/Foundatio.Aws/AwsMessageTransport.cs index c9e6279dc..039fe599a 100644 --- a/src/Foundatio.Aws/AwsMessageTransport.cs +++ b/src/Foundatio.Aws/AwsMessageTransport.cs @@ -221,14 +221,14 @@ public async Task EnsureAsync(IReadOnlyList declarations switch (declaration.Address.Role) { case DestinationRole.Topic: - await ResolveTopicArnAsync(declaration.Address.Name, ct).ConfigureAwait(false); + await ResolveTopicArnAsync(declaration.Address.Name, allowCreate: true, ct).ConfigureAwait(false); break; case DestinationRole.Subscription: case DestinationRole.Binding: await EnsureSubscriptionAsync(declaration.Address, ct).ConfigureAwait(false); break; default: - await ResolveQueueUrlAsync(declaration.Address, ct).ConfigureAwait(false); + await ResolveQueueUrlAsync(declaration.Address, allowCreate: true, ct).ConfigureAwait(false); break; } } @@ -302,12 +302,12 @@ public async ValueTask DisposeAsync() private async Task EnsureSubscriptionAsync(DestinationAddress address, CancellationToken ct) { - string queueUrl = await ResolveQueueUrlAsync(address, ct).ConfigureAwait(false); + string queueUrl = await ResolveQueueUrlAsync(address, allowCreate: true, ct).ConfigureAwait(false); if (String.IsNullOrEmpty(address.Topic)) return; - string topicArn = await ResolveTopicArnAsync(address.Topic, ct).ConfigureAwait(false); + string topicArn = await ResolveTopicArnAsync(address.Topic, allowCreate: true, ct).ConfigureAwait(false); string queueArn = await GetQueueArnAsync(queueUrl, ct).ConfigureAwait(false); // Allow the topic to deliver to the queue, then subscribe with raw delivery so the SQS body/attributes match a @@ -331,7 +331,10 @@ await _sns.Value.SubscribeAsync(new SubscribeRequest // Queue and subscription destinations are both backed by an SQS queue whose logical name is the address key // (Name for queues, "topic/subscription" for subscriptions), so provisioning and every runtime path resolve the // same physical queue from the same address. - private async Task ResolveQueueUrlAsync(DestinationAddress address, CancellationToken ct) + private Task ResolveQueueUrlAsync(DestinationAddress address, CancellationToken ct) => + ResolveQueueUrlAsync(address, allowCreate: _options.AutoCreateDestinations, ct); + + private async Task ResolveQueueUrlAsync(DestinationAddress address, bool allowCreate, CancellationToken ct) { string key = address.Key; if (_queueUrls.TryGetValue(key, out string? cached)) @@ -344,7 +347,7 @@ private async Task ResolveQueueUrlAsync(DestinationAddress address, Canc _queueUrls[key] = response.QueueUrl; return response.QueueUrl; } - catch (QueueDoesNotExistException) when (_options.AutoCreateDestinations) + catch (QueueDoesNotExistException) when (allowCreate) { var response = await _sqs.Value.CreateQueueAsync(new CreateQueueRequest { QueueName = resourceName }, ct).ConfigureAwait(false); _queueUrls[key] = response.QueueUrl; @@ -352,15 +355,32 @@ private async Task ResolveQueueUrlAsync(DestinationAddress address, Canc } } - private async Task ResolveTopicArnAsync(string name, CancellationToken ct) + // Implicit resolution (send/receive paths) honors AutoCreateDestinations; explicit provisioning via EnsureAsync + // always creates — that call IS the administrative intent the option exists to withhold from the data paths. + private Task ResolveTopicArnAsync(string name, CancellationToken ct) => + ResolveTopicArnAsync(name, allowCreate: _options.AutoCreateDestinations, ct); + + private async Task ResolveTopicArnAsync(string name, bool allowCreate, CancellationToken ct) { if (_topicArns.TryGetValue(name, out string? cached)) return cached; - // CreateTopic is idempotent and returns the ARN of an existing topic with the same name. - var response = await _sns.Value.CreateTopicAsync(new CreateTopicRequest { Name = ResourceName(name) }, ct).ConfigureAwait(false); - _topicArns[name] = response.TopicArn; - return response.TopicArn; + if (allowCreate) + { + // CreateTopic is idempotent and returns the ARN of an existing topic with the same name. + var response = await _sns.Value.CreateTopicAsync(new CreateTopicRequest { Name = ResourceName(name) }, ct).ConfigureAwait(false); + _topicArns[name] = response.TopicArn; + return response.TopicArn; + } + + // Auto-create is disabled (locked-down broker): look the topic up instead of creating it, and fail loudly when + // it has not been provisioned out of band. + var existing = await _sns.Value.FindTopicAsync(ResourceName(name)).ConfigureAwait(false); + if (existing is null) + throw new InvalidOperationException($"SNS topic \"{ResourceName(name)}\" does not exist and {nameof(AwsMessageTransportOptions.AutoCreateDestinations)} is disabled. Provision it out of band or enable auto-creation."); + + _topicArns[name] = existing.TopicArn; + return existing.TopicArn; } // SQS queue / SNS topic names allow only [A-Za-z0-9_-] (max 80 chars). Most logical names already conform, but a diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index d5362d6c5..4ff786317 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -252,6 +252,7 @@ public class MessagingBuilder : IFoundatioBuilder private readonly IServiceCollection _services; private bool _routingServicesRegistered; private bool _topologyServicesRegistered; + private TopologyMode _topologyMode = TopologyMode.Ensure; internal MessagingBuilder(IFoundatioBuilder builder) { @@ -259,6 +260,17 @@ internal MessagingBuilder(IFoundatioBuilder builder) _services = builder.Services; } + /// + /// Selects how the messaging client administers topology: creates missing + /// destinations on use and at handler-host startup (default), only checks + /// they exist and throws when missing, and never touches topology. + /// + public MessagingBuilder ConfigureTopology(TopologyMode mode) + { + _topologyMode = mode; + return this; + } + IServiceCollection IFoundatioBuilder.Services => _services; FoundatioBuilder IFoundatioBuilder.Builder => _builder; @@ -409,6 +421,8 @@ private static async Task DispatchAsync(IServiceProvider ser private void RegisterMessagingRuntime(Func factory) { _services.ReplaceSingleton(factory); + // Resolved lazily so ConfigureTopology can be called before or after the Use* transport registration. + _services.ReplaceSingleton(_ => new MessagingTopologyOptions(_topologyMode)); RegisterMessageTopology(); RegisterMessageClients(); } @@ -455,6 +469,7 @@ private void RegisterMessageClients() MessageTypes = sp.GetService() ?? new MessageTypeRegistry(), RuntimeStore = sp.GetService(), RetryPolicy = sp.GetService() ?? new RetryPolicy(), + Topology = sp.GetService()?.Mode ?? TopologyMode.Ensure, // The transport is a shared DI singleton owned by the container; the bus must not dispose it. OwnsTransport = false, TimeProvider = sp.GetService() ?? TimeProvider.System, diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index e93412403..bd64c502e 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -203,8 +203,33 @@ public interface IMessageBus : IAsyncDisposable Task SubscribeAsync(Func handler, MessageSubscriptionOptions? options = null, CancellationToken cancellationToken = default); } +/// +/// Governs the messaging client's topology-administration behavior. Publishing and subscribing never implicitly grant +/// themselves more than this mode allows, so an app on a locked-down broker can state "validate only" or "never touch +/// topology" instead of hoping implicit creation fails gracefully. +/// +/// +/// The mode governs the CORE's provisioning calls. A transport may still lazily create cheap local structures on its +/// own paths (e.g. consumer groups on first receive); combine Validate/None with the transport's own knobs (such as +/// AwsMessageTransportOptions.AutoCreateDestinations = false) for a fully locked-down broker. +/// +public enum TopologyMode +{ + /// Create missing destinations on first use and at handler-host startup (default). + Ensure, + + /// Never create. Verify each destination exists on first use (cached) and throw when missing. + Validate, + + /// No topology calls at all; destinations are assumed pre-provisioned out of band. + None +} + public sealed record MessageBusOptions { + /// How the client administers topology (create on use, validate-only, or never touch). Default . + public TopologyMode Topology { get; init; } = TopologyMode.Ensure; + public ISerializer Serializer { get; init; } = DefaultSerializer.Instance; public string ContentType { get; init; } = "application/json"; public IMessageRouter Router { get; init; } = DefaultMessageRouter.Instance; @@ -242,7 +267,7 @@ public MessageBus(IMessageTransport transport, MessageBusOptions? options = null options ??= new MessageBusOptions(); _logger = (options.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); _core = new MessageClientCore(transport, options.Serializer, options.Router, options.RuntimeStore, options.TimeProvider, _logger, - static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType); + static (message, inner) => inner is null ? new MessageBusException(message) : new MessageBusException(message, inner), options.RetryPolicy, options.OwnsTransport, options.MessageTypes, options.ContentType, options.Topology); } public Task SendAsync(T message, MessageSendOptions? options = null, CancellationToken cancellationToken = default) where T : class diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 598566ca9..39eaeb993 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -78,11 +78,14 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly string? _contentType; private readonly bool _ownsTransport; private readonly ConcurrentDictionary _sources = new(); + private readonly TopologyMode _topologyMode; + private readonly ConcurrentDictionary _validatedDestinations = new(); private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, - IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null, string? contentType = null) + IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null, string? contentType = null, TopologyMode topologyMode = TopologyMode.Ensure) { + _topologyMode = topologyMode; _transport = transport ?? throw new ArgumentNullException(nameof(transport)); _serializer = serializer; _router = router; @@ -107,9 +110,34 @@ public bool SupportsRole(DestinationRole role) public Task EnsureAsync(IReadOnlyList declarations, CancellationToken cancellationToken) { - return _transport is ISupportsProvisioning provisioning - ? provisioning.EnsureAsync(declarations, cancellationToken) - : Task.CompletedTask; + return _topologyMode switch + { + TopologyMode.None => Task.CompletedTask, + TopologyMode.Validate => ValidateDeclarationsAsync(declarations, cancellationToken), + _ => _transport is ISupportsProvisioning provisioning + ? provisioning.EnsureAsync(declarations, cancellationToken) + : Task.CompletedTask + }; + } + + // Validate never creates: each destination is checked once (successes are cached so steady-state publishes pay no + // exists round-trip) and a missing one fails loudly instead of being silently created on a broker the app is not + // supposed to administer. + private async Task ValidateDeclarationsAsync(IReadOnlyList declarations, CancellationToken cancellationToken) + { + if (_transport is not ISupportsProvisioning provisioning) + throw new NotSupportedException($"{nameof(TopologyMode)}.{nameof(TopologyMode.Validate)} requires a transport that can check destination existence; \"{_transport.GetType().Name}\" does not support provisioning. Use {nameof(TopologyMode)}.{nameof(TopologyMode.None)} when the transport cannot inspect a pre-provisioned broker."); + + foreach (var declaration in declarations) + { + if (_validatedDestinations.ContainsKey(declaration.Address)) + continue; + + if (!await provisioning.ExistsAsync(declaration.Address, cancellationToken).AnyContext()) + throw _exceptionFactory($"Message topology destination {declaration.Address} does not exist and {nameof(TopologyMode)}.{nameof(TopologyMode.Validate)} never creates topology. Provision it out of band or use {nameof(TopologyMode)}.{nameof(TopologyMode.Ensure)}.", null); + + _validatedDestinations.TryAdd(declaration.Address, 0); + } } public async Task SendAsync(ScheduledDispatchKind kind, Type messageType, object message, MessageEnvelopeOptions options, DestinationAddress destination, Func? ensureDestination, CancellationToken cancellationToken) diff --git a/src/Foundatio/Messaging/MessageHandlerHostedService.cs b/src/Foundatio/Messaging/MessageHandlerHostedService.cs index b6cb795f1..627061b90 100644 --- a/src/Foundatio/Messaging/MessageHandlerHostedService.cs +++ b/src/Foundatio/Messaging/MessageHandlerHostedService.cs @@ -20,6 +20,9 @@ internal sealed class MessageHandlerRegistration public required Func> StartAsync { get; init; } } +/// The DI-selected , applied by the handler host at startup and by the message clients on use. +internal sealed record MessagingTopologyOptions(TopologyMode Mode); + /// /// Hosts every declaratively-registered message handler for the app's lifetime: on start it launches each handler's /// consumer/subscription; on stop it disposes them. Auto-registered when the first handler is added, so users register @@ -42,6 +45,8 @@ public MessageHandlerHostedService(IServiceProvider serviceProvider, IEnumerable public async Task StartAsync(CancellationToken cancellationToken) { + await ApplyTopologyAsync(cancellationToken).AnyContext(); + try { foreach (var registration in _registrations) @@ -60,6 +65,37 @@ public async Task StartAsync(CancellationToken cancellationToken) } } + // Apply the app's declared topology before any handler starts consuming: Ensure creates what the routing config + // declares, Validate proves it exists and fails startup when it doesn't (a missing destination should stop the app + // at boot, not surface as runtime send errors), and None trusts out-of-band provisioning entirely. + private async Task ApplyTopologyAsync(CancellationToken cancellationToken) + { + var mode = (_serviceProvider.GetService(typeof(MessagingTopologyOptions)) as MessagingTopologyOptions)?.Mode ?? TopologyMode.Ensure; + if (mode == TopologyMode.None) + return; + + if (_serviceProvider.GetService(typeof(IMessageTopology)) is not IMessageTopology topology) + return; + + if (mode == TopologyMode.Validate) + { + await topology.ValidateAsync(cancellationToken).AnyContext(); + _logger.LogInformation("Validated declared message topology"); + return; + } + + try + { + await topology.EnsureAsync(cancellationToken).AnyContext(); + _logger.LogInformation("Ensured declared message topology"); + } + catch (NotSupportedException) + { + // The transport cannot provision; the runtime use-time paths no-op the same way, so startup should not fail. + _logger.LogDebug("Transport does not support topology provisioning; skipping startup ensure"); + } + } + public Task StopAsync(CancellationToken cancellationToken) => DisposeStartedAsync(); private async Task DisposeStartedAsync() diff --git a/tests/Foundatio.Tests/Messaging/TopologyModeTests.cs b/tests/Foundatio.Tests/Messaging/TopologyModeTests.cs new file mode 100644 index 000000000..d9d2752d0 --- /dev/null +++ b/tests/Foundatio.Tests/Messaging/TopologyModeTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Tests.Extensions; +using Xunit; + +namespace Foundatio.Tests.Messaging; + +public class TopologyModeTests +{ + [Fact] + public async Task Validate_WithPreProvisionedTopology_DeliversAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + // Provision out of band (the admin path), then run the bus in validate-only mode. + var topic = DestinationAddress.ForTopic("topology-event"); + var subscription = DestinationAddress.ForSubscription("topology-event", "svc"); + await transport.EnsureAsync([new DestinationDeclaration { Address = topic }, new DestinationDeclaration { Address = subscription }], cancellationToken); + + await using var bus = new MessageBus(transport, new MessageBusOptions { Topology = TopologyMode.Validate, OwnsTransport = false }); + var received = new AsyncCountdownEvent(1); + await using var handle = await bus.SubscribeAsync((message, _) => + { + Assert.Equal("hello", message.Message.Data); + received.Signal(); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { Subscription = "svc" }, cts.Token); + + await bus.PublishAsync(new TopologyEvent { Data = "hello" }, cancellationToken: cancellationToken); + await received.WaitAsync(TimeSpan.FromSeconds(2)); + } + + [Fact] + public async Task Validate_WithMissingTopology_ThrowsAndCreatesNothingAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport, new MessageBusOptions { Topology = TopologyMode.Validate, OwnsTransport = false }); + + await Assert.ThrowsAsync(() => bus.PublishAsync(new TopologyEvent { Data = "hello" }, cancellationToken: cancellationToken)); + await Assert.ThrowsAsync(() => bus.SubscribeAsync((_, _) => Task.CompletedTask, cancellationToken: cancellationToken)); + + Assert.False(await transport.ExistsAsync(DestinationAddress.ForTopic("topology-event"), cancellationToken)); + } + + [Fact] + public async Task None_NeverTouchesTopologyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + await using var bus = new MessageBus(transport, new MessageBusOptions { Topology = TopologyMode.None, OwnsTransport = false }); + + // Publishing to a topic that was never provisioned must not create it (real pub/sub drop semantics). + await bus.PublishAsync(new TopologyEvent { Data = "dropped" }, cancellationToken: cancellationToken); + Assert.False(await transport.ExistsAsync(DestinationAddress.ForTopic("topology-event"), cancellationToken)); + } + + [Fact] + public async Task None_WithPreProvisionedTopology_DeliversAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var transport = new InMemoryMessageTransport(); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(10)); + + var topic = DestinationAddress.ForTopic("topology-event"); + var subscription = DestinationAddress.ForSubscription("topology-event", "svc"); + await transport.EnsureAsync([new DestinationDeclaration { Address = topic }, new DestinationDeclaration { Address = subscription }], cancellationToken); + + await using var bus = new MessageBus(transport, new MessageBusOptions { Topology = TopologyMode.None, OwnsTransport = false }); + var received = new AsyncCountdownEvent(1); + await using var handle = await bus.SubscribeAsync((message, _) => + { + received.Signal(); + return Task.CompletedTask; + }, new MessageSubscriptionOptions { Subscription = "svc" }, cts.Token); + + await bus.PublishAsync(new TopologyEvent { Data = "hello" }, cancellationToken: cancellationToken); + await received.WaitAsync(TimeSpan.FromSeconds(2)); + } + + [MessageRoute("topology-event")] + private sealed class TopologyEvent + { + public string? Data { get; set; } + } +} From 60a52ba4ce2afb0bba1412b3ccd283ace0876e40 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:39:15 -0500 Subject: [PATCH 52/57] Extract IScheduledDispatchStore from IJobRuntimeStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runtime store bundled five concerns; the one messaging actually depends on — durable storage for delayed sends, store-parked retries, and occurrence triggers — is now its own four-member IScheduledDispatchStore contract, and MessageBusOptions.RuntimeStore takes exactly that, so a provider can offer durable message scheduling without implementing the full job runtime. IJobRuntimeStore composes it, so existing providers are unchanged. Job state, leases, and cancellation stay one contract on purpose: transitions verify ownership atomically and splitting them would break the compare-and-set semantics correctness depends on. (Review feedback #8, the modest version.) Co-Authored-By: Claude Fable 5 --- src/Foundatio/FoundatioServicesExtensions.cs | 2 +- src/Foundatio/Jobs/JobRuntime.cs | 25 ++++++++++++++++---- src/Foundatio/Messaging/MessageBus.cs | 9 +++---- src/Foundatio/Messaging/MessageClientCore.cs | 10 ++++---- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 4ff786317..b0034f0cd 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -467,7 +467,7 @@ private void RegisterMessageClients() Serializer = sp.GetService() ?? DefaultSerializer.Instance, Router = sp.GetService() ?? DefaultMessageRouter.Instance, MessageTypes = sp.GetService() ?? new MessageTypeRegistry(), - RuntimeStore = sp.GetService(), + RuntimeStore = sp.GetService() ?? sp.GetService(), RetryPolicy = sp.GetService() ?? new RetryPolicy(), Topology = sp.GetService()?.Mode ?? TopologyMode.Ensure, // The transport is a shared DI singleton owned by the container; the bus must not dispose it. diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index bf067fe9a..a9754ba44 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -292,7 +292,26 @@ public interface IJobWorker Task RecoverStaleAsync(int maxAttempts, int limit = 100, CancellationToken cancellationToken = default); } -public interface IJobRuntimeStore : IJobMonitor +/// +/// Durable storage for time-gated dispatches: delayed messages beyond a transport's native ceiling, store-parked +/// retry delays, and CRON occurrence triggers. This is the only store contract the messaging client depends on — +/// a provider that offers durable scheduling without the full job runtime implements just this. +/// +public interface IScheduledDispatchStore +{ + Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default); + Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); + Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken cancellationToken = default); + Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken cancellationToken = default); +} + +/// +/// The full job runtime store: job state persistence, queries, lease/ownership management, cancellation signaling, +/// and scheduled-dispatch storage. The state/lease/cancellation members are deliberately one contract — transitions +/// verify ownership atomically (see / ), so +/// splitting them would break the compare-and-set semantics correctness depends on. +/// +public interface IJobRuntimeStore : IJobMonitor, IScheduledDispatchStore { Task CreateIfAbsentAsync(JobState initial, CancellationToken cancellationToken = default); // When expectedNodeId is non-null, the transition only succeeds if the job is currently owned by that node. @@ -315,10 +334,6 @@ public interface IJobRuntimeStore : IJobMonitor Task IncrementAttemptAsync(string jobId, CancellationToken cancellationToken = default); Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); Task IsCancellationRequestedAsync(string jobId, CancellationToken cancellationToken = default); - Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken cancellationToken = default); - Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default); - Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken cancellationToken = default); - Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken cancellationToken = default); } public sealed class InMemoryJobRuntimeStore : IJobRuntimeStore diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index bd64c502e..54eba65a1 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -236,11 +236,12 @@ public sealed record MessageBusOptions public IMessageTypeRegistry MessageTypes { get; init; } = new MessageTypeRegistry(); /// /// Enables durable scheduling: delayed sends beyond a transport ceiling and store-parked retry delays are written - /// here and drained by the job runtime pump. The DI builder registers the pump automatically with the store; when - /// wiring options by hand, ensure a pump (JobRuntimePumpService / JobScheduleProcessor) is running or parked - /// messages will never be dispatched. + /// here and drained by the job runtime pump. Messaging depends only on the dispatch-storage contract — any + /// satisfies it, but a provider can implement + /// alone. The DI builder registers the pump automatically with the store; when wiring options by hand, ensure a + /// pump (JobRuntimePumpService / JobScheduleProcessor) is running or parked messages will never be dispatched. /// - public IJobRuntimeStore? RuntimeStore { get; init; } + public IScheduledDispatchStore? RuntimeStore { get; init; } public RetryPolicy RetryPolicy { get; init; } = new(); /// diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index 39eaeb993..ddef6591c 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -69,7 +69,7 @@ internal sealed class MessageClientCore : IAsyncDisposable private readonly IMessageTransport _transport; private readonly ISerializer _serializer; private readonly IMessageRouter _router; - private readonly IJobRuntimeStore? _runtimeStore; + private readonly IScheduledDispatchStore? _runtimeStore; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; private readonly Func _exceptionFactory; @@ -83,7 +83,7 @@ internal sealed class MessageClientCore : IAsyncDisposable private int _isDisposed; public MessageClientCore(IMessageTransport transport, ISerializer serializer, IMessageRouter router, - IJobRuntimeStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null, string? contentType = null, TopologyMode topologyMode = TopologyMode.Ensure) + IScheduledDispatchStore? runtimeStore, TimeProvider timeProvider, ILogger logger, Func exceptionFactory, RetryPolicy? retryPolicy = null, bool ownsTransport = true, IMessageTypeRegistry? typeRegistry = null, string? contentType = null, TopologyMode topologyMode = TopologyMode.Ensure) { _topologyMode = topologyMode; _transport = transport ?? throw new ArgumentNullException(nameof(transport)); @@ -981,13 +981,13 @@ internal class MessageContext : IMessageContext { private readonly IMessageTransport _transport; private readonly TransportEntry _entry; - private readonly IJobRuntimeStore? _runtimeStore; + private readonly IScheduledDispatchStore? _runtimeStore; private readonly TimeProvider _timeProvider; private readonly string? _deadLetterDestination; private readonly ILogger _logger; private int _isHandled; - public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) + public MessageContext(IMessageTransport transport, TransportEntry entry, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) { _transport = transport; _entry = entry; @@ -1181,7 +1181,7 @@ private static int ParseAttemptsHeader(MessageHeaders headers) internal sealed class MessageContext : MessageContext, IMessageContext where T : class { - public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IJobRuntimeStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) + public MessageContext(IMessageTransport transport, TransportEntry entry, T message, CancellationToken cancellationToken, IScheduledDispatchStore? runtimeStore = null, TimeProvider? timeProvider = null, string? deadLetterDestination = null, ILogger? logger = null) : base(transport, entry, cancellationToken, runtimeStore, timeProvider, deadLetterDestination, logger) { Message = message; From 1e77cb35cf0f45f9e1ad164fdb7afa0b3262fd96 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:48:42 -0500 Subject: [PATCH 53/57] Typed durable-job payloads: EnqueueAsync(args) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A durable job had no way to receive per-invocation data — JobState had no payload field, so every run of a type was identical and real work had to be smuggled through DI singletons. IJobClient.EnqueueAsync(args) serializes the arguments through the runtime's ISerializer into JobState.Payload with the argument type's full name as the stored discriminator; the job reads them back with JobExecutionContext.GetArguments() (HasArguments to probe), which throws a descriptive error naming the stored discriminator when the payload is absent or unreadable. CRON schedules carry arguments too (ScheduledJobDefinition/CronJobOptions.Arguments, serialized into each occurrence), the detached test context accepts an arguments object directly, the Redis store round-trips the new fields, and the store conformance suite asserts the payload survives persistence. (Review feedback #5.) Co-Authored-By: Claude Fable 5 --- src/Foundatio.Redis/RedisJobRuntimeStore.cs | 4 + .../Jobs/JobRuntimeStoreConformanceTests.cs | 5 + src/Foundatio/FoundatioServicesExtensions.cs | 10 +- src/Foundatio/Jobs/JobRuntime.cs | 96 +++++++++++++++++-- src/Foundatio/Jobs/JobScheduler.cs | 18 +++- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 73 +++++++++++++- 6 files changed, 190 insertions(+), 16 deletions(-) diff --git a/src/Foundatio.Redis/RedisJobRuntimeStore.cs b/src/Foundatio.Redis/RedisJobRuntimeStore.cs index 3bddc6c9e..ea88cb1a3 100644 --- a/src/Foundatio.Redis/RedisJobRuntimeStore.cs +++ b/src/Foundatio.Redis/RedisJobRuntimeStore.cs @@ -409,6 +409,8 @@ private static HashEntry[] ToHash(JobState state) }; if (state.JobType is not null) entries.Add(new("jobType", state.JobType)); + if (state.Payload is { } payload) entries.Add(new("payload", Convert.ToBase64String(payload.Span))); + if (state.PayloadType is not null) entries.Add(new("payloadType", state.PayloadType)); if (state.Progress is { } progress) entries.Add(new("progress", progress)); if (state.ProgressMessage is not null) entries.Add(new("progressMessage", state.ProgressMessage)); if (state.NodeId is not null) entries.Add(new("nodeId", state.NodeId)); @@ -431,6 +433,8 @@ private static JobState FromHash(HashEntry[] entries) JobId = (string)Get("jobId")!, Name = (string)Get("name")!, JobType = ToStringOrNull(Get("jobType")), + Payload = Get("payload").IsNullOrEmpty ? null : Convert.FromBase64String((string)Get("payload")!), + PayloadType = ToStringOrNull(Get("payloadType")), Status = Enum.Parse((string)Get("status")!), Progress = Get("progress").IsNullOrEmpty ? null : (int)Get("progress"), ProgressMessage = ToStringOrNull(Get("progressMessage")), diff --git a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs index 7f6243624..aac029fa7 100644 --- a/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs +++ b/src/Foundatio.TestHarness/Jobs/JobRuntimeStoreConformanceTests.cs @@ -52,6 +52,8 @@ public virtual async Task JobLifecycle_RoundTripsAndTransitionsAsync() var job = NewJob(time, "job-1", "emailer") with { JobType = "Acme.EmailJob", + Payload = new byte[] { 1, 2, 3, 4 }, + PayloadType = "Acme.EmailJobArgs", Progress = 10, ProgressMessage = "starting", Attempt = 1, @@ -63,6 +65,9 @@ public virtual async Task JobLifecycle_RoundTripsAndTransitionsAsync() Assert.NotNull(got); Assert.Equal("emailer", got.Name); Assert.Equal("Acme.EmailJob", got.JobType); + Assert.NotNull(got.Payload); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, got.Payload.Value.ToArray()); + Assert.Equal("Acme.EmailJobArgs", got.PayloadType); Assert.Equal(JobStatus.Queued, got.Status); Assert.Equal(10, got.Progress); Assert.Equal("starting", got.ProgressMessage); diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index b0034f0cd..07e5ee36b 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -546,7 +546,8 @@ public FoundatioBuilder AddCronJob(string cronSchedule, Action(sp => new JobTypeRegistry(sp.GetServices())); _services.ReplaceSingleton(sp => sp.GetRequiredService()); - _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService(), sp.GetRequiredService())); - _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService(), jobTypes: sp.GetRequiredService())); + _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService(), sp.GetRequiredService(), sp.GetService())); + _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService(), jobTypes: sp.GetRequiredService(), serializer: sp.GetService())); _services.ReplaceSingleton(); _services.ReplaceSingleton(sp => new JobScheduleProcessor( sp.GetRequiredService(), @@ -577,7 +578,8 @@ private void RegisterJobServices() sp.GetRequiredService(), sp.GetService(), transport: sp.GetService(), - jobTypes: sp.GetRequiredService())); + jobTypes: sp.GetRequiredService(), + serializer: sp.GetService())); // A runtime store is inert without something draining it, so register the pump alongside the store: in a // hosted process it runs jobs and the messaging delayed-delivery fallback automatically (no separate diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index a9754ba44..b875c9c33 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Messaging; +using Foundatio.Serializer; using Microsoft.Extensions.DependencyInjection; namespace Foundatio.Jobs; @@ -46,6 +47,12 @@ public sealed record JobState public required string JobId { get; init; } public required string Name { get; init; } public string? JobType { get; init; } + + /// Serialized per-invocation arguments (see ); null when the job takes none. + public ReadOnlyMemory? Payload { get; init; } + + /// Discriminator for the payload type (the argument type's full name), stored for forensics and mismatch diagnostics. + public string? PayloadType { get; init; } public JobStatus Status { get; init; } = JobStatus.Queued; public int? Progress { get; init; } public string? ProgressMessage { get; init; } @@ -230,8 +237,12 @@ public sealed class JobExecutionContext private readonly IJobRuntimeStore? _store; private readonly string _nodeId; private readonly TimeSpan _lease; + private readonly ReadOnlyMemory? _payload; + private readonly string? _payloadType; + private readonly ISerializer? _serializer; + private readonly object? _detachedArguments; - internal JobExecutionContext(string jobId, int attempt, CancellationToken cancellationToken, IJobRuntimeStore store, string nodeId, TimeSpan lease) + internal JobExecutionContext(string jobId, int attempt, CancellationToken cancellationToken, IJobRuntimeStore store, string nodeId, TimeSpan lease, ReadOnlyMemory? payload = null, string? payloadType = null, ISerializer? serializer = null) { JobId = jobId; Attempt = attempt; @@ -239,13 +250,17 @@ internal JobExecutionContext(string jobId, int attempt, CancellationToken cancel _store = store; _nodeId = nodeId; _lease = lease; + _payload = payload; + _payloadType = payloadType; + _serializer = serializer; } /// /// Creates a detached context for running a job outside the durable runtime (tests or one-off invocations). - /// Progress reporting and lease renewal are no-ops; cancellation reflects . + /// Progress reporting and lease renewal are no-ops; cancellation reflects ; + /// surfaces through without serialization. /// - public JobExecutionContext(CancellationToken cancellationToken = default, string? jobId = null, int attempt = 1) + public JobExecutionContext(CancellationToken cancellationToken = default, string? jobId = null, int attempt = 1, object? arguments = null) { JobId = jobId ?? Guid.NewGuid().ToString("N"); Attempt = attempt; @@ -253,12 +268,46 @@ public JobExecutionContext(CancellationToken cancellationToken = default, string _store = null; _nodeId = String.Empty; _lease = TimeSpan.Zero; + _detachedArguments = arguments; } public string JobId { get; } public int Attempt { get; } public CancellationToken CancellationToken { get; } + /// Whether this invocation carries typed arguments (see ). + public bool HasArguments => _detachedArguments is not null || _payload is not null; + + /// + /// The typed per-invocation arguments this job was enqueued with. Throws a descriptive + /// when the job was enqueued without arguments or the payload cannot be + /// read as (the stored discriminator is included for triage). + /// + public TArgs GetArguments() where TArgs : class + { + if (_detachedArguments is not null) + { + return _detachedArguments as TArgs + ?? throw new InvalidOperationException($"Job \"{JobId}\" arguments are of type \"{_detachedArguments.GetType().FullName}\", not the requested \"{typeof(TArgs).FullName}\"."); + } + + if (_payload is not { } payload) + throw new InvalidOperationException($"Job \"{JobId}\" was enqueued without arguments. Use EnqueueAsync(args) to supply a typed payload."); + + var serializer = _serializer ?? DefaultSerializer.Instance; + TArgs? args; + try + { + args = serializer.Deserialize(payload, typeof(TArgs)) as TArgs; + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to deserialize job \"{JobId}\" arguments (stored type \"{_payloadType}\") as \"{typeof(TArgs).FullName}\".", ex); + } + + return args ?? throw new InvalidOperationException($"Job \"{JobId}\" arguments (stored type \"{_payloadType}\") deserialized to null as \"{typeof(TArgs).FullName}\"."); + } + public Task ReportProgressAsync(int? percent = null, string? message = null, CancellationToken cancellationToken = default) => _store?.SetProgressAsync(JobId, percent, message, cancellationToken) ?? Task.CompletedTask; @@ -279,6 +328,14 @@ public interface IJobMonitor public interface IJobClient { Task EnqueueAsync(JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob; + + /// + /// Enqueues a job with typed per-invocation arguments. The args are serialized into the durable + /// via the runtime's serializer and surface to the job through + /// . + /// + Task EnqueueAsync(TArgs args, JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob where TArgs : class; + Task EnqueueAsync(Type jobType, JobRequestOptions? options = null, CancellationToken cancellationToken = default); Task RequestCancellationAsync(string jobId, CancellationToken cancellationToken = default); } @@ -666,20 +723,33 @@ public sealed class JobClient : IJobClient private readonly IJobRuntimeStore _store; private readonly TimeProvider _timeProvider; private readonly IJobTypeRegistry _jobTypes; + private readonly ISerializer _serializer; - public JobClient(IJobRuntimeStore store, TimeProvider? timeProvider = null, IJobTypeRegistry? jobTypes = null) + public JobClient(IJobRuntimeStore store, TimeProvider? timeProvider = null, IJobTypeRegistry? jobTypes = null, ISerializer? serializer = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); _timeProvider = timeProvider ?? TimeProvider.System; _jobTypes = jobTypes ?? new JobTypeRegistry(); + _serializer = serializer ?? DefaultSerializer.Instance; } public Task EnqueueAsync(JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob { - return EnqueueAsync(typeof(TJob), options, cancellationToken); + return EnqueueCoreAsync(typeof(TJob), args: null, options, cancellationToken); + } + + public Task EnqueueAsync(TArgs args, JobRequestOptions? options = null, CancellationToken cancellationToken = default) where TJob : IJob where TArgs : class + { + ArgumentNullException.ThrowIfNull(args); + return EnqueueCoreAsync(typeof(TJob), args, options, cancellationToken); + } + + public Task EnqueueAsync(Type jobType, JobRequestOptions? options = null, CancellationToken cancellationToken = default) + { + return EnqueueCoreAsync(jobType, args: null, options, cancellationToken); } - public async Task EnqueueAsync(Type jobType, JobRequestOptions? options = null, CancellationToken cancellationToken = default) + private async Task EnqueueCoreAsync(Type jobType, object? args, JobRequestOptions? options, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(jobType); if (!typeof(IJob).IsAssignableFrom(jobType)) @@ -695,6 +765,10 @@ await _store.CreateIfAbsentAsync(new JobState JobId = jobId, Name = name, JobType = _jobTypes.GetName(jobType), + // Explicitly typed: the byte[] -> ReadOnlyMemory conversion maps a null array to an EMPTY memory, which + // would make an argless job look like it carries a zero-byte payload. + Payload = args is null ? null : (ReadOnlyMemory?)_serializer.SerializeToBytes(args), + PayloadType = args?.GetType().FullName, Status = JobStatus.Queued, CreatedUtc = now, LastUpdatedUtc = now @@ -737,16 +811,18 @@ public sealed class JobWorker : IJobWorker private readonly IServiceProvider _serviceProvider; private readonly TimeProvider _timeProvider; private readonly IJobTypeRegistry _jobTypes; + private readonly ISerializer _serializer; private readonly string _nodeId; private readonly TimeSpan _lease; private readonly TimeSpan _cancellationPollInterval; - public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null, TimeSpan? cancellationPollInterval = null) + public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null, TimeSpan? cancellationPollInterval = null, ISerializer? serializer = null) { _store = store ?? throw new ArgumentNullException(nameof(store)); _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); _timeProvider = timeProvider ?? TimeProvider.System; _jobTypes = jobTypes ?? new JobTypeRegistry(); + _serializer = serializer ?? DefaultSerializer.Instance; _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _lease = lease ?? DefaultLease; @@ -857,9 +933,9 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc var jobType = ResolveJobType(state); var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); - // Hand the job its execution context (identity, attempt, progress, heartbeat, cancellation). The store was - // already incremented to this attempt by the Queued -> Processing transition above. - var context = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease); + // Hand the job its execution context (identity, attempt, typed payload, progress, heartbeat, cancellation). + // The store was already incremented to this attempt by the Queued -> Processing transition above. + var context = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease, state.Payload, state.PayloadType, _serializer); var result = await job.TryRunAsync(context).ConfigureAwait(false); var completedAt = _timeProvider.GetUtcNow(); diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 10f7a76a2..1fef5de1b 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Foundatio.Cronos; +using Foundatio.Serializer; using Foundatio.Messaging; namespace Foundatio.Jobs; @@ -38,6 +39,12 @@ public sealed record ScheduledJobDefinition /// public Func? RetryBackoff { get; init; } + /// + /// Typed arguments serialized into every occurrence's ; the job reads them via + /// . Null when the job takes none. + /// + public object? Arguments { get; init; } + public bool Enabled { get; init; } = true; } @@ -67,6 +74,9 @@ public sealed class CronJobOptions /// Time zone the CRON expression is evaluated in. Null uses the scheduler default (UTC). public TimeZoneInfo? TimeZone { get; set; } + + /// Typed arguments serialized into every occurrence's payload (see ). + public object? Arguments { get; set; } } public interface IJobScheduler @@ -123,16 +133,18 @@ public sealed class JobScheduleProcessor private readonly IJobWorker _jobWorker; private readonly TimeProvider _timeProvider; private readonly IJobTypeRegistry _jobTypes; + private readonly ISerializer _serializer; private readonly string _nodeId; private readonly IMessageTransport? _transport; - public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobWorker jobWorker, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null, IJobTypeRegistry? jobTypes = null) + public JobScheduleProcessor(IJobScheduler scheduler, IJobRuntimeStore store, IJobWorker jobWorker, TimeProvider? timeProvider = null, string? nodeId = null, IMessageTransport? transport = null, IJobTypeRegistry? jobTypes = null, ISerializer? serializer = null) { _scheduler = scheduler ?? throw new ArgumentNullException(nameof(scheduler)); _store = store ?? throw new ArgumentNullException(nameof(store)); _jobWorker = jobWorker ?? throw new ArgumentNullException(nameof(jobWorker)); _timeProvider = timeProvider ?? TimeProvider.System; _jobTypes = jobTypes ?? new JobTypeRegistry(); + _serializer = serializer ?? DefaultSerializer.Instance; _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _transport = transport; } @@ -191,6 +203,10 @@ await _store.CreateIfAbsentAsync(new JobState JobId = jobId, Name = definition.Name, JobType = GetJobTypeName(definition.JobType), + // Explicitly typed: the byte[] -> ReadOnlyMemory conversion maps a null array to an EMPTY memory, + // which would make an argless occurrence look like it carries a zero-byte payload. + Payload = definition.Arguments is null ? null : (ReadOnlyMemory?)_serializer.SerializeToBytes(definition.Arguments), + PayloadType = definition.Arguments?.GetType().FullName, Status = JobStatus.Scheduled, CreatedUtc = utcNow, LastUpdatedUtc = utcNow, diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index ddf060a45..1703723a3 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -310,10 +310,81 @@ private sealed class JobRuntimeProbe public TaskCompletionSource Cancelled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public int RunCount => Volatile.Read(ref _runCount); + public string? LastMessage { get; private set; } - public void RecordRun() + public void RecordRun(string? message = null) { Interlocked.Increment(ref _runCount); + LastMessage = message; + } + } + + [Fact] + public async Task EnqueueAsync_WithTypedArguments_JobReceivesDeserializedPayloadAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobRuntimeProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + + var handle = await client.EnqueueAsync(new ResizeArgs { Path = "/img/1.png", Width = 640 }, cancellationToken: cancellationToken); + + // The payload and its discriminator are durable state, not in-process context. + var state = await store.GetAsync(handle.JobId, cancellationToken); + Assert.NotNull(state?.Payload); + Assert.Equal(typeof(ResizeArgs).FullName, state.PayloadType); + + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); + Assert.Equal("/img/1.png:640", probe.LastMessage); + Assert.Equal(JobStatus.Completed, (await handle.GetStateAsync(cancellationToken))!.Status); + } + + [Fact] + public async Task GetArguments_WhenEnqueuedWithout_ThrowsDescriptiveErrorAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var probe = new JobRuntimeProbe(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(probe) + .BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + + // The args-requiring job was enqueued via the argless API: the run fails (job faults) rather than silently + // executing with defaults, and the error names the fix. + var handle = await client.EnqueueAsync(cancellationToken: cancellationToken); + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); + + var state = await handle.GetStateAsync(cancellationToken); + Assert.Equal(JobStatus.Failed, state!.Status); + Assert.Contains("without arguments", state.Error); + } + + private sealed class ResizeArgs + { + public string? Path { get; set; } + public int Width { get; set; } + } + + private sealed class ArgsConsumingJob : IJob + { + private readonly JobRuntimeProbe _probe; + + public ArgsConsumingJob(JobRuntimeProbe probe) + { + _probe = probe; + } + + public Task RunAsync(JobExecutionContext context) + { + var args = context.GetArguments(); + _probe.RecordRun($"{args.Path}:{args.Width}"); + return Task.FromResult(JobResult.Success); } } From a20cf37108d78e2a8e7c8c0fb66cc202688f7ccc Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 18:54:54 -0500 Subject: [PATCH 54/57] Job execution scopes, bounded worker concurrency, decoupled pump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three execution-model fixes. (1) Every job run now executes inside its own async DI scope, owned for exactly the run — scoped services (DbContexts, units of work) resolve per execution and are disposed when it ends, instead of silently resolving as effective singletons from the root container. (2) The worker gains a bounded pool: JobWorker(maxConcurrency) / JobRuntimePumpOptions.WorkerConcurrency caps in-flight queued jobs per node (default 1 preserves per-node ordering); a slot frees as each job settles, and TryTransition-guarded claims make the parallelism double-run-safe. (3) The pump's scheduling cadence is decoupled from job duration: CRON materialization runs every poll while dispatch/recovery/execution run as one overlapped pass (at most one in flight, drained on shutdown), and the schedule processor now materializes delayed queue/pub-sub message dispatches BEFORE running job occurrences, so the messaging delayed-delivery fallback is never head-of-line blocked by a long job claimed in the same batch. (Review feedback #6.) Co-Authored-By: Claude Fable 5 --- src/Foundatio/FoundatioServicesExtensions.cs | 3 +- src/Foundatio/Jobs/JobRuntime.cs | 59 +++++++++- src/Foundatio/Jobs/JobRuntimePumpService.cs | 56 +++++++-- src/Foundatio/Jobs/JobScheduler.cs | 10 +- tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs | 107 ++++++++++++++++++ 5 files changed, 218 insertions(+), 17 deletions(-) diff --git a/src/Foundatio/FoundatioServicesExtensions.cs b/src/Foundatio/FoundatioServicesExtensions.cs index 07e5ee36b..53abffdc3 100644 --- a/src/Foundatio/FoundatioServicesExtensions.cs +++ b/src/Foundatio/FoundatioServicesExtensions.cs @@ -570,7 +570,8 @@ private void RegisterJobServices() _services.ReplaceSingleton(sp => new JobTypeRegistry(sp.GetServices())); _services.ReplaceSingleton(sp => sp.GetRequiredService()); _services.ReplaceSingleton(sp => new JobClient(sp.GetRequiredService(), sp.GetService(), sp.GetRequiredService(), sp.GetService())); - _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService(), jobTypes: sp.GetRequiredService(), serializer: sp.GetService())); + _services.ReplaceSingleton(sp => new JobWorker(sp.GetRequiredService(), sp, sp.GetService(), jobTypes: sp.GetRequiredService(), serializer: sp.GetService(), + maxConcurrency: sp.GetService()?.WorkerConcurrency ?? 1)); _services.ReplaceSingleton(); _services.ReplaceSingleton(sp => new JobScheduleProcessor( sp.GetRequiredService(), diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index b875c9c33..3e62fe42f 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -815,8 +815,9 @@ public sealed class JobWorker : IJobWorker private readonly string _nodeId; private readonly TimeSpan _lease; private readonly TimeSpan _cancellationPollInterval; + private readonly int _maxConcurrency; - public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null, TimeSpan? cancellationPollInterval = null, ISerializer? serializer = null) + public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeProvider? timeProvider = null, string? nodeId = null, TimeSpan? lease = null, IJobTypeRegistry? jobTypes = null, TimeSpan? cancellationPollInterval = null, ISerializer? serializer = null, int maxConcurrency = 1) { _store = store ?? throw new ArgumentNullException(nameof(store)); _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); @@ -826,6 +827,10 @@ public JobWorker(IJobRuntimeStore store, IServiceProvider serviceProvider, TimeP _nodeId = !String.IsNullOrEmpty(nodeId) ? nodeId : NodeIdentity.Current; _lease = lease ?? DefaultLease; + // Default 1 preserves per-node ordering and today's behavior; raise for I/O-bound jobs. Each in-flight job + // still gets its own DI scope, lease renewal, and cancellation watcher. + _maxConcurrency = Math.Max(1, maxConcurrency); + // Cooperative cancellation is observed by polling the runtime store. The default is intentionally // conservative (one poll per second per running job) so a real store isn't hammered when many jobs run // concurrently; callers that need snappier cancellation can opt into a tighter interval. @@ -843,13 +848,42 @@ public async Task RunQueuedAsync(int limit = 100, CancellationToken cancell ExcludeOccurrences = true }, cancellationToken).ConfigureAwait(false); + if (_maxConcurrency <= 1) + { + int sequentialCompleted = 0; + foreach (var state in queued) + { + if (await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false)) + sequentialCompleted++; + } + + return sequentialCompleted; + } + + // Bounded pool: at most _maxConcurrency jobs in flight; a slot frees the moment a job settles, so one slow + // job never idles the rest of the batch. Claims are TryTransition-guarded, so concurrency cannot double-run. int completed = 0; + using var slots = new SemaphoreSlim(_maxConcurrency, _maxConcurrency); + var inFlight = new List(queued.Count); + foreach (var state in queued) { - if (await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false)) - completed++; + await slots.WaitAsync(cancellationToken).ConfigureAwait(false); + inFlight.Add(Task.Run(async () => + { + try + { + if (await RunJobStateAsync(state, cancellationToken).ConfigureAwait(false)) + Interlocked.Increment(ref completed); + } + finally + { + slots.Release(); + } + }, CancellationToken.None)); } + await Task.WhenAll(inFlight).ConfigureAwait(false); return completed; } @@ -931,13 +965,12 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc try { var jobType = ResolveJobType(state); - var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); // Hand the job its execution context (identity, attempt, typed payload, progress, heartbeat, cancellation). // The store was already incremented to this attempt by the Queued -> Processing transition above. var context = new JobExecutionContext(state.JobId, state.Attempt + 1, linkedCancellationTokenSource.Token, _store, _nodeId, _lease, state.Payload, state.PayloadType, _serializer); - var result = await job.TryRunAsync(context).ConfigureAwait(false); + var result = await ExecuteJobAsync(jobType, context).ConfigureAwait(false); var completedAt = _timeProvider.GetUtcNow(); if (result.IsCancelled) @@ -997,6 +1030,22 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc } } + // Every execution gets its own async DI scope, owned for exactly the run: scoped services (DbContexts, units of + // work) resolve per run and are disposed when it ends, instead of silently resolving as effective singletons from + // the root container. A bare provider without scope support (custom IServiceProvider) runs unscoped. + private async Task ExecuteJobAsync(Type jobType, JobExecutionContext context) + { + if (_serviceProvider.GetService(typeof(IServiceScopeFactory)) is IServiceScopeFactory scopeFactory) + { + await using var scope = scopeFactory.CreateAsyncScope(); + var job = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(scope.ServiceProvider, jobType); + return await job.TryRunAsync(context).ConfigureAwait(false); + } + + var unscoped = (IJob)ActivatorUtilities.GetServiceOrCreateInstance(_serviceProvider, jobType); + return await unscoped.TryRunAsync(context).ConfigureAwait(false); + } + private Type ResolveJobType(JobState state) { if (String.IsNullOrEmpty(state.JobType)) diff --git a/src/Foundatio/Jobs/JobRuntimePumpService.cs b/src/Foundatio/Jobs/JobRuntimePumpService.cs index 76cda7099..fb4a01502 100644 --- a/src/Foundatio/Jobs/JobRuntimePumpService.cs +++ b/src/Foundatio/Jobs/JobRuntimePumpService.cs @@ -28,6 +28,12 @@ public class JobRuntimePumpOptions /// Maximum processing attempts for an ad-hoc job before a stale (lease-expired) instance is dead-lettered. Default 3. public int MaxJobAttempts { get; set; } = 3; + + /// + /// Maximum queued jobs the worker executes concurrently per node. Default 1, which preserves per-node run + /// ordering; raise it for I/O-bound jobs. Every in-flight job gets its own DI scope, lease, and cancellation watcher. + /// + public int WorkerConcurrency { get; set; } = 1; } /// @@ -85,7 +91,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) return; } - _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize})", _options.PollInterval, _options.BatchSize); + _logger.LogInformation("Job runtime pump starting (poll interval {PollInterval}, batch size {BatchSize}, worker concurrency {WorkerConcurrency})", _options.PollInterval, _options.BatchSize, _options.WorkerConcurrency); + + // Execution (dispatching due work and running jobs) is an overlapped pass: the scheduling stage below keeps + // its cadence every poll even while a long job runs, so CRON materialization and the messaging delayed- + // delivery fallback are never head-of-line blocked by job duration. At most one pass is in flight; if the + // prior pass is still running when the loop comes around, this tick only materializes. Overlap-adjacent races + // are safe: dispatch claims are leased and job claims are TryTransition-guarded, so nothing double-runs. + var executionPass = Task.CompletedTask; while (!stoppingToken.IsCancellationRequested) { @@ -96,15 +109,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Materialize CRON occurrences due within the misfire window (deduped, idempotent). await _processor.EnqueueDueOccurrencesAsync(now, stoppingToken).AnyContext(); - // Claim and run due dispatches: CRON occurrences plus delayed queue/pub-sub messages, recovering - // occurrences whose processing lease expired and applying retry/dead-letter. - await _processor.RunDueOccurrencesAsync(now, _options.BatchSize, lease: null, stoppingToken).AnyContext(); - - // Recover ad-hoc (non-CRON) jobs whose processing lease expired (a worker crash mid-run). - await _worker.RecoverStaleAsync(_options.MaxJobAttempts, _options.BatchSize, stoppingToken).AnyContext(); - - // Run jobs submitted via IJobClient sitting in the Queued state. - await _worker.RunQueuedAsync(_options.BatchSize, stoppingToken).AnyContext(); + if (executionPass.IsCompleted) + executionPass = Task.Run(() => RunExecutionPassAsync(now, stoppingToken), CancellationToken.None); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { @@ -125,6 +131,36 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + // Drain the in-flight pass so shutdown does not abandon running jobs mid-settlement. + try + { + await executionPass.AnyContext(); + } + catch (OperationCanceledException) { } + _logger.LogInformation("Job runtime pump stopped"); } + + private async Task RunExecutionPassAsync(DateTimeOffset now, CancellationToken stoppingToken) + { + try + { + // Claim and run due dispatches: delayed queue/pub-sub messages first, then CRON occurrences, recovering + // occurrences whose processing lease expired and applying retry/dead-letter. + await _processor.RunDueOccurrencesAsync(now, _options.BatchSize, lease: null, stoppingToken).AnyContext(); + + // Recover ad-hoc (non-CRON) jobs whose processing lease expired (a worker crash mid-run). + await _worker.RecoverStaleAsync(_options.MaxJobAttempts, _options.BatchSize, stoppingToken).AnyContext(); + + // Run jobs submitted via IJobClient sitting in the Queued state. + await _worker.RunQueuedAsync(_options.BatchSize, stoppingToken).AnyContext(); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + } + catch (Exception ex) + { + _logger.LogError(ex, "Error running job runtime execution pass: {Message}", ex.Message); + } + } } diff --git a/src/Foundatio/Jobs/JobScheduler.cs b/src/Foundatio/Jobs/JobScheduler.cs index 1fef5de1b..623f3c3b9 100644 --- a/src/Foundatio/Jobs/JobScheduler.cs +++ b/src/Foundatio/Jobs/JobScheduler.cs @@ -247,14 +247,22 @@ public async Task RunDueOccurrencesAsync(DateTimeOffset utcNow, int limit = var dispatches = await _store.ClaimDueDispatchesAsync(utcNow, limit, _nodeId, lease ?? DefaultLease, cancellationToken).ConfigureAwait(false); int completed = 0; + // Materialize delayed/scheduled MESSAGES before running any job occurrence: message dispatch is cheap and + // latency-sensitive (it is the messaging delayed-delivery fallback), so it must never wait behind a long job + // run that happened to be claimed earlier in the same batch. foreach (var dispatch in dispatches) { if (dispatch.Kind is ScheduledDispatchKind.QueueMessage or ScheduledDispatchKind.PubSubMessage) { await MaterializeMessageDispatchAsync(dispatch, cancellationToken).ConfigureAwait(false); completed++; - continue; } + } + + foreach (var dispatch in dispatches) + { + if (dispatch.Kind is ScheduledDispatchKind.QueueMessage or ScheduledDispatchKind.PubSubMessage) + continue; if (dispatch.Kind != ScheduledDispatchKind.JobOccurrence) { diff --git a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs index 1703723a3..4d5119a90 100644 --- a/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs +++ b/tests/Foundatio.Tests/Jobs/JobRuntimeTests.cs @@ -365,6 +365,113 @@ public async Task GetArguments_WhenEnqueuedWithout_ThrowsDescriptiveErrorAsync() Assert.Contains("without arguments", state.Error); } + [Fact] + public async Task RunJob_ResolvesScopedServicesPerExecutionAndDisposesThemAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var tracker = new ScopedLifetimeTracker(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(tracker) + .AddScoped() + .AddTransient() + .BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a"); + + var first = await client.EnqueueAsync(cancellationToken: cancellationToken); + var second = await client.EnqueueAsync(cancellationToken: cancellationToken); + Assert.True(await worker.RunAsync(first.JobId, cancellationToken)); + Assert.True(await worker.RunAsync(second.JobId, cancellationToken)); + + // Two runs -> two scoped instances (not one root-container singleton), each disposed when its run ended. + Assert.Equal(2, tracker.Created); + Assert.Equal(2, tracker.Disposed); + } + + [Fact] + public async Task RunQueuedAsync_WithMaxConcurrency_RespectsCapAndRunsInParallelAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new InMemoryJobRuntimeStore(); + var gauge = new ConcurrencyGauge(); + await using var serviceProvider = new ServiceCollection() + .AddSingleton(gauge) + .BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a", maxConcurrency: 2); + + for (int i = 0; i < 6; i++) + await client.EnqueueAsync(cancellationToken: cancellationToken); + + Assert.Equal(6, await worker.RunQueuedAsync(cancellationToken: cancellationToken)); + Assert.True(gauge.MaxObserved <= 2, $"expected at most 2 in-flight jobs, observed {gauge.MaxObserved}"); + Assert.True(gauge.MaxObserved > 1, "expected the pool to actually run jobs in parallel"); + } + + private sealed class ScopedLifetimeTracker + { + private int _created; + private int _disposed; + public int Created => Volatile.Read(ref _created); + public int Disposed => Volatile.Read(ref _disposed); + public void RecordCreated() => Interlocked.Increment(ref _created); + public void RecordDisposed() => Interlocked.Increment(ref _disposed); + } + + private sealed class ScopedDependency : IDisposable + { + private readonly ScopedLifetimeTracker _tracker; + + public ScopedDependency(ScopedLifetimeTracker tracker) + { + _tracker = tracker; + _tracker.RecordCreated(); + } + + public void Dispose() => _tracker.RecordDisposed(); + } + + private sealed class ScopedConsumingJob : IJob + { + // The dependency's usefulness is its lifetime tracking; resolving it is the test. + public ScopedConsumingJob(ScopedDependency dependency) => _ = dependency; + + public Task RunAsync(JobExecutionContext context) => Task.FromResult(JobResult.Success); + } + + private sealed class ConcurrencyGauge + { + private int _inFlight; + private int _maxObserved; + + public int MaxObserved => Volatile.Read(ref _maxObserved); + + public async Task TrackAsync() + { + int current = Interlocked.Increment(ref _inFlight); + int max; + while (current > (max = Volatile.Read(ref _maxObserved))) + Interlocked.CompareExchange(ref _maxObserved, current, max); + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + Interlocked.Decrement(ref _inFlight); + } + } + + private sealed class ConcurrencyProbeJob : IJob + { + private readonly ConcurrencyGauge _gauge; + + public ConcurrencyProbeJob(ConcurrencyGauge gauge) => _gauge = gauge; + + public async Task RunAsync(JobExecutionContext context) + { + await _gauge.TrackAsync(); + return JobResult.Success; + } + } + private sealed class ResizeArgs { public string? Path { get; set; } From beebbadcf94c50cc93940552bf1917364d8d162e Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 19:00:42 -0500 Subject: [PATCH 55/57] Supervised lease renewal/cancellation loops; shared-Key policies compared by identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lease renewal and cancellation polling were fire-and-forget timers whose discarded tasks swallowed every exception: a store outage silently stopped renewals, the lease lapsed, another node reclaimed the job, and the still- running original double-executed its side effects. Both are now supervised async loops owned by the run — started with it, stopped and awaited when it ends. The renewal loop treats a clean "renewal denied" as lease lost (cancelling the run, as before) and now also treats renewal that keeps THROWING the same way once the lease window passes without one success; the cancellation poll loop rides through store failures since a missed poll only delays cooperative cancellation. Also closes the shared-Key policy-divergence footgun: subscriptions sharing a consumer Key form one competing group, and their RedeliveryBackoff / DeadLetterWhen delegates are now compared by identity instead of mere presence, so two members whose failure logic differs are rejected at subscribe time instead of settling the same message differently by receiver. (Review feedback #7.) Co-Authored-By: Claude Fable 5 --- src/Foundatio/Jobs/JobRuntime.cs | 97 ++++++++++++----- src/Foundatio/Messaging/MessageBus.cs | 3 +- src/Foundatio/Messaging/MessageClientCore.cs | 16 +-- .../Jobs/LeaseSupervisionTests.cs | 101 ++++++++++++++++++ .../Foundatio.Tests/Messaging/PubSubTests.cs | 26 +++++ 5 files changed, 209 insertions(+), 34 deletions(-) create mode 100644 tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs diff --git a/src/Foundatio/Jobs/JobRuntime.cs b/src/Foundatio/Jobs/JobRuntime.cs index 3e62fe42f..d9bb8be69 100644 --- a/src/Foundatio/Jobs/JobRuntime.cs +++ b/src/Foundatio/Jobs/JobRuntime.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Foundatio.Messaging; using Foundatio.Serializer; +using Foundatio.Utility; using Microsoft.Extensions.DependencyInjection; namespace Foundatio.Jobs; @@ -959,8 +960,12 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc JobInstruments.Started.Add(1, jobTag); using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - using var cancellationWatcher = WatchCancellation(state.JobId, linkedCancellationTokenSource); - using var leaseRenewer = RenewLeasePeriodically(state.JobId, linkedCancellationTokenSource); + + // Lease renewal and cancellation polling are supervised loops owned by this run — started here, stopped and + // awaited when the run ends — not fire-and-forget timers whose failures would vanish unobserved. + using var supervisionCancellationTokenSource = new CancellationTokenSource(); + var leaseLoop = Task.Run(() => RunLeaseRenewalLoopAsync(state.JobId, linkedCancellationTokenSource, supervisionCancellationTokenSource.Token), CancellationToken.None); + var cancellationLoop = Task.Run(() => RunCancellationPollLoopAsync(state.JobId, linkedCancellationTokenSource, supervisionCancellationTokenSource.Token), CancellationToken.None); try { @@ -1028,6 +1033,13 @@ private async Task RunJobStateAsync(JobState state, CancellationToken canc JobInstruments.RunTime.Record((failedAt - now).TotalMilliseconds, jobTag); throw; } + finally + { + // Stop and await the supervision loops so no renewal/poll outlives its run (and so their final state is + // observed rather than dropped on the floor). The loops never throw; they classify failures themselves. + await supervisionCancellationTokenSource.CancelAsync().ConfigureAwait(false); + await Task.WhenAll(leaseLoop, cancellationLoop).ConfigureAwait(false); + } } // Every execution gets its own async DI scope, owned for exactly the run: scoped services (DbContexts, units of @@ -1061,47 +1073,78 @@ private Type ResolveJobType(JobState state) } } - private IDisposable WatchCancellation(string jobId, CancellationTokenSource cancellationTokenSource) - { - return new Timer(_ => _ = PollCancellationAsync(jobId, cancellationTokenSource), null, _cancellationPollInterval, _cancellationPollInterval); - } - - private IDisposable RenewLeasePeriodically(string jobId, CancellationTokenSource cancellationTokenSource) + // Lease renewal supervises its own failures. A clean "renewal denied" means the lease was lost to another node. + // Renewal that keeps THROWING (a store outage) is treated the same once the lease window passes without one + // success — the lease has lapsed on the broker's clock too, so another node may already have reclaimed the job, + // and letting this run continue would double-execute its side effects. Both paths cancel the run; the terminal + // transition (guarded by expectedNodeId) then cannot overwrite the new owner's state. + private async Task RunLeaseRenewalLoopAsync(string jobId, CancellationTokenSource jobCancellation, CancellationToken supervision) { // Renew well before the lease elapses so a slow-but-alive worker keeps ownership and is not reclaimed. var interval = TimeSpan.FromMilliseconds(Math.Max(250, _lease.TotalMilliseconds / 3)); - return new Timer(_ => _ = RenewLeaseAsync(jobId, cancellationTokenSource), null, interval, interval); - } - - private async Task RenewLeaseAsync(string jobId, CancellationTokenSource cancellationTokenSource) - { - if (cancellationTokenSource.IsCancellationRequested) - return; + long lastSuccessTimestamp = _timeProvider.GetTimestamp(); - try + while (!supervision.IsCancellationRequested) { - // If renewal fails the lease was lost to another node; cancel the run so this worker stops and - // its terminal transition (guarded by expectedNodeId) cannot overwrite the new owner's state. - if (!await _store.RenewClaimAsync(jobId, _nodeId, _lease, CancellationToken.None).ConfigureAwait(false)) - await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await _timeProvider.SafeDelay(interval, supervision).ConfigureAwait(false); + if (supervision.IsCancellationRequested) + return; + + try + { + if (!await _store.RenewClaimAsync(jobId, _nodeId, _lease, CancellationToken.None).ConfigureAwait(false)) + { + await CancelRunAsync(jobCancellation).ConfigureAwait(false); + return; + } + + lastSuccessTimestamp = _timeProvider.GetTimestamp(); + } + catch (Exception) + { + // Transient store failure: retry next tick — but never outlive the lease on hope. + if (_timeProvider.GetElapsedTime(lastSuccessTimestamp) >= _lease) + { + await CancelRunAsync(jobCancellation).ConfigureAwait(false); + return; + } + } } - catch (ObjectDisposedException) + } + + // Cancellation polling keeps polling through store failures (a missed poll only delays cooperative cancellation, + // it cannot double-run anything), and stops when the run ends or cancellation is observed. + private async Task RunCancellationPollLoopAsync(string jobId, CancellationTokenSource jobCancellation, CancellationToken supervision) + { + while (!supervision.IsCancellationRequested) { + await _timeProvider.SafeDelay(_cancellationPollInterval, supervision).ConfigureAwait(false); + if (supervision.IsCancellationRequested) + return; + + try + { + if (await _store.IsCancellationRequestedAsync(jobId, CancellationToken.None).ConfigureAwait(false)) + { + await CancelRunAsync(jobCancellation).ConfigureAwait(false); + return; + } + } + catch (Exception) + { + } } } - private async Task PollCancellationAsync(string jobId, CancellationTokenSource cancellationTokenSource) + private static async Task CancelRunAsync(CancellationTokenSource jobCancellation) { - if (cancellationTokenSource.IsCancellationRequested) - return; - try { - if (await _store.IsCancellationRequestedAsync(jobId, CancellationToken.None).ConfigureAwait(false)) - await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await jobCancellation.CancelAsync().ConfigureAwait(false); } catch (ObjectDisposedException) { + // The run already completed and disposed its token source; nothing left to cancel. } } } diff --git a/src/Foundatio/Messaging/MessageBus.cs b/src/Foundatio/Messaging/MessageBus.cs index 54eba65a1..87089ea2a 100644 --- a/src/Foundatio/Messaging/MessageBus.cs +++ b/src/Foundatio/Messaging/MessageBus.cs @@ -126,7 +126,8 @@ public sealed class MessageSubscriptionOptions /// /// Consumer identity. Subscriptions sharing a key on the same channel form one consumer group and compete; /// defaults to a per-channel key derived from the route. Subscriptions sharing a key must configure identical - /// failure policies — only the presence of a backoff/DeadLetterWhen is verified, not the delegate itself. + /// failure policies — the backoff/DeadLetterWhen DELEGATES are compared by identity, so share the same delegate + /// instances (a lambda recreated per subscription will be rejected as a conflicting registration). /// public string? Key { get; set; } diff --git a/src/Foundatio/Messaging/MessageClientCore.cs b/src/Foundatio/Messaging/MessageClientCore.cs index ddef6591c..8e8c8f0d6 100644 --- a/src/Foundatio/Messaging/MessageClientCore.cs +++ b/src/Foundatio/Messaging/MessageClientCore.cs @@ -1259,8 +1259,8 @@ internal sealed record MessageListenerRegistration public required AckMode AckMode { get; init; } public required int MaxConcurrency { get; init; } public required int? MaxAttempts { get; init; } - public required bool HasRedeliveryBackoff { get; init; } - public required bool HasDeadLetterWhen { get; init; } + public required Func? RedeliveryBackoff { get; init; } + public required Func? DeadLetterWhen { get; init; } public static MessageListenerRegistration Create(Delegate handler, ListenerConfig config) { @@ -1272,20 +1272,24 @@ public static MessageListenerRegistration Create(Delegate handler, ListenerConfi AckMode = config.AckMode, MaxConcurrency = Math.Max(1, config.MaxConcurrency), MaxAttempts = config.MaxAttempts, - HasRedeliveryBackoff = config.RedeliveryBackoff is not null, - HasDeadLetterWhen = config.DeadLetterWhen is not null + RedeliveryBackoff = config.RedeliveryBackoff, + DeadLetterWhen = config.DeadLetterWhen }; } public bool Matches(MessageListenerRegistration other) { + // Failure policies are compared by delegate identity, not mere presence: subscriptions sharing a consumer Key + // form ONE competing group, and two members whose retry/dead-letter LOGIC differs would settle the same + // message differently depending on which member happened to receive it. Callers sharing a Key must share the + // actual delegate instances. return MessageType == other.MessageType && Source == other.Source && Handler == other.Handler && AckMode == other.AckMode && MaxConcurrency == other.MaxConcurrency && MaxAttempts == other.MaxAttempts - && HasRedeliveryBackoff == other.HasRedeliveryBackoff - && HasDeadLetterWhen == other.HasDeadLetterWhen; + && Equals(RedeliveryBackoff, other.RedeliveryBackoff) + && Equals(DeadLetterWhen, other.DeadLetterWhen); } } diff --git a/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs b/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs new file mode 100644 index 000000000..61bef643f --- /dev/null +++ b/tests/Foundatio.Tests/Jobs/LeaseSupervisionTests.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Jobs; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Foundatio.Tests.Jobs; + +public class LeaseSupervisionTests +{ + [Fact] + public async Task RenewalDenied_CancelsRunningJobAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new LeaseFailingStore(new InMemoryJobRuntimeStore()) { DenyRenewals = true }; + await using var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a", lease: TimeSpan.FromSeconds(1)); + + var handle = await client.EnqueueAsync(cancellationToken: cancellationToken); + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); + + // A clean "renewal denied" means another node owns the lease: the run must be cancelled, not left executing. + var state = await handle.GetStateAsync(cancellationToken); + Assert.Equal(JobStatus.Cancelled, state!.Status); + } + + [Fact] + public async Task RenewalThrowingPastLeaseWindow_CancelsRunningJobAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var store = new LeaseFailingStore(new InMemoryJobRuntimeStore()) { ThrowOnRenewals = true }; + await using var serviceProvider = new ServiceCollection().BuildServiceProvider(); + var client = new JobClient(store); + var worker = new JobWorker(store, serviceProvider, nodeId: "node-a", lease: TimeSpan.FromSeconds(1)); + + var handle = await client.EnqueueAsync(cancellationToken: cancellationToken); + Assert.True(await worker.RunAsync(handle.JobId, cancellationToken)); + + // Renewal that keeps THROWING must not let the run outlive its lease: once the window passes without one + // successful renewal, another node may have reclaimed the job, so continuing would double-run side effects. + var state = await handle.GetStateAsync(cancellationToken); + Assert.Equal(JobStatus.Cancelled, state!.Status); + } + + private sealed class WaitForCancellationJob : IJob + { + public async Task RunAsync(JobExecutionContext context) + { + try + { + // Runs "forever" unless the supervision loop cancels the run. + await Task.Delay(TimeSpan.FromSeconds(30), context.CancellationToken); + } + catch (OperationCanceledException) + { + return JobResult.CancelledWithMessage("lease lost"); + } + + return JobResult.FailedWithMessage("was never cancelled"); + } + } + + // Delegates everything to the inner store; renewals can be denied (clean lease loss) or made to throw (store outage). + private sealed class LeaseFailingStore : IJobRuntimeStore + { + private readonly IJobRuntimeStore _inner; + + public LeaseFailingStore(IJobRuntimeStore inner) => _inner = inner; + + public bool DenyRenewals { get; set; } + public bool ThrowOnRenewals { get; set; } + + public Task RenewClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken cancellationToken = default) + { + if (ThrowOnRenewals) + throw new TimeoutException("store unreachable"); + + return DenyRenewals ? Task.FromResult(false) : _inner.RenewClaimAsync(jobId, nodeId, lease, cancellationToken); + } + + public Task GetAsync(string jobId, CancellationToken ct = default) => _inner.GetAsync(jobId, ct); + public Task> QueryAsync(JobQuery query, CancellationToken ct = default) => _inner.QueryAsync(query, ct); + public Task CreateIfAbsentAsync(JobState initial, CancellationToken ct = default) => _inner.CreateIfAbsentAsync(initial, ct); + public Task TryTransitionAsync(string jobId, JobStatus expectedStatus, JobStatus newStatus, JobStatePatch? patch = null, string? expectedNodeId = null, CancellationToken ct = default) => _inner.TryTransitionAsync(jobId, expectedStatus, newStatus, patch, expectedNodeId, ct); + public Task TryClaimAsync(string jobId, string nodeId, TimeSpan lease, CancellationToken ct = default) => _inner.TryClaimAsync(jobId, nodeId, lease, ct); + public Task ReleaseClaimAsync(string jobId, string nodeId, CancellationToken ct = default) => _inner.ReleaseClaimAsync(jobId, nodeId, ct); + public Task> GetExpiredProcessingAsync(DateTimeOffset now, int limit, CancellationToken ct = default) => _inner.GetExpiredProcessingAsync(now, limit, ct); + public Task TryReclaimExpiredAsync(string jobId, DateTimeOffset now, string expectedNodeId, JobStatus newStatus, JobStatePatch? patch = null, CancellationToken ct = default) => _inner.TryReclaimExpiredAsync(jobId, now, expectedNodeId, newStatus, patch, ct); + public Task SetProgressAsync(string jobId, int? percent = null, string? message = null, CancellationToken ct = default) => _inner.SetProgressAsync(jobId, percent, message, ct); + public Task IncrementAttemptAsync(string jobId, CancellationToken ct = default) => _inner.IncrementAttemptAsync(jobId, ct); + public Task RequestCancellationAsync(string jobId, CancellationToken ct = default) => _inner.RequestCancellationAsync(jobId, ct); + public Task IsCancellationRequestedAsync(string jobId, CancellationToken ct = default) => _inner.IsCancellationRequestedAsync(jobId, ct); + public Task ScheduleDispatchAsync(ScheduledDispatchState dispatch, CancellationToken ct = default) => _inner.ScheduleDispatchAsync(dispatch, ct); + public Task> ClaimDueDispatchesAsync(DateTimeOffset now, int limit, string nodeId, TimeSpan lease, CancellationToken ct = default) => _inner.ClaimDueDispatchesAsync(now, limit, nodeId, lease, ct); + public Task CompleteDispatchAsync(string dispatchId, string nodeId, CancellationToken ct = default) => _inner.CompleteDispatchAsync(dispatchId, nodeId, ct); + public Task ReleaseDispatchAsync(string dispatchId, string nodeId, DateTimeOffset nextDueUtc, CancellationToken ct = default) => _inner.ReleaseDispatchAsync(dispatchId, nodeId, nextDueUtc, ct); + } +} diff --git a/tests/Foundatio.Tests/Messaging/PubSubTests.cs b/tests/Foundatio.Tests/Messaging/PubSubTests.cs index 2d4acd221..9728a3fef 100644 --- a/tests/Foundatio.Tests/Messaging/PubSubTests.cs +++ b/tests/Foundatio.Tests/Messaging/PubSubTests.cs @@ -305,6 +305,32 @@ await Assert.ThrowsAsync(async () => await pubSub.SubscribeAsync((_, _) => Task.CompletedTask, new MessageSubscriptionOptions { Subscription = "same-key", Key = "shared" }, cancellationToken)); } + [Fact] + public async Task SubscribeAsync_WithSameKeyAndDifferentFailurePolicy_ThrowsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + await using var pubSub = new MessageBus(new InMemoryMessageTransport()); + Func, CancellationToken, Task> handler = (_, _) => Task.CompletedTask; + + await using var first = await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions + { + Subscription = "same-key", + Key = "shared", + DeadLetterWhen = static ex => ex is InvalidOperationException + }, cancellationToken); + + // Shared-key subscriptions form ONE competing group; members with different retry/dead-letter LOGIC would + // settle the same message differently depending on who received it, so a divergent policy must be rejected — + // by delegate identity, not by mere has-a-policy presence. + await Assert.ThrowsAsync(async () => + await pubSub.SubscribeAsync(handler, new MessageSubscriptionOptions + { + Subscription = "same-key", + Key = "shared", + DeadLetterWhen = static ex => ex is ArgumentException + }, cancellationToken)); + } + [Fact] public async Task SubscribeAsync_WithGroupedTopicAndSubscriptionIdentity_ReceivesRawMessagesAsync() { From 9ec7adcdec1d0473d42e7a58e9b44f664fa1fa84 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 19:10:13 -0500 Subject: [PATCH 56/57] Rewrite the redesign guide and Foundatio skill for the final API Both documents still described the superseded IQueue/IPubSub design. They now document what shipped: one IMessageBus whose verb decides delivery, topology-free handlers with delivery intent, canonical DestinationAddress with routing-as-topology under TopologyMode, core-owned retry/dead-letter over role-aware transport capabilities with the scheduled-dispatch fallback, the durable jobs runtime (typed payloads, per-run scopes, bounded concurrency, supervised leases), the Foundatio.Testing harness, and the legacy namespaces. Co-Authored-By: Claude Fable 5 --- .agents/skills/foundatio/SKILL.md | 334 +++++++++++-------------- docs/guide/messaging-jobs-redesign.md | 335 ++++++++++---------------- 2 files changed, 276 insertions(+), 393 deletions(-) diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index d4cf806d8..a7b6d4df6 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -2,106 +2,92 @@ name: foundatio description: > Use when working with Foundatio infrastructure abstractions for .NET -- caching, - queuing, messaging, file storage, distributed locking, or background jobs. Apply - when using ICacheClient, IQueue, IMessageBus, IFileStorage, ILockProvider, IJob, + messaging, background jobs, file storage, distributed locking, or queuing. Apply + when using ICacheClient, IMessageBus, IJobClient, IJob, IFileStorage, ILockProvider, or resilience patterns like retry and circuit breakers. Covers in-memory and - production implementations (Redis, Azure, AWS, Kafka, RabbitMQ). Use context7 - MCP to fetch current API docs and examples. + production implementations (Redis, AWS, Azure). Use context7 MCP to fetch current + API docs and examples. --- # Foundatio -Pluggable infrastructure abstractions for distributed .NET apps. Interface-first, testable, swappable between in-memory (dev/test) and production providers (Redis, Azure, AWS) with zero application code changes. +Pluggable infrastructure abstractions for distributed .NET apps. Interface-first, testable, swappable between in-memory (dev/test) and production providers (Redis, AWS, Azure) with zero application code changes. ## Documentation via context7 Use context7 MCP for complete, up-to-date API docs and examples. The main library ID covers all abstractions and implementations: ```text -query-docs(libraryId="/foundatiofx/foundatio", query="How to configure queue retry policies and dead letter handling") +query-docs(libraryId="/foundatiofx/foundatio", query="How to configure messaging retry policies and dead letter handling") ``` Query with specific questions, not single keywords. All provider docs (Redis, Azure, AWS, Kafka, etc.) are included in the main library. - -## Messaging/Jobs Redesign Notes - -- New queue/pub-sub APIs live under `Foundatio.Messaging`: app-facing `IQueue`, `IPubSub`, shared `IReceivedMessage` / `IReceivedMessage`, `QueueMessageOptions`, `QueueReceiveOptions`, `QueueConsumerOptions`, `PubSubMessageOptions`, and `PubSubSubscriptionOptions`. -- Route resolution is centralized in `IMessageRouter`: operation override > explicit route map > interface/base-type map > `MessageRouteAttribute` > configured default/convention. Configure with `.Messaging.ConfigureRouting(...)`; use `UseDefaultQueue(...)`, `UseDefaultTopic(...)`, `MapQueue(...)`, and `MapTopic(...)` as the normal path. `Destination`, `Source`, `Topic`, and `Subscription` are advanced operation overrides. -- Routing configuration also declares startup topology. `IMessageTopology.GetDeclarations()` returns configured queues/topics/subscriptions, `EnsureAsync()` creates them through `ISupportsProvisioning`, and `ValidateAsync()` checks that they already exist for apps without create permissions. Operation overrides are not included in topology declarations. -- Pub/sub topic routing and subscription identity are separate. Topic answers where an event is published; subscription answers the logical service/consumer group. Same subscription across instances means competing consumers; different subscriptions on the same topic fan out. `PubSubSubscriptionOptions.Key` is only the local duplicate-listener key. -- Raw/envelope paths make grouped/default routes usable: `queue.ReceiveAsync(...)`, `queue.StartConsumerAsync(Func)`, and `pubsub.SubscribeAsync(Func)`. Typed `ReceiveAsync` / `SubscribeAsync` remain the simple path. -- Listener startup returns handles: `StartConsumerAsync` returns `IMessageConsumer`; `SubscribeAsync` returns `IMessageSubscription`. Same-key duplicate registrations are idempotent only for the same handler/options; conflicting registrations throw. Use `RunConsumerAsync` or `RunSubscriptionAsync` only for blocking lifetime loops. -- Received-message settlement uses explicit verbs only: `CompleteAsync`, `AbandonAsync`, `DeadLetterAsync`, `RenewLockAsync`, and `ReportProgressAsync`. Unsupported capabilities should throw clearly instead of silently downgrading. -- New durable job runtime roles are separated: `IJobClient` submits and returns `JobHandle`, `IJobMonitor` queries state, `IJobRuntimeStore` persists runtime state, and `IJobWorker` claims and executes queued jobs. Job types persist stable registry names via `IJobTypeRegistry` / `.Jobs.Register(name)`, not assembly-qualified names. -- In-memory setup for the redesign is `services.AddFoundatio().Messaging.ConfigureRouting(...).UseInMemory().Jobs.UseInMemoryRuntime()`. +## Messaging and Jobs (current API) + +- One messaging client: `IMessageBus` in `Foundatio.Messaging`. The caller's verb decides delivery -- `SendAsync` is a command processed by exactly one handler instance across the fleet (competing consumers); `PublishAsync` is an event received once per subscribing service (a scaled service's instances compete), or by every instance when the subscription sets `PerInstance`. `SendBatchAsync` / `PublishBatchAsync` batch both verbs. Per-operation options: `MessageSendOptions` / `MessagePublishOptions` (priority, `Delay`/`DeliverAt`, TTL, correlation id, headers, `Destination`/`Topic` override). +- Handlers are topology-free. Implement `IMessageHandler` and register with `.Messaging.AddHandler(o => ...)`; a hosted service (`MessageHandlerHostedService`) starts them all and each message is dispatched in its own DI scope. `IMessageBus.SubscribeAsync` is the dynamic path and returns an `IMessageSubscription` handle. +- `MessageSubscriptionOptions` declares delivery intent: `Deliveries` (`MessageDeliveries.Sent`/`Published`/`Both`, default `Both`), `Subscription` / `SubscriptionQualifier` / `PerInstance` for subscriber-group identity, `MaxConcurrency` (default 1, preserves per-handler ordering), `MaxAttempts` / `RedeliveryBackoff` / `DeadLetterWhen` (+ `DeadLetterOn()` shorthand) retry overrides, `AckMode` (`Auto` default / `Manual`), and `Key` (subscriptions sharing a key form one competing group; their backoff/dead-letter DELEGATES are compared by identity, so share delegate instances). +- Routing is central: `.Messaging.ConfigureRouting(r => r.UseDefaultQueue(...).UseDefaultTopic(...).MapQueue(...).MapTopic(...).UseServiceIdentity(...).UseSubscriptionIdentity(...).UseConvention(...))`. Precedence: operation override > exact map > interface/base-type map > `MessageRouteAttribute` > configured default > convention > kebab-cased type name. +- Routing config doubles as topology declarations (`DestinationDeclaration` with a canonical `DestinationAddress` -- `ForQueue`/`ForTopic`/`ForSubscription`). `IMessageTopology` exposes `GetDeclarations()` / `EnsureAsync()` / `ValidateAsync()`. `.Messaging.ConfigureTopology(TopologyMode.Ensure | Validate | None)` picks whether the client creates missing destinations (default), only verifies they exist (throws at startup when missing), or never touches topology; the handler host applies the mode at startup. +- The CORE owns retry/dead-lettering identically on every transport: default `RetryPolicy` is `MaxAttempts` 5 with immediate-then-10s/20s/30s backoff (+/-20% jitter); configure via `.Messaging.ConfigureRetry(p => p with { ... })`. Dead-lettered messages go to the transport's native sink or a derived `"{source}.deadletter"` destination, stamped with `message.dead_letter.*` forensics headers (`KnownHeaders.DeadLetter*`). Never configure broker-native redrive policies. +- Message settlement: `IMessageContext` / `IMessageContext` with `CompleteAsync()`, `RejectAsync(RejectOptions)` (non-terminal = retry, optionally with `RedeliveryDelay`; `Terminal = true` = dead-letter with `Reason`/`Exception`), and `RenewLockAsync()`. Auto-ack is the default. +- Transports advertise role-aware capabilities: `ITransportInfo.GetCapabilities(DestinationRole)` returns `TransportCapabilities` (e.g. the AWS transport's queue role has a native 15-minute `MaxDeliveryDelay`; its topic role has none). Delays beyond a ceiling and store-parked retries fall back to the durable runtime store (`IScheduledDispatchStore`, satisfied by any `IJobRuntimeStore`) and are drained by the job runtime pump -- never silently truncated. +- Durable jobs: implement `IJob` (`Task RunAsync(JobExecutionContext context)`). `IJobClient.EnqueueAsync()` / `EnqueueAsync(args)` (typed payloads) returns a `JobHandle`; `IJobMonitor` queries state; `IJobWorker` executes with per-run DI scopes, bounded concurrency, and supervised lease renewal. `JobExecutionContext` gives `JobId`/`Attempt`/`CancellationToken`, `GetArguments()`, `ReportProgressAsync`, `RenewLeaseAsync`, `IsCancellationRequestedAsync`; its public constructor makes a detached context for tests. +- CRON: `.Jobs.AddCronJob("0 */6 * * *", o => ...)` with `CronJobOptions` (`Scope` Global/PerNode, `Overlap`, `MisfireWindow`, `MaxRetries`, `TimeZone`, typed `Arguments`). Scheduled automatically when the runtime pump starts. Tune the pump with `.Jobs.ConfigureRuntimePump(o => ...)` (`JobRuntimePumpOptions`: `Enabled`, `PollInterval`, `BatchSize`, `MaxJobAttempts`, `WorkerConcurrency`). +- Stable wire names: `.Messaging.RegisterMessageType("name")` and `.Jobs.Register("name")` so persisted discriminators survive assembly/namespace moves; unregistered types fall back to `Type.FullName`. +- Legacy APIs still ship under the `Foundatio.Messaging.Legacy` (old `IMessageBus`/`InMemoryMessageBus`) and `Foundatio.Jobs.Legacy` (`JobBase`, `QueueJobBase`, `JobWithLockBase`, `JobRunner`) namespaces for migration. Prefer the current API in new code. ## Core Interfaces | Interface | Purpose | In-Memory | Production | | --------- | ------- | --------- | ---------- | | `ICacheClient` | Key-value caching with TTL | `InMemoryCacheClient` | Redis, Hybrid | -| `IQueue` | FIFO message queuing | `InMemoryQueue` | Redis, SQS, Azure | -| `IMessageBus` | Pub/sub messaging | `InMemoryMessageBus` | Redis, Kafka, RabbitMQ, Azure | +| `IMessageBus` | Commands (`SendAsync`) + events (`PublishAsync`) over one client | `InMemoryMessageTransport` | Redis Streams, AWS SQS/SNS | +| `IJobClient` / `IJobMonitor` | Submit and observe durable background jobs | `InMemoryJobRuntimeStore` | `RedisJobRuntimeStore` | +| `IQueue` | Work-item queue (classic API) | `InMemoryQueue` | Redis, SQS, Azure | | `IFileStorage` | File storage abstraction | `InMemoryFileStorage` | S3, Azure Blob, Minio | | `ILockProvider` | Distributed locking | `CacheLockProvider` | Redis-backed | -| `IJob` | Background job processing | N/A | Hosted services | | `ISerializer` / `ITextSerializer` | Binary and text serialization | `SystemTextJsonSerializer` | MessagePack, JsonNet | | `IResiliencePolicy` | Retry, circuit breaker, timeout | `ResiliencePolicyBuilder` | N/A | ## DI Registration -All services are **singletons** (maintain internal state/connections). Jobs are scoped. +Use the `AddFoundatio()` fluent builder; infrastructure services register as **singletons**. Handlers and jobs resolve in their own DI scope per message/run, so they can inject scoped dependencies. ```csharp var builder = WebApplication.CreateBuilder(args); -// Quick start -- all in-memory defaults -builder.Services.AddFoundatio(); - -// Or register individually with options -builder.Services.AddSingleton(sp => - new InMemoryCacheClient(o => o.MaxItems(1000) - .LoggerFactory(sp.GetRequiredService()))); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton>(sp => - new InMemoryQueue()); - -// Lock provider (message bus optional but enables faster lock release via pub/sub) -builder.Services.AddSingleton(sp => - new CacheLockProvider( - sp.GetRequiredService(), - sp.GetService(), - sp.GetService(), - sp.GetService())); +builder.Services.AddFoundatio() + .Caching.UseInMemory() + .Storage.UseFolder("data") + .Locking.UseCache() + .Messaging + .ConfigureRouting(r => r + .UseServiceIdentity("billing") + .MapQueue("orders") + .MapTopic("order-events", typeof(IOrderEvent))) + .ConfigureRetry(p => p with { MaxAttempts = 5 }) + .UseInMemory() + .Messaging.AddHandler() + .Jobs.UseInMemoryRuntime() + .Jobs.Register("search.rebuild"); ``` -Swap to production by changing only DI registration: +Swap to production by changing only the provider lines: ```csharp -var redis = await ConnectionMultiplexer.ConnectAsync("localhost:6379"); +builder.Services.AddFoundatio() + .Messaging.UseRedis(connectionString: "localhost:6379") // Redis Streams transport + .Jobs.UseRedis(); // Redis job runtime store -builder.Services.AddSingleton(sp => - new RedisCacheClient(o => - { - o.ConnectionMultiplexer = redis; - o.LoggerFactory = sp.GetRequiredService(); - })); -builder.Services.AddSingleton(sp => - new RedisMessageBus(o => - { - o.Subscriber = redis.GetSubscriber(); - o.LoggerFactory = sp.GetRequiredService(); - })); -builder.Services.AddSingleton>(sp => - new RedisQueue(o => - { - o.ConnectionMultiplexer = redis; - o.LoggerFactory = sp.GetRequiredService(); - })); +// or AWS (SQS queues, SNS+SQS pub/sub; point ServiceUrl at LocalStack for local dev) +builder.Services.AddFoundatio() + .Messaging.UseAws(o => o.ResourcePrefix = "myapp"); ``` +Custom providers plug in via `.Messaging.UseTransport(...)` (any `IMessageTransport`) and `.Jobs.UseRuntimeStore(...)` (any `IJobRuntimeStore`). + ## Usage Patterns ### Caching @@ -117,29 +103,39 @@ await _cache.IncrementAsync("requests:today", 1); await _cache.RemoveByPrefixAsync("user:"); ``` -### Queues +### Messaging + +The verb carries the delivery semantic; handlers never choose queue vs. topic: ```csharp -await _queue.EnqueueAsync(new OrderWorkItem { OrderId = orderId }); +// Command: exactly one handler instance across the fleet processes it. +await _bus.SendAsync(new ResizeImage(imageId)); -var entry = await _queue.DequeueAsync(TimeSpan.FromSeconds(5)); -if (entry is not null) +// Event: every subscribing service receives one copy. +await _bus.PublishAsync(new OrderSubmitted(orderId)); +``` + +```csharp +public class SendConfirmationHandler : IMessageHandler { - await ProcessAsync(entry.Value); - await entry.CompleteAsync(); // success - // or: await entry.AbandonAsync(); // retry later + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) + => _email.SendConfirmationAsync(context.Message.OrderId, cancellationToken); } + +services.AddFoundatio() + .Messaging.AddHandler(o => + { + o.MaxConcurrency = 4; // default 1 preserves per-handler ordering + o.DeadLetterOn(); // retries cannot fix validation failures + }); ``` -### Messaging (Pub/Sub) +Throwing from `HandleAsync` triggers the core retry/dead-letter policy. With `AckMode.Manual`, settle explicitly: ```csharp -await _messageBus.SubscribeAsync(async (msg, ct) => -{ - await HandleOrderCreatedAsync(msg, ct); -}); - -await _messageBus.PublishAsync(new OrderCreated { OrderId = orderId }); +await context.CompleteAsync(); +await context.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.FromSeconds(30) }); +await context.RejectAsync(new RejectOptions { Terminal = true, Reason = "malformed" }); ``` ### File Storage @@ -155,7 +151,8 @@ await _storage.DeleteFilesAsync("reports/old-*"); ### Distributed Locks ```csharp -await using var lck = await _locker.AcquireAsync( +// TryAcquireAsync returns null when the lock is unavailable; AcquireAsync throws instead. +await using var lck = await _locker.TryAcquireAsync( "resource:order-123", timeUntilExpires: TimeSpan.FromMinutes(1)); @@ -183,146 +180,108 @@ await policy.ExecuteAsync(async ct => ## Jobs -### Standard Job +### Durable Job -```csharp -public class CleanupJob : JobBase -{ - public CleanupJob( - TimeProvider timeProvider, - IResiliencePolicyProvider resiliencePolicyProvider, - ILoggerFactory loggerFactory = null) - : base(timeProvider, resiliencePolicyProvider, loggerFactory) { } - - protected override async Task RunInternalAsync(JobContext context) - { - await CleanupOldRecordsAsync(context.CancellationToken); - return JobResult.Success; - } -} -``` - -### Job with Lock (Singleton / Leader Election) - -`JobWithLockBase` acquires a distributed lock before each run. If the lock isn't available the run is cancelled. Implements `IJobWithOptions`. +Implement `IJob`; enqueue through `IJobClient`. Arguments are typed and persisted with the job: ```csharp -[Job(Description = "Singleton maintenance", Interval = "5s")] -public class MaintenanceJob : JobWithLockBase +public class RebuildSearchIndexJob : IJob { - private readonly ILockProvider _lockProvider; - - public MaintenanceJob( - ICacheClient cache, IMessageBus messageBus, - TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, - ILoggerFactory loggerFactory) : base(timeProvider, resiliencePolicyProvider, loggerFactory) + public async Task RunAsync(JobExecutionContext context) { - _lockProvider = new CacheLockProvider(cache, messageBus, loggerFactory); - } + var args = context.GetArguments(); - // new CancellationToken(true) = try once, skip if lock is held - protected override Task GetLockAsync(CancellationToken cancellationToken) => - _lockProvider.AcquireAsync(nameof(MaintenanceJob), TimeSpan.FromMinutes(15), - cancellationToken: new CancellationToken(true)); + await context.ReportProgressAsync(10, "starting"); + foreach (var batch in GetBatches(args.Index)) + { + if (await context.IsCancellationRequestedAsync()) + return JobResult.Cancelled; + + await IndexBatchAsync(batch, context.CancellationToken); + await context.RenewLeaseAsync(); // heartbeat for long runs + } - protected override async Task RunInternalAsync(JobContext context) - { - await DoMaintenanceAsync(context.CancellationToken); return JobResult.Success; } } + +JobHandle handle = await _jobs.EnqueueAsync( + new RebuildSearchIndexArgs { Index = "orders" }); +JobState? state = await handle.GetStateAsync(); +await handle.RequestCancellationAsync(); ``` -### Queue Processor Job +The worker gives every run its own DI scope, claims jobs with compare-and-set transitions (no double-runs), and supervises the lease: a run is cancelled when the lease is lost to another node or renewal keeps failing past the lease window. Crashed runs are reclaimed and retried until the attempt budget (`JobRuntimePumpOptions.MaxJobAttempts`, default 3) is exhausted, then dead-lettered. + +### CRON Job ```csharp -public class OrderProcessorJob : QueueJobBase -{ - public OrderProcessorJob( - IQueue queue, - TimeProvider timeProvider, - IResiliencePolicyProvider resiliencePolicyProvider, - ILoggerFactory loggerFactory = null) - : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { } - - protected override async Task ProcessQueueEntryAsync( - QueueEntryContext context) +services.AddFoundatio() + .Jobs.UseInMemoryRuntime() + .Jobs.AddCronJob("0 2 * * *", o => { - var item = context.QueueEntry.Value; - await ProcessOrderAsync(item.OrderId, context.CancellationToken); - return JobResult.Success; - } -} + o.Scope = ScheduledJobScope.Global; // one instance per tick (default); PerNode = every instance + o.MaxRetries = 3; + o.Arguments = new ExportArgs { Format = "csv" }; + }); ``` -### Hosting Integration +Occurrences are materialized durably through the runtime store (deduplicated across nodes and misfire windows) and executed by the auto-registered `JobRuntimePumpService`. -Requires `Foundatio.Extensions.Hosting` package: +### Legacy Jobs -```csharp -builder.Services.AddJob(o => o.WaitForStartupActions()); -builder.Services.AddCronJob("0 */6 * * *"); -builder.Services.AddDistributedCronJob("0 */6 * * *"); -``` +`JobBase`, `QueueJobBase`, and `JobWithLockBase` live in `Foundatio.Jobs.Legacy` (hosted via `Foundatio.Extensions.Hosting`'s `AddJob` / `AddCronJob` / `AddDistributedCronJob`). They still work but are the previous model; prefer `IJob` + the durable runtime for new code. ## Testing -Use `Foundatio.Xunit.v3` for test logging and DI integration. Two base classes: +### Messaging: Foundatio.Testing harness -- **`TestWithLoggingBase`** -- lightweight, no DI container. `_logger` (`ILogger`) for logging; `Log` (`ILoggerFactory`) for passing to Foundatio services. -- **`TestLoggerBase`** -- full DI via `TestLoggerFixture`. Override `ConfigureServices` to register services. `Log` (`ILogger`) for logging; `TestLogger` (`ILoggerFactory`) for passing to Foundatio services. +`Foundatio.Testing` runs the real `IMessageBus` over a recording in-memory transport -- deterministic tests without sleeps, including the retry/dead-letter path: ```csharp -using Foundatio.Caching; -using Foundatio.Xunit; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Xunit; +services.AddFoundatio() + .Messaging.UseTestHarness() + .Messaging.AddHandler(); -public class OrderServiceTests : TestLoggerBase -{ - public OrderServiceTests(ITestOutputHelper output, TestLoggerFixture fixture) - : base(output, fixture) { } +// resolve MessagingTestHarness from the container; start hosted services, then: +await bus.PublishAsync(new OrderPlaced(42)); +await harness.WaitForIdleAsync(); // blocks until queues and in-flight handlers drain - protected override void ConfigureServices(IServiceCollection services) - { - services.AddSingleton(sp => - new InMemoryCacheClient(o => o.LoggerFactory(TestLogger))); - services.AddSingleton(); - } +Assert.Single(harness.Published()); +Assert.Single(harness.Handled()); +Assert.Empty(harness.DeadLetteredMessages); +``` - [Fact] - public async Task GetStatusAsync_WithCachedOrder_ReturnsCachedStatus() - { - // Arrange - var cache = Services.GetRequiredService(); - await cache.SetAsync("order:123", "shipped"); - Log.LogInformation("Seeded cache with order status"); +Recordings: `SentMessages` / `PublishedMessages` / `HandledMessages` / `AbandonedMessages` (retries) / `DeadLetteredMessages`, with typed accessors `Sent()`, `Published()`, `Handled()`, `Abandoned()`, `DeadLettered()`. - // Act - var status = await Services.GetRequiredService() - .GetStatusAsync("123"); +For jobs, `new JobExecutionContext(cancellationToken, arguments: myArgs)` builds a detached context to run an `IJob` directly -- progress/lease helpers no-op and `GetArguments()` returns the supplied object. - // Assert - Assert.Equal("shipped", status); - } -} -``` +### Test logging via Foundatio.Xunit.v3 + +Two base classes: + +- **`TestWithLoggingBase`** -- lightweight, no DI container. `_logger` (`ILogger`) for logging; `Log` (`ILoggerFactory`) for passing to Foundatio services. +- **`TestLoggerBase`** -- full DI via `TestLoggerFixture`. Override `ConfigureServices` to register services. `Log` (`ILogger`) for logging; `TestLogger` (`ILoggerFactory`) for passing to Foundatio services. + +### Custom providers + +Validate a custom transport or job store against the shared conformance suites in `Foundatio.TestHarness`: inherit `MessageTransportConformanceTests` (override `CreateTransport`) and `JobRuntimeStoreConformanceTests` (override `CreateStore`). Tests skip automatically for unimplemented optional interfaces or unavailable backends. ## Gotchas -- **Lock returns null**: `TryAcquireAsync` returns `null` when the lock cannot be acquired -- always guard with `is not null` before doing work. `AcquireAsync` throws `LockAcquisitionTimeoutException` instead of returning null. +- **Handlers registered per class get their own event copy**: `AddHandler` defaults the `SubscriptionQualifier` to the handler type name, so two handler classes on one event type EACH receive every published message. Set an explicit shared `Subscription` only when they should compete. +- **Shared subscription keys compare delegates by identity**: subscriptions sharing a `Key` must pass the SAME `RedeliveryBackoff`/`DeadLetterWhen` delegate instances -- a lambda recreated per subscription is rejected as a conflicting registration. +- **Do not configure broker redrive policies**: the core owns retry/dead-lettering (SQS `maxReceiveCount`, DLX, etc. would split authority and make behavior transport-specific). +- **A runtime store needs its pump**: the DI builder auto-registers `JobRuntimePumpService` with any runtime store, but in a non-hosted process (no generic host) nothing starts it -- drive `JobScheduleProcessor`/`IJobWorker` manually or nothing drains. +- **Delayed sends beyond transport ceilings need a runtime store**: e.g. > 15 min on SQS, or any delayed publish on SNS topics. Without a store the operation fails loudly rather than truncating the delay. +- **`WaitForIdleAsync` ignores store-parked work**: delayed sends/retries parked in the runtime store are not transport activity -- drain them via the job schedule processor before asserting. +- **Lock returns null**: `TryAcquireAsync` returns `null` when the lock cannot be acquired -- always guard with `is not null`. `AcquireAsync` throws `LockAcquisitionTimeoutException` instead of returning null. - **Dispose streams and locks**: `ILock` is `IAsyncDisposable` -- use `await using`. Streams from `GetFileStreamAsync` are `IDisposable` -- use `using var`. -- **Cache TTL floor**: Expiration values below 5ms are treated as already-expired and the key is silently removed. If you compute TTL dynamically (e.g., `expiresAt - now`), guard against near-zero values. -- **Cache `GetAsync` returns `CacheValue`**: Check `result.HasValue` before accessing `result.Value`. A missing key returns `HasValue = false`, not an exception. -- **Cache stampede (thundering herd)**: The cache-aside pattern (`Get` -> miss -> load -> `Set`) is vulnerable to stampedes when a popular key expires and many callers regenerate simultaneously. Use `CacheLockProvider` to serialize regeneration: acquire a lock keyed on the cache key, double-check the cache after acquiring, and only then call the backing store. See the [Cache Stampede Protection](https://foundatio.readthedocs.io/guide/caching.html#cache-stampede-protection) docs for the full pattern. -- **Queue auto-complete**: `QueueJobBase` auto-completes entries based on `JobResult` by default. Set `AutoComplete = false` only when you need manual `CompleteAsync()`/`AbandonAsync()` control. Manual `DequeueAsync` does NOT auto-complete. -- **GetQueueEntryLockAsync error handling**: If `GetQueueEntryLockAsync` returns `null`, the queue entry is abandoned. If it throws, the entry is also abandoned and a `JobResult.FromException` is returned. Use `TryAcquireAsync` (not `AcquireAsync`) in your override since the return type is `Task`. -- **JobWithLockBase vs manual locking**: Use `JobWithLockBase` when the entire run must be single-instance (leader election). Use manual `ILockProvider.AcquireAsync` inside `JobBase` for finer-grained locking within a job. -- **JobContext.RenewLockAsync**: Call in long-running jobs (both `JobBase` and `QueueJobBase`) to prevent lock expiration mid-processing. -- **Register as singletons**: All infrastructure services (`ICacheClient`, `IMessageBus`, `IQueue`, `IFileStorage`, `ILockProvider`) maintain internal state and connections -- always register as singletons. -- **CacheLockProvider + IMessageBus**: `IMessageBus` is optional but recommended. Without it, lock release falls back to polling. With it, locks are released instantly via pub/sub notification. -- **In-memory for tests**: All in-memory implementations are functionally equivalent to production providers. Swap via DI for fast, isolated unit tests with no external dependencies. +- **Cache `GetAsync` returns `CacheValue`**: check `result.HasValue` before `result.Value`. A missing key returns `HasValue = false`, not an exception. +- **Cache stampede**: serialize regeneration of hot keys with `CacheLockProvider` (lock on the cache key, double-check after acquiring). See the [Cache Stampede Protection](https://foundatio.readthedocs.io/guide/caching.html#cache-stampede-protection) docs. +- **Register as singletons**: infrastructure services (`ICacheClient`, `IMessageBus`, `IFileStorage`, `ILockProvider`) maintain internal state and connections; the `AddFoundatio()` builder does this for you. +- **In-memory for tests**: in-memory implementations are functionally equivalent to production providers and run the same conformance suites -- swap via DI for fast, isolated tests. +- **Legacy name collisions during migration**: old and new APIs coexist (`Foundatio.Messaging.Legacy.IMessageBus` vs `Foundatio.Messaging.IMessageBus`; `Foundatio.Jobs.Legacy.IJob` vs `Foundatio.Jobs.IJob`). Disambiguate with a `using` alias in files that reference both namespaces. ## NuGet Packages @@ -330,8 +289,8 @@ public class OrderServiceTests : TestLoggerBase | Package | Provides | | ------- | -------- | -| `Foundatio` | Core interfaces, in-memory implementations, resilience, `SystemTextJsonSerializer` | -| `Foundatio.Extensions.Hosting` | `AddJob`, `AddCronJob`, `AddDistributedCronJob`, startup actions, hosted services | +| `Foundatio` | Core interfaces, in-memory implementations, messaging + durable job runtime, resilience, `SystemTextJsonSerializer` | +| `Foundatio.Extensions.Hosting` | `AddJobRuntimeService`, startup actions, legacy `AddJob`/`AddCronJob`/`AddDistributedCronJob` | ### Serializers @@ -347,10 +306,10 @@ public class OrderServiceTests : TestLoggerBase | Package | Provides | | ------- | -------- | -| `Foundatio.Redis` | Redis cache, queue, messaging, locks, storage | +| `Foundatio.Redis` | `RedisStreamsMessageTransport` (messaging), `RedisJobRuntimeStore` (jobs), plus Redis cache/queue/lock/storage | +| `Foundatio.Aws` | `AwsMessageTransport` (SQS queues, SNS+SQS pub/sub), S3 storage | | `Foundatio.AzureStorage` | Azure Blob storage, Azure Storage queues | | `Foundatio.AzureServiceBus` | Azure Service Bus queues + messaging | -| `Foundatio.AWS` | SQS queues, SQS messaging, S3 storage | | `Foundatio.Kafka` | Kafka messaging | | `Foundatio.RabbitMQ` | RabbitMQ messaging | | `Foundatio.Minio` | MinIO S3-compatible storage | @@ -361,7 +320,8 @@ public class OrderServiceTests : TestLoggerBase | Package | Provides | | ------- | -------- | -| `Foundatio.TestHarness` | Shared test base classes for validating custom implementations | +| `Foundatio.Testing` | `MessagingTestHarness` + `UseTestHarness()` recording transport for deterministic messaging tests | +| `Foundatio.TestHarness` | Conformance suites (`MessageTransportConformanceTests`, `JobRuntimeStoreConformanceTests`) for custom providers | | `Foundatio.Xunit` | xUnit v2 test logging, retry attributes | | `Foundatio.Xunit.v3` | xUnit v3 test logging, retry attributes | | `Foundatio.DataProtection` | ASP.NET Core Data Protection key storage via `IFileStorage` | diff --git a/docs/guide/messaging-jobs-redesign.md b/docs/guide/messaging-jobs-redesign.md index ce0e1b169..98d163e15 100644 --- a/docs/guide/messaging-jobs-redesign.md +++ b/docs/guide/messaging-jobs-redesign.md @@ -1,301 +1,224 @@ # Messaging and Jobs Redesign -The new messaging API is app-facing and type-driven. Queue, pub/sub, received-message, headers, options, and common message abstractions live under `Foundatio.Messaging`; folders may separate queue, pub/sub, and provider contracts, but application code should start with `IQueue` and `IPubSub`. +The redesigned messaging API is one client — `IMessageBus` in `Foundatio.Messaging` — with two verbs. The caller's verb carries the delivery semantic, and handlers are registered without any topology decision: -Provider-facing transport contracts such as `IMessageTransport`, `ISupportsPull`, `ISupportsPush`, and `ISupportsDeadLetter` are still public so external providers can implement them. They are infrastructure contracts, not the primary application surface. - -## The core owns behavior; transports stay simple - -The division of responsibility is deliberate: **the core owns behavior, transports stay thin.** A transport is bytes in, bytes out plus a few primitives — send, receive, complete, abandon, and (optionally) a dead-letter sink. Everything that defines *how messaging behaves* — serialization, content types, routing, multi-type dispatch, priority, back-pressure, tracing, metrics, and especially **retry and dead-lettering** — lives in the core and is therefore identical across every transport. A provider only advertises which primitives it supports through small capability interfaces (`ISupportsPull`, `ISupportsPush`, `ISupportsDeadLetter`, `ISupportsDelayedDelivery`, `ISupportsRedeliveryDelay`, …); it never owns policy. - -This keeps providers small and hard to get subtly wrong, and keeps behavior portable: code verified against the in-memory transport behaves the same on a real broker. It also avoids a split-brain retry model — there is exactly one authority (the core), never a tug-of-war between the core's `MaxAttempts` and a broker-native redrive policy. See [Retry and dead-lettering](#retry-and-dead-lettering). - -## Setup - -Register the in-memory messaging transport, central routing policy, and durable job runtime through DI: +- `SendAsync` — a **command** / unit of work: exactly one handler instance across the fleet processes it (competing consumers). +- `PublishAsync` — an **event**: every subscribing service receives one copy (a scaled service's instances compete for it), or every instance when the subscription opts into `PerInstance`. ```csharp -services.AddFoundatio() - .Messaging.ConfigureRouting(r => r - .MapQueue("orders") - .MapTopic("order-events", typeof(IOrderEvent)) - .UseSubscriptionIdentity("billing-service")) - .UseInMemory() - .Jobs.UseInMemoryRuntime() - .Jobs.Register("search.rebuild"); +await bus.SendAsync(new ResizeImage(id)); // one handler instance, somewhere, does the work +await bus.PublishAsync(new OrderSubmitted(id)); // every subscribing service hears about it ``` -Application code should depend on `Foundatio.Messaging.IQueue`, `IPubSub`, `IJobClient`, `IJobMonitor`, and `IJobWorker` instead of constructing `InMemoryMessageTransport`, `MessageQueue`, `PubSub`, or `JobClient` directly. Deployment or admin code can depend on `IMessageTopology` to inspect, create, or validate the destinations implied by routing configuration. +`SendBatchAsync` and `PublishBatchAsync` batch both verbs; the non-generic `IEnumerable` overloads accept heterogeneous batches and group by resolved route. Per-operation options are `MessageSendOptions` and `MessagePublishOptions` (priority, delay/`DeliverAt`, TTL, correlation id, headers, and a `Destination`/`Topic` override as the escape hatch). -## Queue +The legacy publish/subscribe `IMessageBus` and job APIs remain shipped under the `Foundatio.Messaging.Legacy` and `Foundatio.Jobs.Legacy` namespaces while consumers migrate. -The default queue model is send or receive this message type: - -```csharp -await queue.EnqueueAsync(new OrderSubmitted(id)); - -IReceivedMessage? received = await queue.ReceiveAsync(); -``` - -Destination and source are advanced operation overrides: +## The core owns behavior; transports stay simple -```csharp -await queue.EnqueueAsync(message, new QueueMessageOptions { - Destination = "orders-high-priority" -}); +The division of responsibility is deliberate: **the core owns behavior, transports stay thin.** A transport is bytes in, bytes out plus a few primitives — `IMessageTransport` is `SendAsync`, `CompleteAsync`, `AbandonAsync`, and small opt-in operation interfaces (`ISupportsPull`, `ISupportsPush`, `ISupportsDeadLetter`, `ISupportsRedeliveryDelay`, `ISupportsVisibilityTimeout`, `ISupportsLockRenewal`, `ISupportsStats`, `ISupportsProvisioning`). Everything that defines *how messaging behaves* — serialization, routing, multi-type dispatch, settlement, scheduling, and especially **retry and dead-lettering** — lives in the core and is therefore identical across every transport. There is exactly one retry authority (the core), never a tug-of-war between core policy and a broker-native redrive policy. -IReceivedMessage? received = await queue.ReceiveAsync(new QueueReceiveOptions { - Source = "orders-high-priority" -}); -``` +Every transport API takes the same canonical identity: `DestinationAddress` (`Name`, `Role` — `Queue`/`Topic`/`Subscription`/`Binding` — and, for subscriptions, the owning `Topic`; created via `ForQueue`/`ForTopic`/`ForSubscription`). `Key` is its opaque string form (`"{topic}/{name}"` for subscriptions), so the same logical destination can never be spelled two ways on the send path versus the provisioning path. -Grouped or default queues can be consumed through the raw envelope path: +Facts a transport advertises are **role-aware**: the core asks `ITransportInfo.GetCapabilities(DestinationRole)` and gets a `TransportCapabilities` record (`DelayedDelivery`, `MaxDeliveryDelay`, `Priority`, `Expiration`, `Ordering`, `MaxBatchSize`, `MaxMessageBytes`). Capabilities genuinely differ by role on real brokers — the AWS transport's queue role has a native 15-minute `MaxDeliveryDelay` (SQS `DelaySeconds`) while its topic role has no native delay at all. Anything not advertised is treated as unsupported: the core validates, falls back to the runtime store, or fails loudly — a broker never silently drops a requested behavior. -```csharp -IReceivedMessage? received = await queue.ReceiveAsync(new QueueReceiveOptions { - RouteType = typeof(IOrderMessage) -}); - -await queue.EnqueueBatchAsync(new object[] { - new OrderSubmitted(id), - new OrderCancelled(id) -}); -``` - -Consumers return handles and do not block unexpectedly: +## Setup ```csharp -await using IMessageConsumer consumer = await queue.StartConsumerAsync(HandleAsync); +services.AddFoundatio() + .Messaging + .ConfigureRouting(r => r + .UseServiceIdentity("billing") + .MapQueue("orders") + .MapTopic("order-events", typeof(IOrderEvent))) + .ConfigureRetry(p => p with { MaxAttempts = 5 }) + .ConfigureTopology(TopologyMode.Ensure) + .RegisterMessageType("order.submitted") + .UseInMemory() + .Messaging.AddHandler() + .Jobs.UseInMemoryRuntime() + .Jobs.Register("search.rebuild"); ``` -Use `RunConsumerAsync` when the desired behavior is a blocking lifetime loop. Starting the same consumer key with the same handler and options is idempotent; starting the same key with conflicting handler/options throws. +Swap providers by swapping one line: `.Messaging.UseRedis()` (Redis Streams), `.Messaging.UseAws()` (SQS/SNS), `.Jobs.UseRedis()`, or `.Messaging.UseTransport(...)` / `.Jobs.UseRuntimeStore(...)` for anything custom. Application code depends on `IMessageBus`, `IJobClient`, and `IJobMonitor`; deployment or admin code can depend on `IMessageTopology`. -### Multiple message types on one destination +`RegisterMessageType(name)` gives a type a stable wire discriminator so payloads survive assembly/namespace moves; unregistered types fall back to `Type.FullName` (never `AssemblyQualifiedName`). `.Jobs.Register(name)` does the same for persisted job types. -Several message types can share a single destination. Start one typed consumer per type; they attach to a single underlying receive loop that dispatches each message to the consumer registered for its type (read from the `message.type` header): +## Handlers -```csharp -await using var submitted = await queue.StartConsumerAsync(HandleSubmittedAsync); -await using var cancelled = await queue.StartConsumerAsync(HandleCancelledAsync); -// One loop on the shared destination. OrderSubmitted is dispatched to the first handler, OrderCancelled to the second. -``` - -Consumers that share a message type compete: each message is dispatched to one of them, round-robin. All consumers on one destination must agree on `MaxConcurrency` (it is a property of the shared loop). A message whose type has **no** registered consumer on this node is handled loudly — see [Unmatched message types](#unmatched-message-types). - -A consumer whose route type is an interface or base type is a **grouped** consumer and receives the concrete payload (assignable to that type), not raw bytes: +Handlers are topology-free. A handler implements `IMessageHandler` and is registered declaratively; it never decides queue-vs-topic — the sender's verb does: ```csharp -await using var all = await queue.StartConsumerAsync(HandleAnyAsync); -// HandleAnyAsync receives IReceivedMessage whose Message is the concrete OrderSubmitted / OrderCancelled. -``` +public class SendConfirmationHandler : IMessageHandler +{ + public Task HandleAsync(IMessageContext context, CancellationToken cancellationToken) + => _email.SendConfirmationAsync(context.Message.OrderId, cancellationToken); +} -The concrete type is resolved from the `message.type` header through `IMessageTypeRegistry` and deserialized as the actual payload type. The registry is the stable wire discriminator in both directions — register stable names for types that may move between assemblies/namespaces; unregistered types fall back to `Type.FullName` (never `AssemblyQualifiedName`): - -```csharp services.AddFoundatio() - .Messaging.RegisterMessageType("order.submitted"); + .Messaging.AddHandler(o => + { + o.MaxConcurrency = 4; + o.DeadLetterOn(); + }); ``` -The raw-envelope path (non-generic `ReceiveAsync(new QueueReceiveOptions { RouteType = ... })`) remains for callers that want the bytes without deserialization. - -## Pub/Sub - -Pub/sub follows the same type-driven publishing pattern: - -```csharp -await pubsub.PublishAsync(new OrderSubmitted(id)); - -await using IMessageSubscription subscription = await pubsub.SubscribeAsync(HandleAsync); -``` +Each message is dispatched to the handler in its own DI scope (scoped dependencies work), and a single auto-registered hosted service (`MessageHandlerHostedService`) starts every registered handler at app start and disposes them at shutdown. Throwing from `HandleAsync` triggers the core retry/dead-letter policy. Each handler class defaults to its own subscriber group (the `SubscriptionQualifier` is set to the handler type name), so every handler registered for an event type receives its own copy of each published message. -Topic routing and subscription identity are separate. The topic answers where the event is published. The subscription answers which logical service or consumer group receives it. Multiple instances using the same subscription compete on the same transport subscription; different subscriptions on the same topic receive fan-out copies: +For dynamic subscriptions, `IMessageBus.SubscribeAsync(handler, options)` returns an `IMessageSubscription` handle (`Key`, `Destination`, `Topic`, `Subscription`, `Source`); disposing it detaches the handler. -```csharp -services.AddFoundatio() - .Messaging.ConfigureRouting(r => r - .MapTopic("order-events", typeof(IOrderEvent)) - .UseSubscriptionIdentity("billing-service")); -``` +### Subscription options -Advanced operation overrides remain available: +A subscription listens on the type's two delivery channels — sent commands and published events — and `MessageSubscriptionOptions` declares its intent: -```csharp -await pubsub.PublishAsync(message, new PubSubMessageOptions { - Topic = "order-events-replay" -}); - -await using IMessageSubscription subscription = await pubsub.SubscribeAsync( - HandleAsync, - new PubSubSubscriptionOptions { - Topic = "order-events-replay", - Subscription = "billing-replay" - }); -``` +- **`Deliveries`** — `MessageDeliveries.Sent`, `Published`, or `Both` (default). A handler that only ever consumes commands (or only events) states that so no idle listener is wired — and so a queue-only or topic-only transport can serve it. The default `Both` quietly narrows to what the transport supports; explicitly requesting a single channel the transport cannot serve throws `NotSupportedException`. +- **`Subscription`** — the subscriber-group identity for published messages. Defaults to the service identity, so all instances of a service share one subscription and compete. **`SubscriptionQualifier`** distinguishes groups within one service (`"{service-identity}.{qualifier}"`). **`PerInstance`** gives every running instance its own unique subscription (cache invalidation, config reload) and is mutually exclusive with `Subscription`. +- **`MaxConcurrency`** — messages processed concurrently per instance. Default 1: the only default that preserves per-handler ordering, and each handler already gets its own concurrent stream (10 handlers = 10 parallel consumers). Raise it for I/O-bound, order-agnostic handlers. +- **`MaxAttempts`**, **`RedeliveryBackoff`**, **`DeadLetterWhen`** — per-subscription retry overrides; null inherits the default `RetryPolicy`. **`DeadLetterOn()`** is the by-type shorthand for `DeadLetterWhen` and composes (call once per exception type). +- **`AckMode`** — `Auto` (default) or `Manual`. +- **`RouteType`**, **`Destination`**, **`Topic`** — grouped/interface consumption and per-subscription route overrides. +- **`Key`** — consumer identity. Subscriptions sharing a `Key` on the same channel form one competing group and must configure identical failure policies; the backoff/`DeadLetterWhen` **delegates are compared by identity**, so share the same delegate instances — a lambda recreated per subscription is rejected as a conflicting registration. -`PubSubMessageOptions` mirrors queue send options where concepts overlap: priority, delay, TTL, correlation id, deduplication id, headers, and topic override. `PubSubSubscriptionOptions.Key` is only the local duplicate-listener key; `Subscription` is the transport consumer group identity. `PublishBatchAsync(IEnumerable)` supports heterogeneous event batches and groups sends by resolved topic. +Delivery semantics are never invisible: each subscription logs its effective topology (destination, subscriber group, concurrency, retry posture) once at subscribe time. -## Routing +## Routing and topology -Default route precedence is: +`IMessageRouter` resolves the queue destination and topic for a message type. The default router's precedence: ```text -operation override > explicit route map > interface/base-type map > MessageRouteAttribute > configured default/convention -``` - -`IMessageRouter` is shared by queues and pub/sub. Configure routes once with `MessageRoutingOptionsBuilder`: - -```csharp -services.AddFoundatio() - .Messaging.ConfigureRouting(r => r - .UseDefaultQueue("all-work") - .UseDefaultTopic("all-events") - .MapQueue("orders") - .MapQueue("orders", typeof(OrderSubmitted), typeof(OrderCancelled)) - .MapQueue("order-work", typeof(IOrderMessage)) - .MapTopic("order-events", typeof(IOrderEvent)) - .UseConvention(ctx => $"app-{ctx.MessageType.Name.ToLowerInvariant()}")); +operation override > exact type map > interface/base-type map > MessageRouteAttribute > configured default > convention > kebab-cased type name ``` -`QueueMessageOptions.Destination`, `QueueReceiveOptions.Source`, `PubSubMessageOptions.Topic`, and `PubSubSubscriptionOptions.Topic`/`Subscription` are final escape hatches for one operation. Attribute routing remains available for type-local defaults, but central routing should be the normal path. +Configure routes once with `ConfigureRouting` (a `MessageRoutingOptionsBuilder`): `UseDefaultQueue`, `UseDefaultTopic`, `MapQueue` / `MapQueue(destination, params Type[])`, `MapTopic` / `MapTopic(topic, params Type[])`, `UseServiceIdentity`, `UseSubscriptionIdentity`, and `UseConvention`. `UseServiceIdentity` names the service (the default subscriber-group identity); when unset it falls back to the `FOUNDATIO_SUBSCRIPTION_ID` / `FOUNDATIO_SERVICE_ID` environment variables, then the kebab-cased app name. -Routing configuration is also the topology declaration source. `UseDefaultQueue`, `UseDefaultTopic`, `MapQueue`, and `MapTopic` declare the queue destinations or topics they name; `UseSubscriptionIdentity` declares subscriptions for configured topics. Operation-level overrides are intentionally not part of startup topology because they are exceptional one-off routes. +**Routing configuration is also the topology declaration source.** `UseDefaultQueue`, `UseDefaultTopic`, `MapQueue`, and `MapTopic` declare the destinations they name, and setting a service/subscription identity declares the subscription on each configured topic — as `DestinationDeclaration` values carrying the *same* canonical `DestinationAddress` the runtime later sends to and receives from, so provisioning and runtime can never disagree on identity. Per-operation overrides are deliberately excluded: they are exceptional one-off routes. ```csharp IMessageTopology topology = provider.GetRequiredService(); IReadOnlyList declarations = topology.GetDeclarations(); await topology.EnsureAsync(); // deploy/admin process with create permissions -await topology.ValidateAsync(); // app startup check without creating destinations +await topology.ValidateAsync(); // check-only; throws naming what is missing ``` -## Delivery Settlement +`TopologyMode` (via `ConfigureTopology`) governs how the client administers topology at runtime and at startup: + +- **`Ensure`** (default) — create missing destinations on first use, and the handler host ensures the declared topology before any handler starts consuming. +- **`Validate`** — never create; verify each destination exists and throw when missing. Startup fails at boot instead of surfacing as runtime send errors. +- **`None`** — no topology calls at all; everything is pre-provisioned out of band. + +The mode governs the core's provisioning calls; combine `Validate`/`None` with transport knobs such as `AwsMessageTransportOptions.AutoCreateDestinations = false` for a fully locked-down broker. -Received messages settle with two verbs — the same for queue and pub/sub: +## Delivery settlement + +Received messages surface as `IMessageContext` / `IMessageContext` (id, body, headers, correlation id, priority, `Attempts`) and settle with two verbs: ```csharp -await message.CompleteAsync(); // handled successfully -await message.RejectAsync(); // retry, transport-timed redelivery -await message.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.FromSeconds(30) }); // retry after a delay -await message.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation" }); // do not retry -await message.RenewLockAsync(); +await context.CompleteAsync(); // handled successfully +await context.RejectAsync(); // retry (redelivery) +await context.RejectAsync(new RejectOptions { RedeliveryDelay = TimeSpan.FromSeconds(30) }); +await context.RejectAsync(new RejectOptions { Terminal = true, Reason = "validation", Exception = ex }); +await context.RenewLockAsync(); // long handler heartbeat ``` -`RejectAsync` replaces the separate abandon and dead-letter verbs. A non-terminal reject returns the message for redelivery (optionally after `RedeliveryDelay`); `Terminal = true` means "never redeliver" and routes the message to the dead-letter sink, falling back to a configured destination or a drop (see below). - -Auto-ack is the default: a handler that returns without settling is completed automatically, and a handler that throws is rejected according to the [retry policy](#retry-and-dead-lettering). Manual ack is opt-in with `AckMode.Manual` on the consumer options. +A non-terminal reject returns the message for redelivery (optionally after `RedeliveryDelay`); `Terminal = true` means "never redeliver" and routes the message to the dead-letter sink with the `Reason` and `Exception` forensics attached. `RejectOptions.BestEffortDelay` lets a delay the transport cannot honor degrade to immediate redelivery instead of failing (the core's own retry policy uses best-effort delays; an explicit caller delay defaults to strict). -Unsupported capabilities fail clearly with `NotSupportedException` or a validation exception — there are no silent no-ops for lock renewal, priority, expiration, or delayed delivery. Terminal reject is the one deliberate exception: a transport with no dead-letter sink does not throw, it drops the message (at-most-once for terminal messages), which is the honest behavior for an ack-less/broadcast transport. +Auto-ack is the default: a handler that returns without settling is completed, and a handler that throws is rejected per the retry policy. Manual settlement is opt-in with `AckMode.Manual`. ## Retry and dead-lettering -The core owns retry and dead-lettering, so behavior is identical on every transport (see [The core owns behavior](#the-core-owns-behavior-transports-stay-simple)). A transport only has to redeliver an abandoned message and, optionally, expose a dead-letter sink; the core decides how many times to retry, how long to wait between attempts, and when to give up. The broker's own delivery count is used as a crash-safe attempt counter, so the core owns the *policy* without owning durable retry *state*. +The core owns retry and dead-lettering, so behavior is identical on every transport. A transport only redelivers abandoned messages and optionally exposes a dead-letter sink; the core decides how many times to retry, how long to wait, and when to give up — using the broker's own delivery count as the crash-safe attempt counter, so the core owns the *policy* without owning durable retry *state*. -Configure a default policy and override it per consumer: +The default `RetryPolicy`: `MaxAttempts` 5, and `RetryPolicy.DefaultBackoff` — an immediate first retry, then 10s/20s/30s (capped) with ±20% jitter, the delay shape mature messaging stacks converged on. Configure the default and override per subscription: ```csharp services.AddFoundatio() - .Messaging.ConfigureRetry(r => r with { + .Messaging.ConfigureRetry(p => p with + { MaxAttempts = 5, Backoff = attempt => TimeSpan.FromSeconds(Math.Min(30, Math.Pow(2, attempt))), - DeadLetterDestination = "orders-dead-letter" + DeadLetterWhen = ex => ex is ValidationException, // unrecoverable: dead-letter immediately + DeadLetterDestination = "orders-dead-letter" // null derives "{source}.deadletter" }); - -await queue.StartConsumerAsync(HandleAsync, new QueueConsumerOptions { - MaxAttempts = 10 // per-consumer override; null inherits the default policy -}); ``` -When a handler throws, the message is retried (abandoned for redelivery, with the configured backoff) until `MaxAttempts` is reached, then dead-lettered. Where a dead-lettered message lands, in order of preference: - -1. the transport's native dead-letter sink, when it has one (`ISupportsDeadLetter`) — preserving native DLQ tooling; -2. otherwise the configured `RetryPolicy.DeadLetterDestination`, which the core writes to directly (a normal queue on the same transport), recording the reason in the `message.dead_letter.reason` header; -3. otherwise the message is dropped (at-most-once) — the honest outcome when there is nowhere durable to park it. +Deserialization failures are always treated as unrecoverable. Where a dead-lettered message lands, in order of preference: the transport's native dead-letter sink (`ISupportsDeadLetter`), otherwise the configured or derived (`"{source}.deadletter"`) dead-letter destination written by the core. Dead-lettered messages carry forensics headers — `message.dead_letter.reason`, `.attempts`, `.exception_type`, `.exception_message`, `.exception_stack`, `.failed_at`, and `.original_destination` (`KnownHeaders.DeadLetter*`) — so a dead message is triageable with plain transport tooling, and `ISupportsDeadLetter.ReceiveDeadLetteredAsync` reads raw entries back (including poison payloads that never deserialized). -We deliberately do **not** configure broker-native redrive policies (SQS `maxReceiveCount`, Azure Service Bus `MaxDeliveryCount`, RabbitMQ DLX). That would split authority between the broker and the core and make behavior transport-specific. The core is always authoritative; transports stay simple. A destination's structural creation knobs, if any, are limited to `DestinationDeclaration.ProviderArguments`. +We deliberately do **not** configure broker-native redrive policies (SQS `maxReceiveCount`, Azure Service Bus `MaxDeliveryCount`, RabbitMQ DLX): that would split authority between broker and core and make behavior transport-specific. A destination's structural creation knobs, if any, are limited to `DestinationDeclaration.ProviderArguments`. -### Delayed redelivery and capability bounds +### Delays and the runtime-store fallback -An explicit `RedeliveryDelay` (or a configured `Backoff`) is served natively when the transport supports it within its advertised limit — `ISupportsRedeliveryDelay.MaxRedeliveryDelay` and `ISupportsDelayedDelivery.MaxDeliveryDelay`. A delay longer than the broker can honor — for example beyond SQS's 15-minute delivery delay or 12-hour visibility window — is routed through the durable job runtime store instead of being silently truncated. If neither native support nor a runtime store is available, the operation fails loudly rather than dropping the delay. +A send delay or redelivery backoff is served natively when the transport supports it within its advertised ceiling (`TransportCapabilities.MaxDeliveryDelay`, `ISupportsRedeliveryDelay.MaxRedeliveryDelay`). A delay the broker cannot honor — beyond SQS's 15-minute delivery delay, or any delayed publish on SNS — is parked in the durable runtime store instead of being silently truncated: `MessageBusOptions.RuntimeStore` takes an `IScheduledDispatchStore` (any `IJobRuntimeStore` satisfies it; the DI builder wires it automatically when a runtime store is configured), and the job runtime pump dispatches parked messages when due. If neither native support nor a store is available, the operation fails loudly rather than dropping the delay. ### Unmatched message types -A message that arrives on a destination but whose type has no registered consumer on this node — for example a newer message type during a rolling deploy, before every node has been updated — is surfaced loudly rather than quietly swallowed. It increments the `foundatio.messaging.unhandled` metric and throws `UnhandledMessageTypeException`, isolated to that one message so the receive loop and the other type handlers keep running. The message is retried so a node that *does* handle the type can pick it up, and is finally dead-lettered as `"no-handler"` once `RetryPolicy.UnmatchedMaxAttempts` (default 50) is exhausted — so a genuinely orphaned type cannot loop forever. +A message arriving on a shared destination whose type has no registered consumer on this node — a newer message type mid rolling-deploy, or a misconfiguration — is surfaced loudly: it increments the `foundatio.messaging.unhandled` metric and throws `UnhandledMessageTypeException`, isolated to that one message so the receive loop and the other type handlers keep running. It is retried so a node that *does* handle the type can pick it up, and finally dead-lettered as `"no-handler"` after `RetryPolicy.UnmatchedMaxAttempts` (default 50) — a genuinely orphaned type cannot loop forever. ## Jobs -`IJobClient` submits durable work and returns a `JobHandle`; it does not execute jobs synchronously: +`IJobClient` submits durable work and returns a `JobHandle`; `IJobWorker` claims and executes; `IJobMonitor` queries state; `IJobRuntimeStore` persists all of it. Jobs implement `IJob`: ```csharp -JobHandle handle = await jobs.EnqueueAsync(); +public class RebuildSearchIndexJob : IJob +{ + public async Task RunAsync(JobExecutionContext context) + { + var args = context.GetArguments(); + await context.ReportProgressAsync(50, "halfway"); + return JobResult.Success; + } +} + +JobHandle handle = await jobs.EnqueueAsync(new RebuildSearchIndexArgs { Index = "orders" }); JobState? state = await handle.GetStateAsync(); +await handle.RequestCancellationAsync(); ``` -Execution belongs to `IJobWorker`, which claims queued jobs from `IJobRuntimeStore`. State and operational queries belong to `IJobMonitor`. Scheduled occurrences are created by `IJobScheduler` and materialized by `JobScheduleProcessor` through the runtime store. +**Typed payloads.** `EnqueueAsync(args)` serializes the arguments into the durable `JobState.Payload` (with `PayloadType` stored as a discriminator for forensics); the job reads them via `JobExecutionContext.GetArguments()`, guarded by `HasArguments`. Mismatches throw a descriptive exception naming the stored type. + +**Execution context.** `JobExecutionContext` carries `JobId`, `Attempt`, and the `CancellationToken`, plus the store-backed helpers useful inside job code: `ReportProgressAsync`, `RenewLeaseAsync` (heartbeat for long runs), and `IsCancellationRequestedAsync` (cooperative cancellation). Its public constructor creates a *detached* context for tests — helpers no-op, and an `arguments` object surfaces through `GetArguments` without serialization. + +**The worker.** Every run gets its own async DI scope (scoped services resolve per run, not as accidental singletons). `JobWorker` runs a bounded pool — at most `maxConcurrency` jobs in flight, a slot freeing the moment a job settles — and claims are compare-and-set guarded so concurrency cannot double-run. Lease renewal is a supervised loop, not a fire-and-forget timer: a run is cancelled when its lease is lost to another node *or* when renewal keeps failing past the lease window (the lease has lapsed on the broker's clock too, so continuing would risk double-executing side effects); the terminal state transition is ownership-guarded so a stale worker cannot overwrite the new owner's state. Stale `Processing` jobs (a worker crash mid-run) are reclaimed and re-queued while attempts remain, then dead-lettered. -Persisted job type names come from `IJobTypeRegistry`. Register stable names for jobs that may move between assemblies or namespaces: +### CRON scheduling ```csharp services.AddFoundatio() - .Jobs.Register("search.rebuild") - .Jobs.UseInMemoryRuntime(); + .Jobs.UseInMemoryRuntime() + .Jobs.AddCronJob("0 2 * * *", o => + { + o.MaxRetries = 3; + o.Arguments = new ExportArgs { Format = "csv" }; + }); ``` -Unregistered jobs fall back to `Type.FullName`, not `AssemblyQualifiedName`. - -### Execution context - -A job that wants its runtime identity and store-backed operations implements `IJobWithExecutionContext`; the runtime sets `ExecutionContext` before invoking it. The context exposes `JobId`, `Attempt`, the cancellation token, and `ReportProgressAsync`, `RenewLeaseAsync` (heartbeat for long runs), and `IsCancellationRequestedAsync` — the parts of `IJobRuntimeStore` useful from inside job code. Jobs that use it should be registered transient (the context is per-run state). Untracked queue/pub-sub messages have no progress concept, so `IReceivedMessage` has no `ReportProgressAsync`. - -### Recovery +`AddCronJob(cron, o => ...)` registers a `ScheduledJobDefinition`; `CronJobOptions` covers `Name`, `Scope` (`Global` = one instance per tick, `PerNode` = every instance), `Overlap` (`SkipIfRunning` default), `MisfireWindow`, `MaxRetries`, `TimeZone`, `Enabled`, and typed `Arguments` serialized into every occurrence's payload. Definitions are scheduled automatically when the pump starts — no manual `IJobScheduler.ScheduleAsync` call. The scheduler materializes every occurrence due within the misfire window (not just the latest) as durable, deduplicated store entries, and owns occurrence recovery with its own per-definition retry/dead-letter budget. -The runtime pump reclaims jobs stuck in `Processing` past their lease (a worker that crashed mid-run), not just CRON occurrences: `IJobRuntimeStore.GetExpiredProcessingAsync` surfaces them and the worker re-queues them while attempts remain (`JobRuntimeServiceOptions.MaxJobAttempts`), otherwise dead-letters them. The status CAS serializes concurrent reclaimers. +### The runtime pump -### CRON +`JobRuntimePumpService` is registered automatically with any runtime store, so a configured store can never silently accumulate work that nothing drains. Each poll it materializes CRON occurrences, then runs an **overlapped execution pass** — dispatching due work (message dispatches before job occurrences, so the messaging delayed-delivery fallback is never head-of-line blocked by a long job), recovering stale jobs, and running queued jobs. Scheduling keeps its cadence even while a long pass runs. Tune with `ConfigureRuntimePump`: `JobRuntimePumpOptions.Enabled` (false = manual control), `PollInterval` (1s), `BatchSize` (100), `MaxJobAttempts` (3), and `WorkerConcurrency` (1; every in-flight job still gets its own DI scope, lease, and cancellation watcher). -The redesigned durable CRON path materializes durable, recoverable occurrences through `IJobScheduler` → `JobScheduleProcessor` → the runtime store and pump. The legacy hosted `AddCronJob`/`AddJobScheduler` API still wires the in-process `ScheduledJobService` and is retained as **legacy/compat only**; routing the default hosted CRON API onto the durable scheduler is a planned follow-up (best validated alongside a real provider, since durable distributed CRON leans on the runtime store's transition semantics). +## Testing -## Migration - -Legacy queue code usually moves from one queue instance per payload type to one app-facing queue plus routing: +`Foundatio.Testing` runs the real bus over a recording in-memory transport for deterministic, sleep-free tests: ```csharp -// Legacy -await queue.EnqueueAsync(new OrderSubmitted(id)); // IQueue - -// New -await queue.EnqueueAsync(new OrderSubmitted(id)); // Foundatio.Messaging.IQueue -``` - -Legacy `IMessageBus` publish/subscribe code maps to `IPubSub` with explicit subscription identity: - -```csharp -// Legacy -await messageBus.PublishAsync(new OrderSubmitted(id)); -await messageBus.SubscribeAsync(HandleAsync); +services.AddFoundatio() + .Messaging.UseTestHarness() + .Messaging.AddHandler(); -// New -await pubsub.PublishAsync(new OrderSubmitted(id)); -await using var subscription = await pubsub.SubscribeAsync(HandleAsync); +// start hosted services, then: +await bus.PublishAsync(new OrderPlaced(42)); +await harness.WaitForIdleAsync(); +Assert.Single(harness.Published()); +Assert.Empty(harness.DeadLetteredMessages); ``` -For per-type routing, register each type. For grouped routing, map an interface or base type. For default/global-style routing, set one default queue destination or topic for otherwise unmapped messages. Operation-level overrides should be reserved for exceptional paths such as replays or priority lanes. - -### `IQueue` name collision during migration - -Two public `IQueue` types coexist while the legacy queue is still shipped: - -- `Foundatio.Queues.IQueue` / `IQueue` — the legacy one-type-per-queue API. -- `Foundatio.Messaging.IQueue` — the new app-facing queue. - -A file that has `using` directives for both namespaces will get a `CS0104` ambiguous-reference error on the bare name `IQueue`. Until the legacy API is removed, disambiguate per file with a `using` alias rather than fully qualifying every usage: - -```csharp -using IQueue = Foundatio.Messaging.IQueue; // new code -// or, while finishing a migration: -// using LegacyQueue = Foundatio.Queues.IQueue; -``` +Resolve `MessagingTestHarness` from the container. `WaitForIdleAsync` blocks until every destination has nothing queued and nothing in flight (throws a `TimeoutException` naming the still-busy destinations). Recordings cover every movement — `SentMessages`, `PublishedMessages`, `HandledMessages`, `AbandonedMessages`, `DeadLetteredMessages`, with typed accessors `Sent()` / `Published()` / `Handled()` / `Abandoned()` / `DeadLettered()` — so the core retry/dead-letter path is directly assertable: a message redelivered N times and then dead-lettered shows up as N abandonments plus one dead-letter. -New application code should depend on `Foundatio.Messaging.IQueue`; the alias keeps call sites clean without dropping the legacy namespace a file may still need mid-migration. +## Providers -## Rollout Notes +- **In-memory** (`InMemoryMessageTransport`, `InMemoryJobRuntimeStore`) — the reference implementation for local dev and tests; supports every operation interface. +- **Redis** (`Foundatio.Redis`) — `RedisStreamsMessageTransport` (FIFO streams; delays route through the runtime store) and `RedisJobRuntimeStore`, wired via `.Messaging.UseRedis()` / `.Jobs.UseRedis()` over one shared connection. +- **AWS** (`Foundatio.Aws`) — `AwsMessageTransport` (queues on SQS, pub/sub on SNS+SQS) via `.Messaging.UseAws()`; role-aware capabilities as above, `AutoCreateDestinations` to control implicit resource creation, and LocalStack support via `ServiceUrl`. -The in-memory transport proves the API shape and conformance coverage for local development. Before locking this as a stable public API, validate at least one external provider against the same routing, topic/subscription, delayed delivery, dead-letter, TTL, priority, and batch constraints. +A new provider is validated against the shared conformance suites in `Foundatio.TestHarness`: `MessageTransportConformanceTests` (send/receive, settlement, redelivery, dead-letter, visibility, provisioning — tests skip per unimplemented operation interface) and `JobRuntimeStoreConformanceTests` (state round-trips, CAS transitions, leases, stale recovery including the renew-during-reclaim race, and scheduled-dispatch claiming, driven by a fake time provider). From 1901c301e20ce9123427a9f8344bd286bec37467 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 9 Jul 2026 21:40:43 -0500 Subject: [PATCH 57/57] CI-feed package publishing is best-effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Push-triggered builds failed at Publish CI Packages when GitHub Packages returned 403 for Foundatio.Redis/Foundatio.Aws — those package names are linked to their original standalone repos, so this repo's GITHUB_TOKEN cannot push new versions of them. A CI-feed rejecting one package must not fail a build whose compile and tests passed: each failed push now surfaces as a warning annotation and the loop continues. Release publishing to NuGet on tags stays strict. Co-Authored-By: Claude Fable 5 --- .github/workflows/build-workflow.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml index 89a1eb450..d964ce7e5 100644 --- a/.github/workflows/build-workflow.yml +++ b/.github/workflows/build-workflow.yml @@ -122,18 +122,23 @@ jobs: - name: Publish CI Packages if: github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' run: | + # CI-feed publishing is best-effort: a feed rejecting one package (e.g. GitHub Packages returns 403 when the + # package name is linked to a different repo) must not fail a build whose compile and tests passed. Each + # failure surfaces as a warning annotation instead. Release publishing to NuGet (below) stays strict. for package in $(find . -name "*.nupkg" | grep -v "minver" | grep -v "/EmptyFiles/"); do # GitHub if [ -n "${{ secrets.GITHUB_TOKEN }}" ]; then echo "${0##*/}": Pushing $package to GitHub... - dotnet nuget push $package --source https://nuget.pkg.github.com/${{ inputs.org }}/index.json --api-key ${{ secrets.GITHUB_TOKEN }} --skip-duplicate + dotnet nuget push $package --source https://nuget.pkg.github.com/${{ inputs.org }}/index.json --api-key ${{ secrets.GITHUB_TOKEN }} --skip-duplicate \ + || echo "::warning::Failed to push ${package##*/} to GitHub Packages; continuing" fi # Feedz (remove once GitHub supports anonymous access) if [ -n "${{ secrets.FEEDZ_KEY }}" ]; then echo "${0##*/}": Pushing $package to Feedz... - dotnet nuget push $package --source https://f.feedz.io/foundatio/foundatio/nuget --api-key ${{ secrets.FEEDZ_KEY }} --skip-duplicate + dotnet nuget push $package --source https://f.feedz.io/foundatio/foundatio/nuget --api-key ${{ secrets.FEEDZ_KEY }} --skip-duplicate \ + || echo "::warning::Failed to push ${package##*/} to Feedz; continuing" fi done