From 75ad9b99d99d3b34024de658ca09787cc1e9e223 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 18 Nov 2026 12:01:52 +0800 Subject: [PATCH 1/3] fix: Keep the endpoints on messages re-imported by --import-failed-errors --- ...failed_error_imports_are_imported_again.cs | 140 ++++++++++++++++++ .../Commands/ImportFailedErrorsCommand.cs | 7 + 2 files changed, 147 insertions(+) create mode 100644 src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs new file mode 100644 index 0000000000..132a4a2153 --- /dev/null +++ b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs @@ -0,0 +1,140 @@ +namespace ServiceControl.AcceptanceTests.Recoverability.MessageFailures +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Runtime.Loader; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + using NServiceBus; + using NUnit.Framework; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Hosting.Commands; + using ServiceControl.Infrastructure; + using ServiceControl.Operations; + using ServiceControl.Persistence; + + class When_failed_error_imports_are_imported_again : AcceptanceTest + { + [Test] + public async Task Should_store_the_sending_and_receiving_endpoints() + { + var settings = await CreateSettings(); + + var sendingHostId = Guid.NewGuid(); + var receivingHostId = Guid.NewGuid(); + var nativeMessageId = Guid.NewGuid().ToString(); + var headers = new Dictionary + { + [Headers.MessageId] = nativeMessageId, + [Headers.EnclosedMessageTypes] = "Reimport.SomeCommand, Reimport, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + [Headers.OriginatingEndpoint] = "Reimport.Sender", + [Headers.OriginatingMachine] = "SenderHost", + [Headers.OriginatingHostId] = sendingHostId.ToString("N"), + [Headers.ProcessingEndpoint] = "Reimport.Receiver", + [Headers.HostDisplayName] = "ReceiverHost", + [Headers.HostId] = receivingHostId.ToString("N"), + ["NServiceBus.FailedQ"] = "Reimport.Receiver@ReceiverHost", + ["NServiceBus.TimeOfFailure"] = DateTimeOffsetHelper.ToWireFormattedString(DateTimeOffset.UtcNow), + ["NServiceBus.ExceptionInfo.ExceptionType"] = "System.InvalidOperationException", + ["NServiceBus.ExceptionInfo.Message"] = "Simulated failure", + ["NServiceBus.ExceptionInfo.StackTrace"] = " at Reimport.Receiver.Handle()" + }; + var failedMessageId = FailedErrorImport.DeriveKey(headers, nativeMessageId).ToString(); + + IHost host; + using (await StorageConfiguration.UseDatabaseLifecycleLock()) + { + await new SetupCommand().Execute(new HostArguments([]), settings); + + host = ImportFailedErrorsCommand.BuildHost(settings); + await host.StartAsync(); + } + + try + { + var failedImports = host.Services.GetRequiredService(); + await failedImports.StoreFailedErrorImport(new FailedErrorImport + { + Id = failedMessageId, + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = headers, Body = "{}"u8.ToArray() }, + ExceptionInfo = "Simulated import failure" + }); + + // RavenDB finds failed imports through an index, which may not have caught up with the write yet. + await WaitFor(() => failedImports.QueryContainsFailedImports(), "the failed import to become visible"); + + await host.Services.GetRequiredService().Run(); + + var failedMessage = await host.Services.GetRequiredService() + .GetLatestFailedMessageView(failedMessageId); + + Assert.That(failedMessage, Is.Not.Null, "the failed import was not stored as a failed message"); + using (Assert.EnterMultipleScope()) + { + Assert.That(failedMessage.SendingEndpoint?.Name, Is.EqualTo("Reimport.Sender")); + Assert.That(failedMessage.SendingEndpoint?.Host, Is.EqualTo("SenderHost")); + Assert.That(failedMessage.SendingEndpoint?.HostId, Is.EqualTo(sendingHostId)); + Assert.That(failedMessage.ReceivingEndpoint?.Name, Is.EqualTo("Reimport.Receiver")); + Assert.That(failedMessage.ReceivingEndpoint?.Host, Is.EqualTo("ReceiverHost")); + Assert.That(failedMessage.ReceivingEndpoint?.HostId, Is.EqualTo(receivingHostId)); + } + + var monitoring = host.Services.GetRequiredService(); + + // RavenDB reads known endpoints through an index, which may not have caught up with the import yet. + await WaitFor(async () => + { + var knownEndpointNames = (await monitoring.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name).ToArray(); + return knownEndpointNames.Contains("Reimport.Sender") && knownEndpointNames.Contains("Reimport.Receiver"); + }, "Reimport.Sender and Reimport.Receiver to be recorded as known endpoints"); + } + finally + { + await host.StopAsync(); + host.Dispose(); + } + } + + static async Task WaitFor(Func> condition, string description) + { + var timeout = Stopwatch.StartNew(); + + while (timeout.Elapsed < TimeSpan.FromSeconds(30)) + { + if (await condition()) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + Assert.Fail($"Timed out waiting for {description}."); + } + + async Task CreateSettings() + { + var logPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(logPath); + + var settings = new Settings(TransportIntegration.TypeName, StorageConfiguration.PersistenceType, + new LoggingSettings(Settings.SettingsRootNamespace, defaultLevel: LogLevel.Debug, logPath: logPath), + forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)) + { + InstanceName = $"Reimport.{Guid.NewGuid():n}", + TransportConnectionString = TransportIntegration.ConnectionString, + AssemblyLoadContextResolver = static _ => AssemblyLoadContext.Default + }; + + await StorageConfiguration.CustomizeSettings(settings); + + return settings; + } + } +} diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index af8a53840a..79fdf7fc2c 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -4,13 +4,16 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; + using EndpointControl.Handlers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; + using Monitoring; using NServiceBus; using Operations; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; + using Persistence; using Recoverability; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; @@ -54,6 +57,10 @@ internal static IHost BuildHost(Settings settings) var hostBuilder = Host.CreateApplicationBuilder(); hostBuilder.AddServiceControl(settings, endpointConfiguration, new RecoverabilityComponent()); + // Heartbeat monitoring normally registers the enricher that records each message's endpoints, and this host does not run it. + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddErrorMessageEnricher(); + return hostBuilder.Build(); } } From 2d8e6f22c48fa58b9be73805d4a2ede9c94b6f30 Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Wed, 18 Nov 2026 13:09:39 +0800 Subject: [PATCH 2/3] fix: Update retention period settings for event logs in sweep tests --- .../EFCore/RetentionSweepTests.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 41d435247c..2dda8e7c79 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -620,14 +620,15 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed() public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first() { EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000); + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); + await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]); + // The failed messages pass throws before any database call, so the first delete is an event log batch. using var cancellation = new CancellationTokenSource(); - var sweeper = GetSweeper(); + cancelAfterBatchDelete.Cancellation = cancellation; - // The failed messages pass throws before any database call, so by the time the start returns - // the event log pass is waiting on the database and the cancellation lands there. + var sweeper = GetSweeper(); sweeper.TryStartManualSweep(null, null, cancellation.Token); - await cancellation.CancelAsync(); await WaitForManualSweepToFinish(); @@ -643,13 +644,14 @@ public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first [Test] public async Task Manual_sweep_cancelled_part_way_reports_cancelled() { + EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); + await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]); + using var cancellation = new CancellationTokenSource(); - var sweeper = GetSweeper(); + cancelAfterBatchDelete.Cancellation = cancellation; - // The background sweep runs on this thread until its first database call, so it is still - // running when the token is cancelled. + var sweeper = GetSweeper(); sweeper.TryStartManualSweep(null, null, cancellation.Token); - await cancellation.CancelAsync(); await WaitForManualSweepToFinish(); From e1a59dc20e0a38e3caf221463b0c67832d45dfec Mon Sep 17 00:00:00 2001 From: Warwick Schroeder Date: Fri, 20 Nov 2026 10:36:40 +0800 Subject: [PATCH 3/3] fix: Improve error handling and resource management in failed error imports --- ...failed_error_imports_are_imported_again.cs | 29 +++++++++------ .../EFCore/RetentionSweepTests.cs | 35 ++++--------------- .../Commands/ImportFailedErrorsCommand.cs | 3 +- 3 files changed, 28 insertions(+), 39 deletions(-) diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs index 132a4a2153..988e8a254d 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs @@ -47,17 +47,17 @@ public async Task Should_store_the_sending_and_receiving_endpoints() }; var failedMessageId = FailedErrorImport.DeriveKey(headers, nativeMessageId).ToString(); - IHost host; - using (await StorageConfiguration.UseDatabaseLifecycleLock()) + IHost host = null; + try { - await new SetupCommand().Execute(new HostArguments([]), settings); + using (await StorageConfiguration.UseDatabaseLifecycleLock()) + { + await new SetupCommand().Execute(new HostArguments([]), settings); - host = ImportFailedErrorsCommand.BuildHost(settings); - await host.StartAsync(); - } + host = ImportFailedErrorsCommand.BuildHost(settings); + await host.StartAsync(); + } - try - { var failedImports = host.Services.GetRequiredService(); await failedImports.StoreFailedErrorImport(new FailedErrorImport { @@ -96,8 +96,17 @@ await WaitFor(async () => } finally { - await host.StopAsync(); - host.Dispose(); + if (host != null) + { + try + { + await host.StopAsync(); + } + finally + { + host.Dispose(); + } + } } } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 2dda8e7c79..4319cff284 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -619,11 +619,10 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed() [Test] public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first() { + // Subtracting this from the clock cannot be represented, so the failed messages pass fails before the first delete. + // The interceptor cancels the sweep at that delete, so the failure is recorded before the cancellation. EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000); - EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); - await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]); - // The failed messages pass throws before any database call, so the first delete is an event log batch. using var cancellation = new CancellationTokenSource(); cancelAfterBatchDelete.Cancellation = cancellation; @@ -641,30 +640,6 @@ public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first } } - [Test] - public async Task Manual_sweep_cancelled_part_way_reports_cancelled() - { - EFSettings.EventsRetentionPeriod = TimeSpan.FromDays(14); - await Store([.. Enumerable.Range(0, 1500).Select(i => EventLogRow($"expired-{i}", Now.AddDays(-15)))]); - - using var cancellation = new CancellationTokenSource(); - cancelAfterBatchDelete.Cancellation = cancellation; - - var sweeper = GetSweeper(); - sweeper.TryStartManualSweep(null, null, cancellation.Token); - - await WaitForManualSweepToFinish(); - - var status = sweeper.GetStatus(); - - using (Assert.EnterMultipleScope()) - { - Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); - Assert.That(status.LastError, Is.Null); - Assert.That(status.LastFinishedAt, Is.Not.Null); - } - } - [Test] public async Task Starting_a_sweep_clears_the_previous_run_s_outcome() { @@ -760,11 +735,15 @@ public async Task An_event_log_pass_cancelled_between_batches_is_recorded_as_can await WaitForManualSweepToFinish(); + var status = sweeper.GetStatus(); + using (Assert.EnterMultipleScope()) { Assert.That(recorded.Cycles(RetentionEntity.EventLog).Select(cycle => cycle.Result), Is.EqualTo(new[] { "cancelled" })); Assert.That(recorded.Cycles(RetentionEntity.GroupComments), Is.Empty, "the sweep must stop at the pass that was cut short"); - Assert.That(sweeper.GetStatus().LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); + Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); + Assert.That(status.LastError, Is.Null, "a cancelled pass is not an error"); + Assert.That(status.LastFinishedAt, Is.Not.Null); Assert.That(await Query(dbContext => dbContext.EventLogItems.CountAsync()), Is.EqualTo(500), "exactly one batch is deleted before the cancellation"); } } diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index 79fdf7fc2c..d32da70fb1 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -57,7 +57,8 @@ internal static IHost BuildHost(Settings settings) var hostBuilder = Host.CreateApplicationBuilder(); hostBuilder.AddServiceControl(settings, endpointConfiguration, new RecoverabilityComponent()); - // Heartbeat monitoring normally registers the enricher that records each message's endpoints, and this host does not run it. + // HeartbeatMonitoringComponent normally registers these two, and this host leaves that component out. + // Without the enricher, a re-imported message loses its sending and receiving endpoints, and they never reach the known endpoints. hostBuilder.Services.AddSingleton(); hostBuilder.Services.AddErrorMessageEnricher();