diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs new file mode 100644 index 0000000000..988e8a254d --- /dev/null +++ b/src/ServiceControl.AcceptanceTests/Recoverability/MessageFailures/When_failed_error_imports_are_imported_again.cs @@ -0,0 +1,149 @@ +namespace ServiceControl.AcceptanceTests.Recoverability.MessageFailures +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Runtime.Loader; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + using NServiceBus; + using NUnit.Framework; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Hosting.Commands; + using ServiceControl.Infrastructure; + using ServiceControl.Operations; + using ServiceControl.Persistence; + + class When_failed_error_imports_are_imported_again : AcceptanceTest + { + [Test] + public async Task Should_store_the_sending_and_receiving_endpoints() + { + var settings = await CreateSettings(); + + var sendingHostId = Guid.NewGuid(); + var receivingHostId = Guid.NewGuid(); + var nativeMessageId = Guid.NewGuid().ToString(); + var headers = new Dictionary + { + [Headers.MessageId] = nativeMessageId, + [Headers.EnclosedMessageTypes] = "Reimport.SomeCommand, Reimport, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + [Headers.OriginatingEndpoint] = "Reimport.Sender", + [Headers.OriginatingMachine] = "SenderHost", + [Headers.OriginatingHostId] = sendingHostId.ToString("N"), + [Headers.ProcessingEndpoint] = "Reimport.Receiver", + [Headers.HostDisplayName] = "ReceiverHost", + [Headers.HostId] = receivingHostId.ToString("N"), + ["NServiceBus.FailedQ"] = "Reimport.Receiver@ReceiverHost", + ["NServiceBus.TimeOfFailure"] = DateTimeOffsetHelper.ToWireFormattedString(DateTimeOffset.UtcNow), + ["NServiceBus.ExceptionInfo.ExceptionType"] = "System.InvalidOperationException", + ["NServiceBus.ExceptionInfo.Message"] = "Simulated failure", + ["NServiceBus.ExceptionInfo.StackTrace"] = " at Reimport.Receiver.Handle()" + }; + var failedMessageId = FailedErrorImport.DeriveKey(headers, nativeMessageId).ToString(); + + IHost host = null; + try + { + using (await StorageConfiguration.UseDatabaseLifecycleLock()) + { + await new SetupCommand().Execute(new HostArguments([]), settings); + + host = ImportFailedErrorsCommand.BuildHost(settings); + await host.StartAsync(); + } + + var failedImports = host.Services.GetRequiredService(); + await failedImports.StoreFailedErrorImport(new FailedErrorImport + { + Id = failedMessageId, + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = headers, Body = "{}"u8.ToArray() }, + ExceptionInfo = "Simulated import failure" + }); + + // RavenDB finds failed imports through an index, which may not have caught up with the write yet. + await WaitFor(() => failedImports.QueryContainsFailedImports(), "the failed import to become visible"); + + await host.Services.GetRequiredService().Run(); + + var failedMessage = await host.Services.GetRequiredService() + .GetLatestFailedMessageView(failedMessageId); + + Assert.That(failedMessage, Is.Not.Null, "the failed import was not stored as a failed message"); + using (Assert.EnterMultipleScope()) + { + Assert.That(failedMessage.SendingEndpoint?.Name, Is.EqualTo("Reimport.Sender")); + Assert.That(failedMessage.SendingEndpoint?.Host, Is.EqualTo("SenderHost")); + Assert.That(failedMessage.SendingEndpoint?.HostId, Is.EqualTo(sendingHostId)); + Assert.That(failedMessage.ReceivingEndpoint?.Name, Is.EqualTo("Reimport.Receiver")); + Assert.That(failedMessage.ReceivingEndpoint?.Host, Is.EqualTo("ReceiverHost")); + Assert.That(failedMessage.ReceivingEndpoint?.HostId, Is.EqualTo(receivingHostId)); + } + + var monitoring = host.Services.GetRequiredService(); + + // RavenDB reads known endpoints through an index, which may not have caught up with the import yet. + await WaitFor(async () => + { + var knownEndpointNames = (await monitoring.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name).ToArray(); + return knownEndpointNames.Contains("Reimport.Sender") && knownEndpointNames.Contains("Reimport.Receiver"); + }, "Reimport.Sender and Reimport.Receiver to be recorded as known endpoints"); + } + finally + { + if (host != null) + { + try + { + await host.StopAsync(); + } + finally + { + host.Dispose(); + } + } + } + } + + static async Task WaitFor(Func> condition, string description) + { + var timeout = Stopwatch.StartNew(); + + while (timeout.Elapsed < TimeSpan.FromSeconds(30)) + { + if (await condition()) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(200)); + } + + Assert.Fail($"Timed out waiting for {description}."); + } + + async Task CreateSettings() + { + var logPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(logPath); + + var settings = new Settings(TransportIntegration.TypeName, StorageConfiguration.PersistenceType, + new LoggingSettings(Settings.SettingsRootNamespace, defaultLevel: LogLevel.Debug, logPath: logPath), + forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)) + { + InstanceName = $"Reimport.{Guid.NewGuid():n}", + TransportConnectionString = TransportIntegration.ConnectionString, + AssemblyLoadContextResolver = static _ => AssemblyLoadContext.Default + }; + + await StorageConfiguration.CustomizeSettings(settings); + + return settings; + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs index 41d435247c..4319cff284 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs @@ -619,37 +619,15 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed() [Test] public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first() { + // Subtracting this from the clock cannot be represented, so the failed messages pass fails before the first delete. + // The interceptor cancels the sweep at that delete, so the failure is recorded before the cancellation. EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000); using var cancellation = new CancellationTokenSource(); - var sweeper = GetSweeper(); - - // The failed messages pass throws before any database call, so by the time the start returns - // the event log pass is waiting on the database and the cancellation lands there. - sweeper.TryStartManualSweep(null, null, cancellation.Token); - await cancellation.CancelAsync(); - - await WaitForManualSweepToFinish(); - - var status = sweeper.GetStatus(); - - using (Assert.EnterMultipleScope()) - { - Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); - Assert.That(status.LastError, Does.Contain(nameof(RetentionEntity.FailedMessages))); - } - } + cancelAfterBatchDelete.Cancellation = cancellation; - [Test] - public async Task Manual_sweep_cancelled_part_way_reports_cancelled() - { - using var cancellation = new CancellationTokenSource(); var sweeper = GetSweeper(); - - // The background sweep runs on this thread until its first database call, so it is still - // running when the token is cancelled. sweeper.TryStartManualSweep(null, null, cancellation.Token); - await cancellation.CancelAsync(); await WaitForManualSweepToFinish(); @@ -658,8 +636,7 @@ public async Task Manual_sweep_cancelled_part_way_reports_cancelled() using (Assert.EnterMultipleScope()) { Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); - Assert.That(status.LastError, Is.Null); - Assert.That(status.LastFinishedAt, Is.Not.Null); + Assert.That(status.LastError, Does.Contain(nameof(RetentionEntity.FailedMessages))); } } @@ -758,11 +735,15 @@ public async Task An_event_log_pass_cancelled_between_batches_is_recorded_as_can await WaitForManualSweepToFinish(); + var status = sweeper.GetStatus(); + using (Assert.EnterMultipleScope()) { Assert.That(recorded.Cycles(RetentionEntity.EventLog).Select(cycle => cycle.Result), Is.EqualTo(new[] { "cancelled" })); Assert.That(recorded.Cycles(RetentionEntity.GroupComments), Is.Empty, "the sweep must stop at the pass that was cut short"); - Assert.That(sweeper.GetStatus().LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); + Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled)); + Assert.That(status.LastError, Is.Null, "a cancelled pass is not an error"); + Assert.That(status.LastFinishedAt, Is.Not.Null); Assert.That(await Query(dbContext => dbContext.EventLogItems.CountAsync()), Is.EqualTo(500), "exactly one batch is deleted before the cancellation"); } } diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index af8a53840a..d32da70fb1 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -4,13 +4,16 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; + using EndpointControl.Handlers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; + using Monitoring; using NServiceBus; using Operations; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; + using Persistence; using Recoverability; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; @@ -54,6 +57,11 @@ internal static IHost BuildHost(Settings settings) var hostBuilder = Host.CreateApplicationBuilder(); hostBuilder.AddServiceControl(settings, endpointConfiguration, new RecoverabilityComponent()); + // HeartbeatMonitoringComponent normally registers these two, and this host leaves that component out. + // Without the enricher, a re-imported message loses its sending and receiving endpoints, and they never reach the known endpoints. + hostBuilder.Services.AddSingleton(); + hostBuilder.Services.AddErrorMessageEnricher(); + return hostBuilder.Build(); } }