Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
namespace ServiceControl.AcceptanceTests.Recoverability.MessageFailures
{
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.Loader;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NServiceBus;
using NUnit.Framework;
using Particular.ServiceControl.Hosting;
using ServiceBus.Management.Infrastructure.Settings;
using ServiceControl.Hosting.Commands;
using ServiceControl.Infrastructure;
using ServiceControl.Operations;
using ServiceControl.Persistence;

class When_failed_error_imports_are_imported_again : AcceptanceTest
{
[Test]
public async Task Should_store_the_sending_and_receiving_endpoints()
{
var settings = await CreateSettings();

var sendingHostId = Guid.NewGuid();
var receivingHostId = Guid.NewGuid();
var nativeMessageId = Guid.NewGuid().ToString();
var headers = new Dictionary<string, string>
{
[Headers.MessageId] = nativeMessageId,
[Headers.EnclosedMessageTypes] = "Reimport.SomeCommand, Reimport, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null",
[Headers.OriginatingEndpoint] = "Reimport.Sender",
[Headers.OriginatingMachine] = "SenderHost",
[Headers.OriginatingHostId] = sendingHostId.ToString("N"),
[Headers.ProcessingEndpoint] = "Reimport.Receiver",
[Headers.HostDisplayName] = "ReceiverHost",
[Headers.HostId] = receivingHostId.ToString("N"),
["NServiceBus.FailedQ"] = "Reimport.Receiver@ReceiverHost",
["NServiceBus.TimeOfFailure"] = DateTimeOffsetHelper.ToWireFormattedString(DateTimeOffset.UtcNow),
["NServiceBus.ExceptionInfo.ExceptionType"] = "System.InvalidOperationException",
["NServiceBus.ExceptionInfo.Message"] = "Simulated failure",
["NServiceBus.ExceptionInfo.StackTrace"] = " at Reimport.Receiver.Handle()"
};
var failedMessageId = FailedErrorImport.DeriveKey(headers, nativeMessageId).ToString();

IHost host = null;
try
{
using (await StorageConfiguration.UseDatabaseLifecycleLock())
{
await new SetupCommand().Execute(new HostArguments([]), settings);

host = ImportFailedErrorsCommand.BuildHost(settings);
await host.StartAsync();
}

var failedImports = host.Services.GetRequiredService<IFailedErrorImportDataStore>();
await failedImports.StoreFailedErrorImport(new FailedErrorImport
{
Id = failedMessageId,
Message = new FailedTransportMessage { Id = nativeMessageId, Headers = headers, Body = "{}"u8.ToArray() },
ExceptionInfo = "Simulated import failure"
});

// RavenDB finds failed imports through an index, which may not have caught up with the write yet.
await WaitFor(() => failedImports.QueryContainsFailedImports(), "the failed import to become visible");

await host.Services.GetRequiredService<ImportFailedErrors>().Run();

var failedMessage = await host.Services.GetRequiredService<IFailedMessageQueryDataStore>()
.GetLatestFailedMessageView(failedMessageId);

Assert.That(failedMessage, Is.Not.Null, "the failed import was not stored as a failed message");
using (Assert.EnterMultipleScope())
{
Assert.That(failedMessage.SendingEndpoint?.Name, Is.EqualTo("Reimport.Sender"));
Assert.That(failedMessage.SendingEndpoint?.Host, Is.EqualTo("SenderHost"));
Assert.That(failedMessage.SendingEndpoint?.HostId, Is.EqualTo(sendingHostId));
Assert.That(failedMessage.ReceivingEndpoint?.Name, Is.EqualTo("Reimport.Receiver"));
Assert.That(failedMessage.ReceivingEndpoint?.Host, Is.EqualTo("ReceiverHost"));
Assert.That(failedMessage.ReceivingEndpoint?.HostId, Is.EqualTo(receivingHostId));
}

var monitoring = host.Services.GetRequiredService<IMonitoringDataStore>();

// RavenDB reads known endpoints through an index, which may not have caught up with the import yet.
await WaitFor(async () =>
{
var knownEndpointNames = (await monitoring.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name).ToArray();
return knownEndpointNames.Contains("Reimport.Sender") && knownEndpointNames.Contains("Reimport.Receiver");
}, "Reimport.Sender and Reimport.Receiver to be recorded as known endpoints");
}
finally
{
if (host != null)
{
try
{
await host.StopAsync();
}
finally
{
host.Dispose();
}
}
}
}

static async Task WaitFor(Func<Task<bool>> condition, string description)
{
var timeout = Stopwatch.StartNew();

while (timeout.Elapsed < TimeSpan.FromSeconds(30))
{
if (await condition())
{
return;
}

await Task.Delay(TimeSpan.FromMilliseconds(200));
}

Assert.Fail($"Timed out waiting for {description}.");
}

async Task<Settings> CreateSettings()
{
var logPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(logPath);

var settings = new Settings(TransportIntegration.TypeName, StorageConfiguration.PersistenceType,
new LoggingSettings(Settings.SettingsRootNamespace, defaultLevel: LogLevel.Debug, logPath: logPath),
forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10))
{
InstanceName = $"Reimport.{Guid.NewGuid():n}",
TransportConnectionString = TransportIntegration.ConnectionString,
AssemblyLoadContextResolver = static _ => AssemblyLoadContext.Default
};

await StorageConfiguration.CustomizeSettings(settings);

return settings;
}
}
}
37 changes: 9 additions & 28 deletions src/ServiceControl.Persistence.Tests/EFCore/RetentionSweepTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -619,37 +619,15 @@ public async Task Scheduled_sweep_with_a_failing_pass_reports_failed()
[Test]
public async Task A_cancelled_sweep_keeps_the_errors_of_passes_that_failed_first()
{
// Subtracting this from the clock cannot be represented, so the failed messages pass fails before the first delete.
// The interceptor cancels the sweep at that delete, so the failure is recorded before the cancellation.
EFSettings.ErrorRetentionPeriod = TimeSpan.FromDays(1_000_000);

using var cancellation = new CancellationTokenSource();
var sweeper = GetSweeper();

// The failed messages pass throws before any database call, so by the time the start returns
// the event log pass is waiting on the database and the cancellation lands there.
sweeper.TryStartManualSweep(null, null, cancellation.Token);
await cancellation.CancelAsync();

await WaitForManualSweepToFinish();

var status = sweeper.GetStatus();

using (Assert.EnterMultipleScope())
{
Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled));
Assert.That(status.LastError, Does.Contain(nameof(RetentionEntity.FailedMessages)));
}
}
cancelAfterBatchDelete.Cancellation = cancellation;

[Test]
public async Task Manual_sweep_cancelled_part_way_reports_cancelled()
{
using var cancellation = new CancellationTokenSource();
var sweeper = GetSweeper();

// The background sweep runs on this thread until its first database call, so it is still
// running when the token is cancelled.
sweeper.TryStartManualSweep(null, null, cancellation.Token);
await cancellation.CancelAsync();

await WaitForManualSweepToFinish();

Expand All @@ -658,8 +636,7 @@ public async Task Manual_sweep_cancelled_part_way_reports_cancelled()
using (Assert.EnterMultipleScope())
{
Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled));
Assert.That(status.LastError, Is.Null);
Assert.That(status.LastFinishedAt, Is.Not.Null);
Assert.That(status.LastError, Does.Contain(nameof(RetentionEntity.FailedMessages)));
}
}

Expand Down Expand Up @@ -758,11 +735,15 @@ public async Task An_event_log_pass_cancelled_between_batches_is_recorded_as_can

await WaitForManualSweepToFinish();

var status = sweeper.GetStatus();

using (Assert.EnterMultipleScope())
{
Assert.That(recorded.Cycles(RetentionEntity.EventLog).Select(cycle => cycle.Result), Is.EqualTo(new[] { "cancelled" }));
Assert.That(recorded.Cycles(RetentionEntity.GroupComments), Is.Empty, "the sweep must stop at the pass that was cut short");
Assert.That(sweeper.GetStatus().LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled));
Assert.That(status.LastOutcome, Is.EqualTo(RetentionSweepOutcome.Cancelled));
Assert.That(status.LastError, Is.Null, "a cancelled pass is not an error");
Assert.That(status.LastFinishedAt, Is.Not.Null);
Assert.That(await Query(dbContext => dbContext.EventLogItems.CountAsync()), Is.EqualTo(500), "exactly one batch is deleted before the cancellation");
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EndpointControl.Handlers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Monitoring;
using NServiceBus;
using Operations;
using Particular.ServiceControl;
using Particular.ServiceControl.Hosting;
using Persistence;
using Recoverability;
using ServiceBus.Management.Infrastructure.Settings;
using ServiceControl.Infrastructure;
Expand Down Expand Up @@ -54,6 +57,11 @@ internal static IHost BuildHost(Settings settings)
var hostBuilder = Host.CreateApplicationBuilder();
hostBuilder.AddServiceControl(settings, endpointConfiguration, new RecoverabilityComponent());

// HeartbeatMonitoringComponent normally registers these two, and this host leaves that component out.
// Without the enricher, a re-imported message loses its sending and receiving endpoints, and they never reach the known endpoints.
hostBuilder.Services.AddSingleton<IEndpointInstanceMonitoring, EndpointInstanceMonitoring>();
hostBuilder.Services.AddErrorMessageEnricher<DetectNewEndpointsFromErrorImportsEnricher>();

return hostBuilder.Build();
}
}
Expand Down