diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index 27e456c9c0..43cf0163e6 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -9,7 +9,7 @@ The migration does not move the audit instance or the monitoring instance. The a ## How the migration runs -Run `--start-data-migration` with ServiceControl stopped to copy the data it needs to open. **This part is your outage**. After the command succeeds, start ServiceControl to copy the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. +Run `--start-data-migration` with ServiceControl stopped to copy the data it needs to open. **This part is your outage**. After the command succeeds, start ServiceControl to copy the unresolved failed messages, the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. > [!IMPORTANT] > Until ServiceControl opens, you can go back to RavenDB and lose nothing. After it opens, you cannot go back. @@ -70,8 +70,9 @@ Make sure that your setup is supported: Get the data ready: -- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. The copy reads archived and resolved messages only after ServiceControl opens. The exception is a RavenDB index that is broken or leaves out some failed messages: then the copy reads every failed message during the outage, logs a warning, and a large archive makes the outage longer too. +- Let retries finish where you can. ServiceControl stays closed until every retry-issued message and every message that holds a retry claim is copied, and a retry-issued message never ages out. Unresolved messages without a claim copy after ServiceControl opens, so a large unresolved backlog does not lengthen the outage, but it does lengthen the background copy. The copy reads archived and resolved messages only after ServiceControl opens. The exception is a RavenDB index that is broken or leaves out some failed messages: then the copy reads every failed message during the outage, logs a warning, and a large archive makes the outage longer too. - Run `--import-failed-errors` while the instance is still on RavenDB. A failed error import is a message that ServiceControl took off the error queue but failed to store. The migration copies these either way, but imported ones arrive as ordinary failed messages. +- Let retry, archive and unarchive operations finish on RavenDB before stopping ServiceControl. Migration refuses persisted batches, the retry forwarding marker, and archive or unarchive operation documents. Stopping ServiceControl leaves unfinished work persisted. Keep these settings as they are: @@ -146,6 +147,8 @@ If you want less optional data, set a shorter window for the optional categories By default, the migration copies all the optional data that RavenDB still holds (current retension period setting). A window longer than the retention period copies nothing extra. After a category starts copying, you cannot change its window. To stop it early, abandon it. +The unresolved failed messages have no window and no setting turns them off. They copy once ServiceControl opens, however old they are. + ## Step 4: Run the dry run The dry run reads RavenDB and tells you what the migration will do, including an estimate of how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. @@ -205,6 +208,20 @@ No configuration change is needed to finish. Once every category is Done or Aban ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then run `--start-data-migration` again. The dry run runs the same checks, so you can use it to test your correction. +### RavenDB holds unfinished operations + +The refusal names the unfinished operation families and an example document id from each. The migration does not copy active operation state. Completed retry history and outstanding retry claims do not cause this refusal. + +1. Set `ServiceControl/PersistenceType` back to RavenDB. +2. Start ServiceControl. +3. Request the same group action again if an archive or unarchive operation does not resume. +4. Wait until the retry, archive and unarchive operations finish. +5. Stop ServiceControl. +6. Set `ServiceControl/PersistenceType` back to the SQL target. +7. Run `--start-data-migration` again. + +If the refusal persists, contact support rather than deleting operation documents. A batch document can remain without its operation document. + ### A category is Failed A category becomes Failed when it cannot copy rows, or when an error stops it. The rows that it already copied stay in SQL. A restart does not change a Failed category. You must run one of two commands. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 36c4d99b4d..3f6130d09b 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -1,7 +1,7 @@ # Migration Engine: Design Overview > [!NOTE] -> This build does not carry the whole migration yet. This page describes it as it will be when it ships. +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. `MigrationIsReleasedCheck` refuses production copies until it does. [The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. @@ -18,10 +18,10 @@ - **Retention does most of the work.** Error retention is 5 to 45 days and event retention defaults to 14 days, so most of RavenDB ages out on its own within weeks. - **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. -- **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. +- **Everything else is copied in the background while it runs.** This is the *optional* data: the unresolved failed messages, the event log and archived and resolved messages. - **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. -- **The main thing that makes the outage long is a big backlog of unresolved failures.** They are required, rarely age out (see the [6.18 exception](#rows-removed-after-the-copy)), and can be months old. Keep them low by resolving and archiving. The *dry run* tells you where you stand. -- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds unresolved failures through a RavenDB index, and waits for it to catch up first. If the index is broken or incomplete, the copy reads every failed message instead and logs a warning. Then a big archive does make the outage longer. +- **The failed messages copied during the outage are every retry-issued message, whether or not its retry claim remains, and every other failed message that still holds a retry claim.** Unresolved messages that hold no claim copy in the background, however old they are, so a big unresolved backlog does not lengthen the outage. +- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds retry-issued messages through a RavenDB index, and waits for it to catch up first. If the index is broken or incomplete, the copy reads every failed message instead and logs a warning. Then a big archive does make the outage longer. ```mermaid flowchart LR @@ -139,11 +139,11 @@ The copy goes from RavenDB, through the ServiceControl host, to SQL, because doc ### What data moves -There are eighteen *categories*: sixteen required and two optional. +There are nineteen *categories*: sixteen required and three optional. | Category | Kind | Why | | --- | --- | --- | -| Unresolved and retry-issued failed messages, with bodies | Required | The point of the instance. Issuing a retry removes the expiry, so retry-issued messages never age out. Leave one behind and the retry confirmation finds no row to resolve, so the message stays missing even though the retry succeeded | +| Retry-issued failed messages, and failed messages that hold a retry claim, with bodies | Required | Issuing a retry removes the expiry, so retry-issued messages never age out. Leave one behind and the retry confirmation finds no row to resolve, so the message stays missing even though the retry succeeded. A message of any other status that still holds a retry claim copies with it, with its groups, body and claim. The category keeps the checkpoint id `UnresolvedAndRetryIssuedFailedMessages` from the layout that copied every unresolved message here, see [Progress and checkpoints](#progress-and-checkpoints). A delayed retry confirmation for an unresolved message whose claim a re-failure removed is not covered, an accepted edge case, see [Writing to SQL](#writing-to-sql) | | Message redirects | Required | | | Known endpoints, with the monitored flag | Required | The flag is part of the endpoint row | | Endpoint settings | Required | Copied after known endpoints | @@ -158,15 +158,15 @@ There are eighteen *categories*: sixteen required and two optional. | Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been, except a `MessageFailed` event for a message archived or resolved before the move (see [Rows removed after the copy](#rows-removed-after-the-copy)). RavenDB sends them in no particular order anyway, so no ordering is lost | | Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts. Results of the six checks the RavenDB persister runs about its own database, matched on id and category, are left out, as harmless skips | | Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | -| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. Some comments are removed after ServiceControl opens, see [Open decisions](#rows-removed-after-the-copy) | +| Group comments | Required | Copied after the required failed messages and never waiting for an optional category. Blank comments are not copied. A comment on a group whose messages have not reached SQL yet is removed shortly after ServiceControl opens, an accepted edge case, see [Rows removed after the copy](#rows-removed-after-the-copy) | +| Unresolved failed messages, with bodies | Optional | Copied once ServiceControl has opened, whatever the age of the message, because an operator is waiting on each one. It has no window and no setting turns it off. A message that already holds a retry claim copied with the required category, so it counts as already present. An existing SQL row always wins, see [Writing to SQL](#writing-to-sql). Group retry, archive and unarchive act only on the messages copied so far, an accepted edge case | | Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | | Archived and resolved failed messages, with bodies | Optional | The biggest category by far, and most of the copying time | **Never copied, because nothing needs them:** - RavenDB index definitions. -- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations` and `UnarchiveOperations`. -- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. +- In-flight retry, archive and unarchive work: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations`, `UnarchiveOperations`, `ArchiveBatches` and `UnarchiveBatches`. Migration refuses while any of these documents remain. Stopping ServiceControl does not finish persisted work; restart on RavenDB, let the operations finish, stop it, and retry migration. - The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Nothing has read it since 6.2. - Broker and audit service version details, which refill on the next throughput collection. - Failed message edit locks, which stop one failed message being edited twice. See [Open decisions](#values-that-change-or-merge-in-sql). @@ -174,11 +174,11 @@ There are eighteen *categories*: sixteen required and two optional. **Written new by the copy rather than copied:** -- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts that RavenDB still held a claim for, so a retry by id sends or holds back a message just as it did before the move. +- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each message it inserts that RavenDB still held a claim for, whatever the status of the message, so a retry by id sends or holds back a message just as it did before the move. ### Optional data windows -- **Each optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. +- **Each windowed optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. The unresolved failed messages have no window. The migration always selects that category and copies every unresolved message however old. - **The window defaults to the instance's own retention period**: `ServiceControl/EventRetentionPeriod` for the event log, `ServiceControl/ErrorRetentionPeriod` for archived and resolved messages. So by default everything RavenDB still holds is copied. - **`0` turns a category off before it starts.** A longer window than the retention period copies nothing extra, because older rows would only be deleted by SQL's clean-up. - **The window counts back from when the category first started copying**, which the checkpoint records, so a restart copies the same window rather than one that has slid forward. @@ -263,7 +263,9 @@ In this order, cheapest first: If every required category is already Done or Abandoned, the offline command can succeed without opening the source. Normal startup does not need the source; only the online copier opens it. -Checks stop at the first refusal, which names the check. +9. The source holds no persisted retry batches, forwarding marker, archive operations or batches, or unarchive operations or batches. The check names an example document from each unfinished family. Completed retry history and outstanding retry claims do not cause this refusal. + +Checks stop at the first refusal, which names the check. If persisted operations remain, start ServiceControl on RavenDB again, let them finish, stop it, and rerun `--start-data-migration`. ```mermaid flowchart LR @@ -289,6 +291,9 @@ flowchart LR ### Writing to SQL - **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). +- **An existing SQL row always wins.** The copy never updates a row and counts an existing row as already present, not as a skip. When ingestion created the row first, its attempt count and first failure time can omit earlier RavenDB failures, and it can hold an older payload than RavenDB's newest attempt. These outcomes are accepted. +- **The copy accepts unlikely races with live operations.** A delayed retry confirmation can be consumed before an unresolved message whose claim a re-failure removed arrives, leaving it unresolved. A message archived or resolved in SQL and then deleted by retention before the copy reaches it can arrive again as unresolved. A queued edit-and-retry is discarded if its original message has not arrived. +- **Bodies are immutable and written only if absent.** The body store keeps one body per `UniqueMessageId`, and each backend refuses to replace one that exists: the file system through an exclusive move, Azure Blob and S3 through a conditional upload. The first writer wins, so a body the copy stores just before ingestion does is the body that stays. That is safe only because every attempt of one message carries the same body bytes, which ingestion already assumes. An S3-compatible store that ignores `If-None-Match` falls back to a check followed by an unconditional upload, which can replace a live body. - **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. A failure group with no id is the exception, see [Open decisions](#values-that-change-or-merge-in-sql). - **`UniqueMessageId` keeps its value** but becomes a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry key and the body key. - **`StatusChangedAt` is rebuilt.** For archived and resolved messages it is `@expires` minus the current `ServiceControl/ErrorRetentionPeriod` (wrong if that changed, see [Open decisions](#values-that-change-or-merge-in-sql)). For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL`. @@ -349,7 +354,7 @@ A category is *Failed* when something went wrong. What it copied stays committed - **The other categories still run**, so one fix-and-retry covers them all. A category that depends on a Failed one waits. - **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. -- **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. +- **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. For the unresolved failed messages that slice is the unresolved backlog, which then exists only in RavenDB. Some losses leave a category Done without any check noticing. They are listed under [Open decisions](#what-the-checks-do-not-catch). @@ -496,8 +501,8 @@ The dry run counts the subscription and endpoint settings merges before anything ### Rows removed after the copy -- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. The same happens when the group's messages are gone from RavenDB or were skipped by the copy. The copy names each such comment, in a warning starting `Changed GroupComment/ in category GroupComments on the way into SQL: The group '' has no copied failed message`, which also quotes the whole comment. -- **A `MessageFailed` event still waiting to be sent for a message archived or resolved before the move is never sent.** ServiceControl looks the message up when it sends the event. Archived and resolved messages arrive only with the background copy, so the lookup finds nothing, and the event is deleted with no log line. RavenDB would have sent it. +- **A group comment on a group that has no failed message in SQL is removed soon after ServiceControl opens**, because the retention sweep deletes a comment whose group has no row. The group has none when its messages are unresolved without a retry claim and the background copy has not reached them, when they are archived or resolved, when they are gone from RavenDB, or when the copy skipped them. The copy names each such comment, in a warning starting `Changed GroupComment/ in category GroupComments on the way into SQL: The group '' has no copied failed message`, which also quotes the whole comment. This loss is accepted for groups the background copy has not reached yet. +- **A `MessageFailed` event still waiting to be sent for a message archived or resolved before the move is never sent.** ServiceControl looks the message up when it sends the event. Archived and resolved messages arrive only with the background copy, so the lookup finds nothing, and the event is deleted with no log line. RavenDB would have sent it. The same lookup can fail for an unresolved message the background copy has not reached yet, an accepted edge case. - **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. No log line names these messages yet. ### What the checks do not catch @@ -518,7 +523,7 @@ The dry run counts the subscription and endpoint settings merges before anything - **Category:** one kind of data copied as a unit, such as known endpoints or the event log. - **Required category:** one ServiceControl needs the moment it opens, so it copies while ServiceControl is closed. -- **Optional category:** one copied in the background after ServiceControl opens. It can be shortened or turned off before it starts. +- **Optional category:** one copied in the background after ServiceControl opens. An optional category other than the unresolved failed messages can be shortened or turned off before it starts. - **Window:** how far back an optional category copies. - **Closed period:** the time ServiceControl is not serving traffic because the required copy is running. - **Checkpoint:** the row in SQL recording one category's progress, saved in the same transaction as the rows it describes. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md index d066366a1b..e1fee92666 100644 --- a/docs/migration/ravendb-to-sql-migration-system-design.md +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -66,7 +66,7 @@ The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadines ## What runs on every SQL instance - **Registrations.** `BasePersistence.RegisterDataStores`, reached through each provider's `AddPersistence`, registers `IMigrationCheckpointStore` as `EFMigrationCheckpointStore`, `IMigrationTargetReadiness` as `EFCoreMigrationTargetReadiness` and `IMigrationTarget` as `EFCoreMigrationTarget`, all singletons. -- **`MigrationBeforeTheHostOpens`.** `RunCommand` registers a lifecycle service that checks required checkpoints in `StartingAsync`, before any hosted service starts or Kestrel binds. It delegates to `FinishedCopyBeforeAnIngestionNodeOpens`, treating unknown ids as required. An unfinished required row refuses startup and names `--start-data-migration`. The service never copies data. With no checkpoint store registered (RavenDB), it returns immediately. +- **`MigrationBeforeTheHostOpens`.** `RunCommand` registers a lifecycle service that checks required checkpoints in `StartingAsync`, before any hosted service starts or Kestrel binds. It delegates to `FinishedCopyBeforeAnIngestionNodeOpens`, treating unknown ids as required and refusing checkpoints from an earlier failed-message layout. An unfinished required row refuses startup and names `--start-data-migration`. The service never copies data. With no checkpoint store registered (RavenDB), it returns immediately. - **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race the offline command on the checkpoint version. - **`RecordHostOpenedOnTarget` on every writer.** The main host, ingestion workers, and SQL imports register `RecordHostOpenedOnTarget` and stamp `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so rollback advice must never offer a free abort after a worker has written. @@ -77,14 +77,6 @@ The ingestion gate sees categories the copy never reached because `MigrationStar ## The required copy -Unresolved and retry-issued failed messages copy while the host is closed because the copy cannot merge with live writes. `FailedMessagesWriterBase` inserts only messages the target lacks and never updates one it holds. From the moment the host opens, ingestion writes to the same rows: - -- **A retry confirmation finds no row.** `ResolveRetriedMessages` updates only rows already in SQL, and `FailedMessageBatchWriter.ResolveRetried` deletes the retry claim either way. The copy then inserts the message as still failing, though its retry succeeded. -- **A message that fails again loses its history.** Ingestion upserts a row holding only the new attempt. The copy then finds the id present and drops the RavenDB row, with its earlier attempts, first failure time and retry-issued status. -- **Group actions see part of the list.** Retrying or archiving a group in ServicePulse acts only on the messages copied so far. - -Archived and resolved messages are safe to copy after the host opens. If one of them fails again, the live unresolved row should win, and an insert-only copy gives exactly that. Copying unresolved messages in the background would need a writer that merges by attempt time, and retry confirmations held until their row arrives. Both change live ingestion. - For developer smoke tests, Debug builds honor `SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED=true` in both `--start-data-migration` and normal startup. Release builds ignore this override. ```mermaid @@ -106,7 +98,7 @@ sequenceDiagram Checks->>Target: Open Start->>Start: IMigrationCheckpointStore.ReadAll Start->>Source: Open - Start->>Checks: source.ContributedChecks(), empty for RavenDB + Start->>Checks: source.ContributedChecks(), UnfinishedOperationsCheck Start->>Engine: RunRequiredCategories under ClosedWindowProgress Engine-->>Start: one checkpoint per category Start->>Start: ReportWhatTheCopyLeftBehind @@ -140,8 +132,8 @@ Each position in the sequence is deliberate: > [!IMPORTANT] > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexes`. 7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the command skips the copy and succeeds. The operator can then start ServiceControl; its online copier handles any later source outage independently. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. -8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. -9. **Checkpoint initialization.** The command seeds `NotStarted` rows for required and selected optional categories before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. +8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` contributes `UnfinishedOperationsCheck`, which streams one document from each of `RetryBatches/`, `ArchiveOperations/` and `UnarchiveOperations/`. These prefixes cover retry batches and the forwarding marker, plus archive and unarchive operation documents and their batches. Any match refuses the run before checkpoint initialization or copying, names an example document from each unfinished family, and instructs the operator to restart on RavenDB and let the work finish. Completed retry history and retry claims use different prefixes and do not cause this refusal. A refusal here is never tolerated. +9. **Checkpoint initialization.** `RunRequiredCopy` first refuses checkpoints from an earlier failed-message layout. The command then seeds `NotStarted` rows for the selected optional categories and after them the required categories, before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. 10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. 11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. 12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row with the explanation `HarmlessSkipExplanations` holds for each harmless reason. A test fails unless that map covers exactly the `IsBenign` reasons. @@ -152,11 +144,14 @@ Each position in the sequence is deliberate: `OptionalCategoryCopier` is a `BackgroundService` registered by `RunCommand` only on the main instance. It waits for `ApplicationStarted`, then reads checkpoints. With no unfinished optional rows, it exits without creating a source or opening a target. -- It runs the selected, supported optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. The offline command has already seeded the rows that identify the online work. +- It runs the selected, supported optional categories in registry order, `UnresolvedFailedMessages`, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. The offline command has already seeded the rows that identify the online work. - It creates and opens its own source only when a supported category is still copying. On failure it calls `RecordSourceOutage` and exits for this start. It does not depend on in-process state from `--start-data-migration`, which ran in another process. +- It reruns source-contributed checks before copying. The RavenDB unfinished-operation check should still pass because the source is read-only and new operations run on SQL. A refusal stops the online copy for that start, not ServiceControl. - An exception leaves the category `InProgress` with `LastError`, untouched until the next start, so a transient failure never forces an abandon. -Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. +`UnresolvedFailedMessages` has no window. `MigrationEngineOptions.FromSettings` selects it even when both windows are zero. + +Windowing for the other two: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. > [!IMPORTANT] > Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. The online startup path must validate unchanged windows too, because `--start-data-migration` no longer runs during normal startup. @@ -174,7 +169,7 @@ Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader | Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | | Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | | Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | -| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; the two licensing endpoint readers use `ThroughputDocuments.Endpoints`; single-document readers load by id | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessageStatusReader` streams a query on `FailedMessageViewIndex` for its statuses ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does, and `FailedMessagesReader` then reads the claim holders, see [Readers and writers](#readers-and-writers); the two licensing endpoint readers use `ThroughputDocuments.Endpoints`; single-document readers load by id | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both paths resume on the same document id cursor | | Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyMissing` after one attempt, a permanent fault no retry fixes, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | | Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold. A crash leaves orphan bodies, see [open decisions](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch) | | Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `AddLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips, merges and changes are logged after commit. `MigrationWriteResult.Merged` and `Changed` carry the batch's counts of each, which the engine adds up per category for the current run in `MigrationEngine.MergedAndChangedThisRun`; the checkpoint stores neither. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | @@ -182,7 +177,11 @@ Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader | Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | | Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | -Failed message body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. `FailedErrorImportsWriter` places a failed error import's body by the SQL import store's own rule, inline up to `MaxBodySizeToStore` and otherwise external only, under `FailedErrorImportEntity.ExternalBodyId`, also before the transaction opens. +`FailedMessagesWriterBase` inserts only messages the target lacks, so an existing SQL row always wins. Live ingestion can therefore replace RavenDB's earlier failure history with the attempt it stores. The overview describes the [accepted edge cases](ravendb-to-sql-migration-overview.md#writing-to-sql). + +Failed message body placement lives in `FailedMessagesWriterBase`, shared by `FailedMessagesWriter` and `UnresolvedFailedMessagesWriter`, and by the archived and resolved writer when it exists. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. `FailedErrorImportsWriter` places a failed error import's body by the SQL import store's own rule, inline up to `MaxBodySizeToStore` and otherwise external only, under `FailedErrorImportEntity.ExternalBodyId`, also before the transaction opens. + +Every `IBodyStoragePersistence` backend writes a body only if none exists for the id: the file system moves a temporary file without overwrite, Azure Blob uploads with `IfNoneMatch`, and S3 uploads with `IfNoneMatch = "*"`, treating a 412 precondition failure as an existing body. The first writer wins, which relies on one `UniqueMessageId` always carrying the same bytes. An S3-compatible store that ignores the header keeps the check-then-upload race. Invariants enforced across class boundaries: @@ -283,13 +282,14 @@ Every reader derives from `MigrationCategoryReader` and every writer | 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. The reader gives one batch per endpoint from `ThroughputDocuments.Endpoints` with a batch size of one, and ignores the requested batch size, because one endpoint can hold a year or more of days. Each batch's cursor is its endpoint's id. The writer keeps the last row of a day repeated for one source endpoint, compared ordinally. `AddLicensingEndpointThroughput` splits the batch into statements and adds each day's count to the stored one, summing endpoints whose names the key column treats as the same, in one batch or across batches. A re-read is not idempotent, so a resume relies on each batch committing with its checkpoint, and a retry deletes the copied rows first | | 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | | 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | -| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator`. `FailedMessageStream.ResolveRetryClaims` looks up each batch's RavenDB retry claims in one request and gives every retry-issued row the `RetryClaimHeld` metadata key. The writer writes a claim only for a retry-issued message it inserts whose flag is true, so the claims mirror RavenDB's, and fails the batch when the flag is missing or not a bool | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator`. `FailedMessagesReader` and `UnresolvedFailedMessagesReader` both derive from `FailedMessageStatusReader`, which streams the category's statuses in id order through `FailedMessageViewIndex` after waiting for the index, or through `RavenDocumentStream.ByPrefix` when the index cannot be trusted, with the document id as the cursor either way. `FailedMessagesReader` reads in two phases. Phase one is the retry-issued messages, where `FailedMessageStream.ResolveRetryClaims` looks up each batch's claims in one request and sets `RetryClaimHeld` on every row. Phase two is `FailedMessageStream.ClaimHolders`, which streams the `FailedMessageRetries/` documents by id prefix and, per batch of claims, queries the non-retry-issued messages they name by `id()`, with `RetryClaimHeld` true and no index built. A claim whose message is gone makes no row, and memory stays bounded by the batch size. The phase-two cursor is `claim-holder:` and the last claim id. A cursor without the prefix, which is the end of phase one, runs phase one after it and then starts phase two. A prefixed cursor skips phase one and its document-existence check. The writer, through `FailedMessagesWriterBase`, inserts a claim for every inserted row whose flag is true, so the claims mirror RavenDB's. A retry-issued row with no flag fails the batch, and any other status with no flag means no claim. The writer counts retry-issued rows and rows with a claim | | 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | | 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | The RavenDB reader flags a check whose id and category match one of the RavenDB persister's own six checks with the `SourceStorageCheck` metadata key, and the writer skips a flagged row as `SourceStorageCheck` without knowing any id. A RavenDB audit instance's dirty memory result reports under another category, so it is copied | | 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | -| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment`. Reports group ids that differ only in trailing spaces as merges on SQL Server. Reports each copied comment on a group with no copied failed message as a change, because the retention sweep deletes it | -| 1 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | -| 2 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | +| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link, and never depends on an optional category, so a required category never waits for work that cannot run until the host opens. Skips blanks as `BlankGroupComment`. Reports group ids that differ only in trailing spaces as merges on SQL Server. Reports each copied comment on a group with no copied failed message as a change, because the retention sweep deletes it. For a group the background copy has not reached yet, that loss is an accepted edge case | +| 1 | `UnresolvedFailedMessages` | Optional | `UnresolvedFailedMessagesReader`, `UnresolvedFailedMessagesWriter` | Carries bodies, no window, and attaches no claim state. `MigrationEngineOptions.FromSettings` always selects it, and the engine runs it as optional: it throttles between batches, and an exception leaves it copying with `LastError` until the next start. `FailedMessagesWriterBase` inserts only messages the target lacks. The writer counts unresolved rows. `RavenMigrationSource.ReadBody` reads the body attachment for both failed-message categories | +| 2 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | +| 3 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | ## Progress reporting diff --git a/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs index b0ed4e2e08..ed509297d7 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs @@ -21,7 +21,7 @@ public async Task A_small_text_body_stays_inline_and_is_never_written_externally var content = Encoding.UTF8.GetBytes("1"); await SeedSourceWithBody(failure.ToFailedMessage(), content, "text/xml"); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); @@ -47,7 +47,7 @@ public async Task A_large_text_body_leaves_a_search_prefix_inline_and_writes_the var content = Encoding.UTF8.GetBytes(text); await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/json"); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); @@ -81,7 +81,7 @@ public async Task A_binary_body_goes_external_with_nothing_left_inline() var content = new byte[] { 0x08, 0x00, 0x96, 0x01, 0xFF }; await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/x-protobuf"); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); var written = RecordedBodies.Written.Single(); @@ -112,9 +112,9 @@ public async Task A_message_with_no_body_attachment_is_left_out_entirely() await SeedSourceWithBody(readable.ToFailedMessage(), Encoding.UTF8.GetBytes("1"), "text/xml"); await SeedSource(($"FailedMessages/{missing.UniqueMessageId}", missing.ToFailedMessage())); - var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); - var checkpoint = await ReadCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var checkpoint = await ReadCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages); using (Assert.EnterMultipleScope()) { @@ -132,7 +132,7 @@ public async Task A_zero_length_body_is_copied_rather_than_treated_as_missing() var failure = new IngestedFailure(); await SeedSourceWithBody(failure.ToFailedMessage(), [], "text/plain"); - var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); @@ -158,7 +158,7 @@ await SeedSourceWithBodies( (inline.ToFailedMessage(), inlineContent, "text/xml"), (external.ToFailedMessage(), externalContent, "application/x-protobuf")); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); Assert.That(await Fetch(inline.UniqueMessageId), Is.EqualTo(inlineContent)); Assert.That(await Fetch(external.UniqueMessageId), Is.EqualTo(externalContent), "the body was stored under an id TryFetch does not resolve"); diff --git a/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs index 0d5d7751b2..64ae3590af 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs @@ -30,6 +30,7 @@ await SeedSourceGroupComments( (spaces.Groups.Single().Id, " "), ("group-with-no-copied-message", "Raised with the shipping team")); await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var result = await CopyCategory(MigrationCategoryIds.GroupComments); var comments = (await GroupsStore.GetUnresolvedGroupsByClassifier(kept.Groups.Single().Type, null, new PagingInfo())).ToDictionary(view => view.Id, view => view.Comment); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs index c3d218938d..d185a03f5d 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs @@ -14,13 +14,13 @@ class When_a_batch_exceeds_one_statement : MigrationAcceptanceTest [Test] public async Task Copies_more_messages_than_fit_in_one_statement() { - var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedFailedMessages); var batchSize = await Target.BatchSizeFor(category); var failures = Enumerable.Range(0, (batchSize * 2) + 1).Select(_ => new IngestedFailure()).ToArray(); await SeedSourceWithBodies([.. failures.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]); - var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); Assert.That(result.Copied, Is.EqualTo(failures.Length)); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs index a553e8cc93..1caa4e7e05 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs @@ -8,6 +8,7 @@ namespace ServiceControl.Migration.AcceptanceTests; using Microsoft.Extensions.Logging.Abstractions; using NUnit.Framework; using ServiceControl.Hosting.Commands; +using ServiceControl.MessageFailures; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.Tests; @@ -19,7 +20,7 @@ class When_the_copy_merges_or_changes_rows : MigrationAcceptanceTest public async Task The_category_summary_counts_changed_and_merged_rows_apart_from_the_rest() { var retried = new IngestedFailure { MessageType = new string('m', 600) }; - await SeedSourceWithBody(retried.ToFailedMessage(numberOfAttempts: 2), retried.Body, retried.ContentType); + await SeedSourceWithBody(retried.ToFailedMessage(FailedMessageStatus.RetryIssued, numberOfAttempts: 2), retried.Body, retried.ContentType); await SeedSourceSubscriptions( new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs index 25c51dc849..406fee6b5e 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs @@ -28,7 +28,7 @@ public async Task Fills_every_column_the_newest_attempt_supplies() var failure = new IngestedFailure { FailingEndpointAddress = "Sales@MACHINE" }; await SeedFailure(failure); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); @@ -58,7 +58,7 @@ public async Task Collapses_attempt_history_to_the_newest_attempt() var failure = new IngestedFailure(); await SeedFailure(failure, numberOfAttempts: 5); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); @@ -77,7 +77,7 @@ public async Task A_resumed_copy_leaves_the_rows_the_first_run_wrote() var failure = new IngestedFailure(); await SeedFailure(failure); - var first = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var first = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); await QueryTarget(async db => { @@ -87,9 +87,9 @@ await QueryTarget(async db => return existing; }); - await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages); - var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var second = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var row = await GetFailedMessage(failure.UniqueMessageId); @@ -105,14 +105,14 @@ await QueryTarget(async db => } [Test] - public async Task Counts_only_the_statuses_this_category_owns() + public async Task Counts_only_the_rows_each_category_owns() { var unresolved = new IngestedFailure(); var retryIssued = new IngestedFailure(); await SeedFailure(unresolved); await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var offline = await CopyCategory(OfflineCategoryId); // Written straight into the target, because this build has no category that copies archived messages. await QueryTarget(async db => @@ -128,12 +128,22 @@ await QueryTarget(async db => return await db.SaveChangesAsync(); }); - var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var offlineCategory = MigrationCategoryRegistry.All.Single(entry => entry.Id == OfflineCategoryId); using (Assert.EnterMultipleScope()) { - Assert.That(await Target.Count(category), Is.EqualTo(2), "the category's count takes in the archive's rows unless it honours the category's statuses"); - Assert.That(await QueryTarget(db => db.FailedMessages.CountAsync()), Is.EqualTo(3), "the filter is doing the work, not an empty table"); + Assert.That(offline.Copied, Is.EqualTo(1), "an unresolved message without a claim waits for the copy that runs with ServiceControl open"); + Assert.That(await Target.Count(offlineCategory), Is.EqualTo(1), "the category's count takes in the archive's rows unless it honours the category's cohort"); + Assert.That(await QueryTarget(db => db.FailedMessages.CountAsync()), Is.EqualTo(2), "the filter is doing the work, not an empty table"); + } + + var online = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); + var onlineCategory = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(online.Copied, Is.EqualTo(1)); + Assert.That(await Target.Count(onlineCategory), Is.EqualTo(1)); } } @@ -143,7 +153,7 @@ public async Task A_copied_message_keeps_its_failure_groups() var failure = new IngestedFailure(); await SeedFailure(failure); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var groups = await QueryTarget(db => db.FailedMessageGroups .AsNoTracking() @@ -169,9 +179,10 @@ public async Task A_retry_by_id_after_the_copy_stages_the_unresolved_message_and await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); - var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var offline = await CopyCategory(OfflineCategoryId); + var online = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); - Assert.That(copy.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + Assert.That(offline.Copied + online.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); string[] ids = [unresolved.UniqueMessageIdString, retryIssued.UniqueMessageIdString]; var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-by-id", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); @@ -190,7 +201,7 @@ public async Task A_retry_issued_message_resent_from_pending_retries_after_the_c await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); - var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var copy = await CopyCategory(OfflineCategoryId); using (Assert.EnterMultipleScope()) { @@ -217,12 +228,12 @@ public async Task A_message_already_in_the_target_gets_no_group_rows_from_the_co var failure = new IngestedFailure(); await SeedFailure(failure); - await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); await QueryTarget(db => db.FailedMessageGroups.Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId).ExecuteDeleteAsync()); - await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages); - var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var second = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var groups = await QueryTarget(db => db.FailedMessageGroups.AsNoTracking().CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)); @@ -240,6 +251,7 @@ public async Task Answers_for_a_copied_message_on_the_URL_it_had_on_RavenDB() await SeedFailure(failure); await MigrateThenRunHost(); + await WaitUntil(async () => (await ReadCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages))?.State.IsFinished() == true, "the unresolved copy finished"); var response = await HttpClient.GetAsync($"api/errors/last/{failure.UniqueMessageIdString}"); @@ -259,18 +271,20 @@ public async Task Answers_for_a_copied_message_on_the_URL_it_had_on_RavenDB() } [Test] - public async Task The_required_copy_completes_the_category_before_the_host_opens() + public async Task The_required_copy_completes_the_category_before_the_host_opens_and_leaves_the_unresolved_messages_to_the_online_copy() { - var failures = Enumerable.Range(0, 3).Select(_ => new IngestedFailure()).ToArray(); + var retryIssued = Enumerable.Range(0, 3).Select(_ => new IngestedFailure()).ToArray(); - foreach (var failure in failures) + foreach (var failure in retryIssued) { - await SeedFailure(failure); + await SeedFailure(failure, FailedMessageStatus.RetryIssued); } + await SeedFailure(new IngestedFailure()); + await MigrateThenRunHost(); - var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + var checkpoint = await ReadCheckpoint(OfflineCategoryId); using (Assert.EnterMultipleScope()) { @@ -279,21 +293,25 @@ public async Task The_required_copy_completes_the_category_before_the_host_opens Assert.That(checkpoint.SkippedCount, Is.Zero); Assert.That(checkpoint.SourceTotal, Is.Null, "the engine no longer counts the source before copying"); } + + await WaitUntil(async () => (await ReadCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages))?.State.IsFinished() == true, "the unresolved copy finished"); + + Assert.That((await ReadCheckpoint(MigrationCategoryIds.UnresolvedFailedMessages)).CopiedCount, Is.EqualTo(1)); } [Test] public async Task Records_each_skip_under_the_reason_that_caused_it() { var good = new IngestedFailure(); - await SeedFailure(good); + await SeedFailure(good, FailedMessageStatus.RetryIssued); var unparseable = new IngestedFailure(); - var unparseableDocument = unparseable.ToFailedMessage(); + var unparseableDocument = unparseable.ToFailedMessage(FailedMessageStatus.RetryIssued); unparseableDocument.UniqueMessageId = "not-a-guid"; await SeedSourceWithBody(unparseableDocument, unparseable.Body, unparseable.ContentType); var attemptless = new IngestedFailure(); - var attemptlessDocument = attemptless.ToFailedMessage(); + var attemptlessDocument = attemptless.ToFailedMessage(FailedMessageStatus.RetryIssued); attemptlessDocument.ProcessingAttempts.Clear(); await SeedSourceWithBody(attemptlessDocument, attemptless.Body, attemptless.ContentType); @@ -301,7 +319,7 @@ public async Task Records_each_skip_under_the_reason_that_caused_it() var refused = Assert.ThrowsAsync(async () => await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); - var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + var checkpoint = await ReadCheckpoint(OfflineCategoryId); using (Assert.EnterMultipleScope()) { @@ -338,7 +356,7 @@ await SeedSourceWithBodies( (badTimeSentDocument, badTimeSent.Body, badTimeSent.ContentType), (badHostIdDocument, badHostId.Body, badHostId.ContentType)); - var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); using (Assert.EnterMultipleScope()) { @@ -364,7 +382,7 @@ public async Task A_failure_stored_without_its_time_is_skipped_when_newest_and_l await DeleteStoredTimeOfFailure(newestHasNoTime, attempt: 0); await DeleteStoredTimeOfFailure(olderHasNoTime, attempt: 0); - var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await CopyCategory(MigrationCategoryIds.UnresolvedFailedMessages); var copied = await GetFailedMessage(olderHasNoTime.UniqueMessageId); @@ -385,6 +403,8 @@ Task DeleteStoredTimeOfFailure(IngestedFailure failure, int attempt) => null, new PatchRequest { Script = $"delete this.ProcessingAttempts[{attempt}].FailureDetails.TimeOfFailure;" })); + const string OfflineCategoryId = MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; + // Seeded with its body, because the copy leaves out a message whose body the source does not hold. Task SeedFailure(IngestedFailure failure, FailedMessageStatus status = FailedMessageStatus.Unresolved, int numberOfAttempts = 1) => SeedSourceWithBody(failure.ToFailedMessage(status, numberOfAttempts), failure.Body, failure.ContentType); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index 6d8297c073..d79c6eb124 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -27,6 +27,7 @@ sealed class EFCoreMigrationTarget( new KnownEndpointsWriter(migrationDialect), new EndpointSettingsWriter(migrationDialect), new FailedMessagesWriter(migrationDialect, storagePersistence, settings), + new UnresolvedFailedMessagesWriter(migrationDialect, storagePersistence, settings), new LicensingEndpointsWriter(migrationDialect), new LicensingThroughputWriter(migrationDialect), new LicensingReportMasksWriter(migrationDialect), diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs index a1f596b00c..070312530e 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs @@ -9,11 +9,11 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.DataMigration; /// -/// Writes failed messages, their failure group rows and a retry claim for each retry-issued message the source still -/// held one for, inserting only messages the target does not already hold and never updating one it does. The claims +/// Writes failed messages, their failure group rows and a retry claim for each message the source still held one for, +/// inserting only messages the target does not already hold and never updating one it does. The claims /// mirror the source, so a retry by id behaves as it did before the move. A retry-issued row with no bool RetryClaimHeld -/// value fails the batch. Each category that copies failed messages derives from this and names its category id and the -/// statuses it copies. +/// value fails the batch. Each category that copies failed messages derives from this and names its category id and +/// which target rows it counts. /// abstract class FailedMessagesWriterBase(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter { @@ -21,17 +21,14 @@ abstract class FailedMessagesWriterBase(IMigrationSqlDialect migrationDialect, I public static readonly Guid MigratedRetryClaimBatchId = new("5c1b7a0e-2f4d-4e8a-9b36-7d0c4f1e8a52"); /// - /// The statuses this category copies. counts only rows with one of these statuses, because - /// both failed message categories share one table. + /// The rows of the shared failed message table that counts for this category. /// - protected abstract IReadOnlyCollection Statuses { get; } + protected abstract IQueryable Copied(ServiceControlDbContext dbContext); public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => - dbContext.FailedMessages - .Where(message => Statuses.Contains(message.Status)) - .LongCountAsync(cancellationToken); + Copied(dbContext).LongCountAsync(cancellationToken); protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, FailedMessage Document)> documents, CancellationToken cancellationToken = default) { @@ -74,7 +71,7 @@ protected override async Task PrepareDocuments(ServiceControlDbCo externalBodies.Add((row.SourceId, translation.Row, body)); } - if (translation.Row.Status == FailedMessageStatus.RetryIssued && SourceHeldRetryClaim(row)) + if (SourceHeldRetryClaim(row, translation.Row.Status)) { heldClaims.Add(translation.Row.UniqueMessageId); } @@ -129,10 +126,11 @@ .. translation.Groups ]; // Throws rather than guess, because a wrong guess either blocks a retry by id or lets one send the message twice. - static bool SourceHeldRetryClaim(MigrationRow row) => + static bool SourceHeldRetryClaim(MigrationRow row, FailedMessageStatus status) => row.Metadata.GetValueOrDefault(MigrationMetadataKeys.RetryClaimHeld) switch { bool held => held, + null when status != FailedMessageStatus.RetryIssued => false, null => throw new InvalidOperationException($"Row '{row.SourceId}' is retry-issued but arrived without a {MigrationMetadataKeys.RetryClaimHeld} value, which the source resolves for every retry-issued failed message"), var other => throw new InvalidOperationException($"Row '{row.SourceId}' carries a {other.GetType().FullName} as its {MigrationMetadataKeys.RetryClaimHeld} value, where the source sets a {typeof(bool).FullName}") }; @@ -188,12 +186,26 @@ await migrationDialect.InsertMissing(dbContext, [.. inserted } /// -/// Writes the unresolved and retry-issued failed messages. +/// Writes the retry-issued failed messages and the other messages that hold a retry claim, whatever their status. /// sealed class FailedMessagesWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : FailedMessagesWriterBase(migrationDialect, storagePersistence, persisterSettings) { public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; - protected override IReadOnlyCollection Statuses => MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses; + protected override IQueryable Copied(ServiceControlDbContext dbContext) => + dbContext.FailedMessages.Where(message => message.Status == FailedMessageStatus.RetryIssued + || dbContext.FailedMessageRetries.Any(claim => claim.UniqueMessageId == message.UniqueMessageId)); +} + +/// +/// Writes the unresolved failed messages. +/// +sealed class UnresolvedFailedMessagesWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) + : FailedMessagesWriterBase(migrationDialect, storagePersistence, persisterSettings) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedFailedMessages; + + protected override IQueryable Copied(ServiceControlDbContext dbContext) => + dbContext.FailedMessages.Where(message => message.Status == FailedMessageStatus.Unresolved); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs index 3179f62c16..39a8c4788e 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs @@ -59,7 +59,7 @@ protected override async Task PrepareDocuments(ServiceControlDbCo }; } - // Group comments copy after the unresolved failed messages, so the groups the target holds now are the groups it holds when ServiceControl opens. + // Group comments copy after the required failed messages. The unresolved messages without a retry claim arrive after ServiceControl opens, so a group that only they belong to is missing here. async Task> CommentsTheSweepWillDelete(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, GroupCommentEntity Row)> inserted, CancellationToken cancellationToken) { string[] groupIds = [.. inserted.Select(comment => comment.Row.GroupId).Distinct(StringComparer.Ordinal)]; @@ -73,7 +73,7 @@ protected override async Task PrepareDocuments(ServiceControlDbCo return [.. inserted .Where(comment => !held.Contains(comment.Row.GroupId)) .Select(comment => (comment.SourceId, $"The group '{comment.Row.GroupId}' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + - "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + "Its messages have not been copied yet, are archived or resolved, are gone from the source, or were skipped by the copy. " + $"The comment was: '{comment.Row.Comment}'"))]; } } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/S3BodyStoragePersistence.cs b/src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/S3BodyStoragePersistence.cs index f5194dab85..1172f2cdb5 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/S3BodyStoragePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/S3BodyStoragePersistence.cs @@ -29,7 +29,7 @@ public async Task WriteBody(string bodyId, ReadOnlyMemory body, string con { var key = Key(bodyId); - // Bodies are immutable, so an existing object is already correct. + // Bodies are immutable, so an existing object is already correct. This skips the upload; the conditional put below decides a race. if (await Exists(key, cancellationToken)) { return; @@ -43,13 +43,22 @@ public async Task WriteBody(string bodyId, ReadOnlyMemory body, string con BucketName = bucketName, Key = key, InputStream = new ReadOnlyStream(payload), - ContentType = contentType + ContentType = contentType, + // A writer that stored this body after the check above wins, and a live body is never replaced. + IfNoneMatch = "*" }; request.Metadata.Add("format-version", FormatVersion); request.Metadata.Add("body-size", body.Length.ToString()); request.Metadata.Add("is-compressed", (compressed is not null).ToString()); - await client.PutObjectAsync(request, cancellationToken); + try + { + await client.PutObjectAsync(request, cancellationToken); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) + { + // An object for this immutable body already exists. + } } public async Task ReadBody(string bodyId, CancellationToken cancellationToken = default) diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs index 3f32f8227f..7678e631e7 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Raven.Client; @@ -20,7 +21,7 @@ class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) { /// /// Builds the function that turns each streamed failed message into a row. The function returns null for a message whose - /// status is not in , and throws for a status that neither + /// status is not in , and throws for a status that no /// failed message category copies. /// public Func, MigrationRow?> Project(IReadOnlyCollection statuses) => @@ -28,9 +29,9 @@ class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) { var status = result.Document.Status; - if (!MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Contains(status) && !MigrationCategoryRegistry.ArchivedAndResolvedStatuses.Contains(status)) + if (!CopiedStatuses.Contains(status)) { - throw new InvalidOperationException($"The failed message document '{result.Id}' has the status value {(int)status}, which neither failed message category copies, so it would be left behind without a skip to show for it."); + throw new InvalidOperationException($"The failed message document '{result.Id}' has the status value {(int)status}, which no failed message category copies, so it would be left behind without a skip to show for it."); } if (!statuses.Contains(status)) @@ -50,6 +51,62 @@ class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) }); }; + /// + /// Every status some failed message category copies. + /// + public static readonly FailedMessageStatus[] CopiedStatuses = + [.. MigrationCategoryRegistry.RetryIssuedStatuses, .. MigrationCategoryRegistry.UnresolvedStatuses, .. MigrationCategoryRegistry.ArchivedAndResolvedStatuses]; + + /// + /// The cursor of a batch from , which follows the id of the last retry claim document it walked. It cannot be a + /// message id, so a resume tells the retry-issued messages apart from the claim holders that come after them. + /// + public const string ClaimHolderCursorPrefix = "claim-holder:"; + + /// + /// Reads the messages that hold a retry claim and are not retry-issued, whatever their status, after the claim document id in + /// , each row carrying as true. The claim documents + /// stream in id order through , so memory stays bounded by the batch size. + /// A claim whose message is gone makes no row. Every batch's cursor is and the last claim document id it walked. + /// + public async IAsyncEnumerable ClaimHolders(string categoryId, string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var claimPrefix = RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(""); + var project = Project(CopiedStatuses); + + // The claim document stands in as the row so the stream can batch it; the message it names replaces it below. + await foreach (var claims in RavenDocumentStream.ByPrefix( + lifecycle, categoryId, lifecycle.Settings.DatabaseName, claimPrefix, resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, + result => new MigrationRow(result.Id, result.Document, new Dictionary()), cancellationToken)) + { + string[] messageIds = [.. claims.Rows.Select(claim => FailedMessageIdGenerator.MakeDocumentId(claim.SourceId[claimPrefix.Length..]))]; + var found = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (messageIds.Length > 0) + { + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + // Queried by id on the collection, which builds no index. A load would not do, because a session that does not track cannot hand back the document's metadata. + var query = session.Advanced.AsyncDocumentQuery(collectionName: FailedMessageIdGenerator.CollectionName).WhereIn("id()", messageIds.Cast()); + await using var results = await session.Advanced.StreamAsync(query, cancellationToken); + + // The metadata is valid only while the stream is on its result, so each result becomes a row as it arrives. + while (await results.MoveNextAsync()) + { + // A retry-issued message is already in the batches before these. + if (results.Current.Document.Status != FailedMessageStatus.RetryIssued) + { + var row = project(results.Current)!; + found[results.Current.Id] = row with { Metadata = new Dictionary(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = true } }; + } + } + } + + // A claim can outlive its message, which then has no row. + yield return new MigrationBatch([.. messageIds.Where(found.ContainsKey).Select(id => found[id])], ClaimHolderCursorPrefix + claims.Cursor); + } + } + /// /// Returns the batch with every retry-issued row carrying , true when /// RavenDB still holds a retry claim for the message. It looks up every claim in the batch with one request, and returns a diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index a77fe39612..78e1b08b40 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -28,6 +28,7 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig new KnownEndpointsReader(lifecycle), new EndpointSettingsReader(lifecycle), new FailedMessagesReader(lifecycle), + new UnresolvedFailedMessagesReader(lifecycle), new LicensingEndpointsReader(lifecycle), new LicensingThroughputReader(lifecycle), new LicensingReportMasksReader(lifecycle), @@ -45,7 +46,7 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); - public IReadOnlyList ContributedChecks() => []; + public IReadOnlyList ContributedChecks() => [new UnfinishedOperationsCheck(lifecycle)]; public async Task Describe(CancellationToken cancellationToken = default) { @@ -103,7 +104,7 @@ public IAsyncEnumerable Read( public async Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) { // Another category may keep its body inside the document, where a body attachment read would find nothing and skip every row as BodyMissing. - if (category.Id != MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) + if (category.Id is not (MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages or MigrationCategoryIds.UnresolvedFailedMessages)) { throw new NotSupportedException($"The migration source does not read bodies as attachments for the '{category.Id}' category."); } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessageStatusReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessageStatusReader.cs new file mode 100644 index 0000000000..a96980273c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessageStatusReader.cs @@ -0,0 +1,192 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions; +using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the failed messages with one of a category's statuses. The archived and resolved messages in the same collection can +/// outnumber the others many times, so it queries for just these statuses instead of reading them all. +/// +abstract class FailedMessageStatusReader(RavenReadOnlySourceLifecycle lifecycle, IReadOnlyCollection statuses) : MigrationCategoryReader(lifecycle) +{ + /// + /// Waits for to catch up, then streams the messages with the category's statuses in id + /// order, starting after . When the index is missing, errored or disabled, or does not hold + /// every failed message with a status some category copies, it logs a warning and reads the whole collection instead. + /// Both ways use the document id as the cursor, so a copy can switch from one to the other between runs. + /// Throws when the source holds no document with the id in , + /// or when a failed message has a status no category copies. + /// + protected async IAsyncEnumerable ReadStatuses(string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var databaseName = Lifecycle.Settings.DatabaseName; + var project = new FailedMessageStream(Lifecycle).Project(statuses); + + var batches = await IndexHoldsEveryCopiedFailedMessage(databaseName, cancellationToken) + ? ThroughIndex(databaseName, resumeAfter, batchSize, project, cancellationToken) + : Documents($"{FailedMessageIdGenerator.CollectionName}/", resumeAfter, batchSize, project, cancellationToken); + + await foreach (var batch in batches) + { + yield return batch; + } + } + + async IAsyncEnumerable ThroughIndex( + string databaseName, + string? resumeAfter, + int batchSize, + Func, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + await RavenDocumentStream.RefuseMissingCursor(session, CategoryId, databaseName, resumeAfter, cancellationToken); + + var query = Query(session).WhereIn(nameof(FailedMessage.Status), StatusValues(statuses)); + + if (resumeAfter is not null) + { + query = query.AndAlso().WhereGreaterThan(DocumentId, resumeAfter); + } + + await using var enumerator = await session.Advanced.StreamAsync(query.OrderBy(DocumentId).OfType(), cancellationToken); + + await foreach (var batch in RavenDocumentStream.Batches(enumerator, resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, project)) + { + yield return batch; + } + } + + // A full read passes a document RavenDB could not index to the target to skip, and refuses a status no category copies, + // but a query on the index sees neither, so either kind of document sends the copy back to a full read. + async Task IndexHoldsEveryCopiedFailedMessage(string databaseName, CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + while (true) + { + var index = await WaitForIndex(databaseName, cancellationToken); + + if (index is null || index.State is IndexState.Error or IndexState.Disabled) + { + logger.LogWarning("The RavenDB index {IndexName} in {DatabaseName} is {IndexState}, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which makes the copy slower", IndexName, databaseName, index?.State.ToString() ?? "missing", CategoryId); + return false; + } + + // Expiration keeps deleting archived and resolved messages during the copy, so the collection is counted before and + // after the index count, because a deletion in between could hide a document the index left out. A stale index count + // means something changed after the wait, so it waits and counts again. + var storedBefore = await StoredFailedMessages(databaseName, cancellationToken); + int indexed; + QueryStatistics statistics; + try + { + indexed = await Query(session).Statistics(out statistics).WhereIn(nameof(FailedMessage.Status), StatusValues(FailedMessageStream.CopiedStatuses)).CountAsync(cancellationToken); + } + catch (RavenException) + { + // RavenDB can mark the index errored after the wait saw it working, and then refuses the query. The next pass falls back to a full read. + if ((await WaitForIndex(databaseName, cancellationToken))?.State is IndexState.Error or IndexState.Disabled) + { + continue; + } + + throw; + } + + var storedAfter = await StoredFailedMessages(databaseName, cancellationToken); + + if (statistics.IsStale) + { + continue; + } + + if (indexed == storedBefore && indexed == storedAfter) + { + return true; + } + + logger.LogWarning("The RavenDB database {DatabaseName} held {StoredBefore} and then {StoredAfter} failed messages while its index {IndexName} held {Indexed} with a status a category copies, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which makes the copy slower. The counts differ when RavenDB could not index a document, which the index errors in RavenDB Studio name, when a document has a status no category copies, or when expiration deleted documents during the count", databaseName, storedBefore, storedAfter, IndexName, indexed, CategoryId); + return false; + } + } + + async Task StoredFailedMessages(string databaseName, CancellationToken cancellationToken) => + (await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken)) + .Collections.GetValueOrDefault(FailedMessageIdGenerator.CollectionName); + + // A query on an index that is behind leaves out what it has not indexed yet, which here means unresolved messages nothing + // would copy. It waits rather than halts, because halting would need an operator to retry for a wait that is normally seconds. + async Task WaitForIndex(string databaseName, CancellationToken cancellationToken) + { + var waiting = Stopwatch.StartNew(); + var nextLog = TimeSpan.Zero; + + while (true) + { + var statistics = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + var index = statistics.Indexes.SingleOrDefault(candidate => candidate.Name == IndexName); + + // A missing, errored or disabled index never catches up, so waiting for it would keep ServiceControl closed for good. + if (index is null || !index.IsStale || index.State is IndexState.Error or IndexState.Disabled) + { + return index; + } + + if (waiting.Elapsed >= nextLog) + { + await LogWait(databaseName, waiting.Elapsed, cancellationToken); + nextLog += LogEvery; + } + + await Task.Delay(PollEvery, cancellationToken); + } + } + + // A paused index stays stale with its state still Normal, so without this the log reads exactly like an index that is merely slow. + async Task LogWait(string databaseName, TimeSpan waited, CancellationToken cancellationToken) + { + var indexing = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexingStatusOperation(), cancellationToken); + var paused = indexing.Status == IndexRunningStatus.Paused + || indexing.Indexes.Any(index => index.Name == IndexName && index.Status == IndexRunningStatus.Paused); + + if (paused) + { + logger.LogWarning("Copying {CategoryId} is waiting for the RavenDB index {IndexName} in {DatabaseName}, but indexing is paused, so the copy waits until it is resumed in RavenDB Studio. Waited {Waited} so far", CategoryId, IndexName, databaseName, waited); + return; + } + + logger.LogInformation("Waiting for the RavenDB index {IndexName} in {DatabaseName} to catch up before copying {CategoryId}, waited {Waited} so far", IndexName, databaseName, CategoryId, waited); + } + + static IAsyncDocumentQuery Query(IAsyncDocumentSession session) => + session.Advanced.AsyncDocumentQuery(); + + static object[] StatusValues(IEnumerable statuses) => statuses.Select(status => (object)(int)status).ToArray(); + + static readonly string IndexName = new FailedMessageViewIndex().IndexName; + static readonly TimeSpan PollEvery = TimeSpan.FromSeconds(1); + static readonly TimeSpan LogEvery = TimeSpan.FromSeconds(30); + static readonly ILogger logger = LoggerUtil.CreateStaticLogger(); + + const string DocumentId = "id()"; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs index 1d32916de9..0a668e6b5d 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs @@ -4,194 +4,41 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Raven.Client.Documents.Commands; -using Raven.Client.Documents.Indexes; -using Raven.Client.Documents.Operations; -using Raven.Client.Documents.Operations.Indexes; -using Raven.Client.Documents.Session; -using Raven.Client.Exceptions; -using ServiceControl.Infrastructure; -using ServiceControl.MessageFailures; -using ServiceControl.MessageFailures.Api; using ServiceControl.Persistence.DataMigration; /// -/// Reads the failed messages that are still waiting on somebody: unresolved, or with a retry issued. This category is -/// copied while ServiceControl is closed, and the archived and resolved messages in the same collection can outnumber it -/// many times, so it queries for just these statuses instead of reading them all. +/// Reads the retry-issued messages and the messages that hold a retry claim, which are copied while ServiceControl is closed: every +/// retry-issued message, whether or not its retry claim remains, and then every other message that still holds a retry claim, +/// whatever its status. The unresolved messages without a claim are 's. /// -sealed class FailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +sealed class FailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : FailedMessageStatusReader(lifecycle, MigrationCategoryRegistry.RetryIssuedStatuses) { public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; /// - /// Waits for to catch up, then streams the unresolved and retry issued messages in id - /// order, starting after . When the index is missing, errored or disabled, or does not hold - /// every failed message with a status some category copies, it logs a warning and reads the whole collection instead. - /// Both ways use the document id as the cursor, so a copy can switch from one to the other between runs. - /// Throws when the source holds no document with the id in , - /// or when a failed message has a status no category copies. + /// Streams the retry-issued messages through , then the claim holders, + /// whose cursors start with . A resume from such a cursor skips the first part. /// public override async IAsyncEnumerable Read(string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var databaseName = Lifecycle.Settings.DatabaseName; var stream = new FailedMessageStream(Lifecycle); - var project = stream.Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses); + var inClaimHolders = resumeAfter?.StartsWith(FailedMessageStream.ClaimHolderCursorPrefix, StringComparison.Ordinal) == true; - var batches = await IndexHoldsEveryCopiedFailedMessage(databaseName, cancellationToken) - ? ThroughIndex(databaseName, resumeAfter, batchSize, project, cancellationToken) - : Documents($"{FailedMessageIdGenerator.CollectionName}/", resumeAfter, batchSize, project, cancellationToken); - - await foreach (var batch in batches) + if (!inClaimHolders) { - yield return await stream.ResolveRetryClaims(batch, cancellationToken); - } - } - - async IAsyncEnumerable ThroughIndex( - string databaseName, - string? resumeAfter, - int batchSize, - Func, MigrationRow?> project, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - using var session = Lifecycle.OpenSession(databaseName); - - await RavenDocumentStream.RefuseMissingCursor(session, CategoryId, databaseName, resumeAfter, cancellationToken); - - var query = Query(session).WhereIn(nameof(FailedMessage.Status), StatusValues(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses)); - - if (resumeAfter is not null) - { - query = query.AndAlso().WhereGreaterThan(DocumentId, resumeAfter); - } - - await using var enumerator = await session.Advanced.StreamAsync(query.OrderBy(DocumentId).OfType(), cancellationToken); - - await foreach (var batch in RavenDocumentStream.Batches(enumerator, resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, project)) - { - yield return batch; - } - } - - // A full read passes a document RavenDB could not index to the target to skip, and refuses a status no category copies, - // but a query on the index sees neither, so either kind of document sends the copy back to a full read. - async Task IndexHoldsEveryCopiedFailedMessage(string databaseName, CancellationToken cancellationToken) - { - using var session = Lifecycle.OpenSession(databaseName); - - while (true) - { - var index = await WaitForIndex(databaseName, cancellationToken); - - if (index is null || index.State is IndexState.Error or IndexState.Disabled) - { - logger.LogWarning("The RavenDB index {IndexName} in {DatabaseName} is {IndexState}, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer", IndexName, databaseName, index?.State.ToString() ?? "missing", CategoryId); - return false; - } - - // Expiration keeps deleting archived and resolved messages during the copy, so the collection is counted before and - // after the index count, because a deletion in between could hide a document the index left out. A stale index count - // means something changed after the wait, so it waits and counts again. - var storedBefore = await StoredFailedMessages(databaseName, cancellationToken); - int indexed; - QueryStatistics statistics; - try + await foreach (var batch in ReadStatuses(resumeAfter, batchSize, cancellationToken)) { - indexed = await Query(session).Statistics(out statistics).WhereIn(nameof(FailedMessage.Status), StatusValues(CopiedStatuses)).CountAsync(cancellationToken); + yield return await stream.ResolveRetryClaims(batch, cancellationToken); } - catch (RavenException) - { - // RavenDB can mark the index errored after the wait saw it working, and then refuses the query. The next pass falls back to a full read. - if ((await WaitForIndex(databaseName, cancellationToken))?.State is IndexState.Error or IndexState.Disabled) - { - continue; - } - - throw; - } - - var storedAfter = await StoredFailedMessages(databaseName, cancellationToken); - - if (statistics.IsStale) - { - continue; - } - - if (indexed == storedBefore && indexed == storedAfter) - { - return true; - } - - logger.LogWarning("The RavenDB database {DatabaseName} held {StoredBefore} and then {StoredAfter} failed messages while its index {IndexName} held {Indexed} with a status a category copies, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer. The counts differ when RavenDB could not index a document, which the index errors in RavenDB Studio name, when a document has a status no category copies, or when expiration deleted documents during the count", databaseName, storedBefore, storedAfter, IndexName, indexed, CategoryId); - return false; - } - } - - async Task StoredFailedMessages(string databaseName, CancellationToken cancellationToken) => - (await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken)) - .Collections.GetValueOrDefault(FailedMessageIdGenerator.CollectionName); - - // A query on an index that is behind leaves out what it has not indexed yet, which here means unresolved messages nothing - // would copy. It waits rather than halts, because halting would need an operator to retry for a wait that is normally seconds. - async Task WaitForIndex(string databaseName, CancellationToken cancellationToken) - { - var waiting = Stopwatch.StartNew(); - var nextLog = TimeSpan.Zero; - - while (true) - { - var statistics = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); - var index = statistics.Indexes.SingleOrDefault(candidate => candidate.Name == IndexName); - - // A missing, errored or disabled index never catches up, so waiting for it would keep ServiceControl closed for good. - if (index is null || !index.IsStale || index.State is IndexState.Error or IndexState.Disabled) - { - return index; - } - - if (waiting.Elapsed >= nextLog) - { - await LogWait(databaseName, waiting.Elapsed, cancellationToken); - nextLog += LogEvery; - } - - await Task.Delay(PollEvery, cancellationToken); } - } - // A paused index stays stale with its state still Normal, so without this the log reads exactly like an index that is merely slow. - async Task LogWait(string databaseName, TimeSpan waited, CancellationToken cancellationToken) - { - var indexing = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexingStatusOperation(), cancellationToken); - var paused = indexing.Status == IndexRunningStatus.Paused - || indexing.Indexes.Any(index => index.Name == IndexName && index.Status == IndexRunningStatus.Paused); + var claimHoldersAfter = inClaimHolders ? resumeAfter![FailedMessageStream.ClaimHolderCursorPrefix.Length..] : null; - if (paused) + await foreach (var batch in stream.ClaimHolders(CategoryId, claimHoldersAfter, batchSize, cancellationToken)) { - logger.LogWarning("Copying {CategoryId} is waiting for the RavenDB index {IndexName} in {DatabaseName}, but indexing is paused, so ServiceControl stays closed until it is resumed in RavenDB Studio. Waited {Waited} so far", CategoryId, IndexName, databaseName, waited); - return; + yield return batch; } - - logger.LogInformation("Waiting for the RavenDB index {IndexName} in {DatabaseName} to catch up before copying {CategoryId}, waited {Waited} so far", IndexName, databaseName, CategoryId, waited); } - - static IAsyncDocumentQuery Query(IAsyncDocumentSession session) => - session.Advanced.AsyncDocumentQuery(); - - static object[] StatusValues(IEnumerable statuses) => statuses.Select(status => (object)(int)status).ToArray(); - - static readonly FailedMessageStatus[] CopiedStatuses = [.. MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses, .. MigrationCategoryRegistry.ArchivedAndResolvedStatuses]; - static readonly string IndexName = new FailedMessageViewIndex().IndexName; - static readonly TimeSpan PollEvery = TimeSpan.FromSeconds(1); - static readonly TimeSpan LogEvery = TimeSpan.FromSeconds(30); - static readonly ILogger logger = LoggerUtil.CreateStaticLogger(); - - const string DocumentId = "id()"; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/UnresolvedFailedMessagesReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/UnresolvedFailedMessagesReader.cs new file mode 100644 index 0000000000..3e43214cdb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/UnresolvedFailedMessagesReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads every unresolved failed message, whatever its age, for the copy that runs while ServiceControl is open. A message that +/// holds a retry claim was copied with the required category and is already present, which the copy counts as such. +/// +sealed class UnresolvedFailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : FailedMessageStatusReader(lifecycle, MigrationCategoryRegistry.UnresolvedStatuses) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedFailedMessages; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadStatuses(resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/UnfinishedOperationsCheck.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/UnfinishedOperationsCheck.cs new file mode 100644 index 0000000000..8a6b67138e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/UnfinishedOperationsCheck.cs @@ -0,0 +1,46 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using ServiceControl.Persistence.DataMigration; + +// Archive and unarchive operations share an id prefix with their batches. Completed retry history is a separate document and does not count. +sealed class UnfinishedOperationsCheck(RavenReadOnlySourceLifecycle lifecycle) : IMigrationStartupCheck +{ + public string Name => "RavenDB holds no unfinished retry, archive or unarchive operations"; + + public async Task Run(CancellationToken cancellationToken = default) + { + var unfinished = new List(); + + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + foreach (var (description, prefix) in Prefixes) + { + await using var documents = await session.Advanced.StreamAsync(prefix, pageSize: 1, token: cancellationToken); + + if (await documents.MoveNextAsync()) + { + unfinished.Add($"{description} (for example '{documents.Current.Id}')"); + } + } + + if (unfinished.Count > 0) + { + throw new InvalidOperationException( + $"The RavenDB database '{lifecycle.Settings.DatabaseName}' holds unfinished operations: {string.Join("; ", unfinished)}. The migration does not copy them. Start ServiceControl on RavenDB again, wait until its retry, archive and unarchive operations have finished, stop it, and run --start-data-migration again. If an archive or unarchive operation does not resume, request the same group action again on RavenDB. Stopping ServiceControl does not finish them."); + } + } + + static readonly (string Description, string Prefix)[] Prefixes = + [ + ("retry batches and the forwarding marker", "RetryBatches/"), + ("archive operations and their batches", "ArchiveOperations/"), + ("unarchive operations and their batches", "UnarchiveOperations/") + ]; +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs index beaee964fe..4478dd1ba9 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs @@ -32,6 +32,18 @@ public async Task Reads_the_attachment_and_its_own_content_type() } } + [Test] + public async Task Reads_the_attachment_for_the_unresolved_failed_messages_too() + { + var content = Encoding.UTF8.GetBytes("2"); + var documentId = await SeedWithAttachment(content, "application/json"); + + await using var source = await OpenMigrationSource(); + var body = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedFailedMessages), documentId); + + Assert.That(body?.Content.ToArray(), Is.EqualTo(content)); + } + [Test] public async Task Reads_a_zero_length_attachment_as_an_empty_body() { diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index 21502d2d23..7b5f4784f3 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -11,6 +11,7 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using Raven.Client.Documents.Operations.Indexes; using ServiceControl.Contracts.CustomChecks; using ServiceControl.ExternalIntegrations; +using ServiceControl.MessageFailures; using ServiceControl.Infrastructure.RavenDB.Subscriptions; using ServiceControl.Notifications; using ServiceControl.Operations; @@ -136,7 +137,8 @@ public async Task Reading_every_category_creates_no_index() { [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }), - [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()), + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)), + [MigrationCategoryIds.UnresolvedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()), [MigrationCategoryIds.LicensingEndpoints] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingThroughput] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingReportMasks] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso"] }), diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RetryIssuedFailedMessageSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RetryIssuedFailedMessageSourceTests.cs new file mode 100644 index 0000000000..d7c5c276b7 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RetryIssuedFailedMessageSourceTests.cs @@ -0,0 +1,192 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +[TestFixture] +class RetryIssuedFailedMessageSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + [Test] + public async Task Reads_every_retry_issued_message_whether_or_not_its_claim_remains() + { + var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var released = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedClaim(claimed); + await CompleteDatabaseOperation(); + + var rows = (await ReadAll()).ToDictionary(row => row.SourceId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Keys, Is.EquivalentTo(new[] { claimed.Id, released.Id })); + Assert.That(rows[claimed.Id].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True); + Assert.That(rows[released.Id].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.False); + } + } + + [Test] + public async Task Reads_a_message_that_holds_a_claim_whatever_its_status_and_no_message_without_one() + { + var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + var archived = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + var resolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); + await SeedClaim(unresolved); + await SeedClaim(archived); + await SeedClaim(resolved); + await CompleteDatabaseOperation(); + + var rows = await ReadAll(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { unresolved.Id, archived.Id, resolved.Id })); + Assert.That(rows.Select(row => row.Metadata[MigrationMetadataKeys.RetryClaimHeld]), Is.All.True, "the claim is copied for a holder of any status, so a retry by id stays blocked"); + Assert.That(rows.Select(row => ((FailedMessage)row.Document).Status), Is.EquivalentTo(new[] { FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Resolved })); + } + } + + [Test] + public async Task Reads_an_unresolved_message_without_a_claim_in_neither_category_phase() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + Assert.That(await ReadAll(), Is.Empty, "the unresolved copy that runs with the host open owns it"); + } + + [Test] + public async Task Reads_a_retry_issued_message_with_a_claim_once() + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedClaim(message); + await CompleteDatabaseOperation(); + + Assert.That((await ReadAll()).Select(row => row.SourceId), Is.EqualTo(new[] { message.Id })); + } + + [Test] + public async Task Skips_a_claim_whose_message_is_gone() + { + var gone = new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived); + var present = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedClaim(present); + await SeedClaim(FailedMessageIdGenerator.MakeDocumentId(gone.UniqueMessageId), gone.UniqueMessageId); + await CompleteDatabaseOperation(); + + Assert.That((await ReadAll()).Select(row => row.SourceId), Is.EqualTo(new[] { present.Id })); + } + + [Test] + public async Task The_claim_holders_follow_the_retry_issued_messages_and_their_cursor_resumes_in_that_phase() + { + var holders = new List(); + var retryIssued = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + + for (var count = 0; count < 5; count++) + { + var holder = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(count % 2 == 0 ? FailedMessageStatus.Archived : FailedMessageStatus.Unresolved)); + await SeedClaim(holder); + holders.Add(holder); + } + + await CompleteDatabaseOperation(); + var holderIds = holders.Select(holder => holder.Id).OrderBy(id => id, StringComparer.OrdinalIgnoreCase).ToArray(); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category, batchSize: 2); + + Assert.That(batches.Select(batch => batch.Cursor).Skip(1), Has.All.StartWith("claim-holder:")); + + var resumed = await CollectBatches(source, Category, resumeAfter: batches[1].Cursor, batchSize: 2); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { retryIssued.Id }.Concat(holderIds)), "retry-issued first, then every claim holder once, in id order"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(batches.Skip(2).SelectMany(batch => batch.Rows).Select(row => row.SourceId)), "a claim-holder cursor resumes after its message and does not look for it in the retry-issued phase"); + } + } + + [Test] + public async Task A_cursor_from_the_earlier_layout_still_copies_the_claim_holders_before_it() + { + var holder = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedClaim(holder); + var later = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + var cursorPastTheHolder = new[] { holder.Id, later.Id }.Max(StringComparer.Ordinal); + + await using var source = await OpenMigrationSource(); + var rows = (await CollectBatches(source, Category, resumeAfter: cursorPastTheHolder)).SelectMany(batch => batch.Rows).ToList(); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { holder.Id }), "the earlier layout never copied an archived claim holder, so the claim phase starts from the beginning whatever the plain cursor is"); + } + + [Test] + public async Task Reads_the_claim_holders_when_the_index_is_missing() + { + await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(new FailedMessageViewIndex().IndexName), TestContext.CurrentContext.CancellationToken); + var retryIssued = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var holder = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedClaim(holder); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { retryIssued.Id, holder.Id })); + } + + [Test] + public async Task Counts_the_rows_the_copy_reads() + { + var holder = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedClaim(holder); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(Category), Is.EqualTo(2)); + } + + [Test] + public async Task Carries_the_last_modified_of_a_claim_holder() + { + var holder = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedClaim(holder); + await CompleteDatabaseOperation(); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.LastModified], Is.Not.Null, "the target needs the time the status changed, and a loaded document carries it in its metadata like a streamed one"); + } + + async Task SeedClaim(FailedMessage message) => await SeedClaim(message.Id, message.UniqueMessageId); + + async Task SeedClaim(string failedMessageId, string uniqueMessageId) + { + using var session = DocumentStore.OpenAsyncSession(); + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = failedMessageId, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(uniqueMessageId)); + await session.SaveChangesAsync(); + } + + async Task> ReadAll() + { + await using var source = await OpenMigrationSource(); + + return (await CollectBatches(source, Category)).SelectMany(batch => batch.Rows).ToList(); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnfinishedOperationsCheckTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnfinishedOperationsCheckTests.cs new file mode 100644 index 0000000000..261fe40d9d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnfinishedOperationsCheckTests.cs @@ -0,0 +1,150 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.Recoverability; +using ServiceControl.Recoverability; + +class UnfinishedOperationsCheckTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task The_source_contributes_the_check() + { + await using var source = await OpenMigrationSource(); + + Assert.That(source.ContributedChecks().Select(check => check.Name), Has.Member("RavenDB holds no unfinished retry, archive or unarchive operations")); + } + + [Test] + public async Task Passes_when_no_operation_is_persisted() + { + await using var source = await OpenMigrationSource(); + + Assert.DoesNotThrowAsync(() => RunChecks(source)); + } + + [Test] + public async Task Passes_when_only_completed_retry_history_is_persisted() + { + await RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + + Assert.DoesNotThrowAsync(() => RunChecks(source)); + } + + [Test] + public async Task Passes_when_only_a_retry_claim_is_persisted() + { + var messageId = Guid.NewGuid().ToString(); + var claimId = RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(messageId); + using (var session = DocumentStore.OpenAsyncSession(new Raven.Client.Documents.Session.SessionOptions { Database = ((RavenPersisterSettings)PersistenceSettings).DatabaseName })) + { + await session.StoreAsync(new FailedMessageRetry + { + Id = claimId, + FailedMessageId = messageId, + RetryBatchId = RetryDocumentDataStore.MakeDocumentId("completed-batch") + }, claimId); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + + Assert.DoesNotThrowAsync(() => RunChecks(source)); + } + + [TestCase("retry batch")] + [TestCase("forwarding marker")] + [TestCase("archive operation")] + [TestCase("archive batch")] + [TestCase("unarchive operation")] + [TestCase("unarchive batch")] + public async Task Refuses_when_an_unfinished_operation_is_persisted_and_says_how_to_finish_it(string unfinished) + { + var documentId = await Seed(unfinished); + await using var source = await OpenMigrationSource(); + + var refusal = Assert.ThrowsAsync(() => RunChecks(source)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain(documentId), "the operator is told which document is unfinished"); + Assert.That(refusal.Message, Does.Contain("Start ServiceControl on RavenDB again")); + Assert.That(refusal.Message, Does.Contain("--start-data-migration again")); + Assert.That(refusal.Message, Does.Contain("request the same group action again on RavenDB")); + } + } + + [Test] + public async Task Names_every_kind_of_unfinished_operation() + { + await Seed("retry batch"); + await Seed("archive operation"); + await Seed("unarchive operation"); + await using var source = await OpenMigrationSource(); + + var refusal = Assert.ThrowsAsync(() => RunChecks(source)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("retry batches")); + Assert.That(refusal.Message, Does.Contain("archive operations")); + Assert.That(refusal.Message, Does.Contain("unarchive operations")); + } + } + + static async Task RunChecks(Persistence.DataMigration.IMigrationSource source) + { + foreach (var check in source.ContributedChecks()) + { + await check.Run(); + } + } + + async Task Seed(string unfinished) + { + using var session = DocumentStore.OpenAsyncSession(new Raven.Client.Documents.Session.SessionOptions { Database = ((RavenPersisterSettings)PersistenceSettings).DatabaseName }); + + string id; + + switch (unfinished) + { + case "retry batch": + id = RetryDocumentDataStore.MakeDocumentId(Guid.NewGuid().ToString()); + await session.StoreAsync(new RetryBatch { Id = id, Status = RetryBatchStatus.Forwarding }, id); + break; + case "forwarding marker": + id = RetryDocumentDataStore.NowForwardingDocumentId; + await session.StoreAsync(new RetryBatchNowForwarding { RetryBatchId = RetryDocumentDataStore.MakeDocumentId("batch-1") }, id); + break; + case "archive operation": + id = ArchiveOperation.MakeId("group-1", ArchiveType.FailureGroup); + await session.StoreAsync(new ArchiveOperation { Id = id, RequestId = "group-1", ArchiveType = ArchiveType.FailureGroup }, id); + break; + case "archive batch": + id = ArchiveBatch.MakeId("group-1", ArchiveType.FailureGroup, 1); + await session.StoreAsync(new ArchiveBatch { Id = id }, id); + break; + case "unarchive operation": + id = UnarchiveOperation.MakeId("group-1", ArchiveType.FailureGroup); + await session.StoreAsync(new UnarchiveOperation { Id = id, RequestId = "group-1", ArchiveType = ArchiveType.FailureGroup }, id); + break; + case "unarchive batch": + id = UnarchiveBatch.MakeId("group-1", ArchiveType.FailureGroup, 1); + await session.StoreAsync(new UnarchiveBatch { Id = id }, id); + break; + default: + throw new ArgumentOutOfRangeException(nameof(unfinished), unfinished, null); + } + + await session.SaveChangesAsync(); + await CompleteDatabaseOperation(); + + return id; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs index 9faec057e1..dd55778870 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs @@ -20,10 +20,10 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; class UnresolvedFailedMessageSourceTests : RavenMigrationSourceTestBase { [Test] - public async Task Reads_both_unresolved_statuses_and_neither_of_the_others() + public async Task Reads_the_unresolved_status_and_none_of_the_others() { var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); - var retryIssued = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); await CompleteDatabaseOperation(); @@ -32,7 +32,19 @@ public async Task Reads_both_unresolved_statuses_and_neither_of_the_others() Assert.That( rows.Select(row => ((FailedMessage)row.Document).UniqueMessageId), - Is.EquivalentTo(new[] { unresolved.UniqueMessageId, retryIssued.UniqueMessageId })); + Is.EquivalentTo(new[] { unresolved.UniqueMessageId })); + } + + [Test] + public async Task Reads_an_unresolved_message_however_old_it_is() + { + var failure = new IngestedFailure(); + var message = failure.ToFailedMessage(FailedMessageStatus.Unresolved); + message.ProcessingAttempts.ForEach(attempt => attempt.AttemptedAt = DateTime.UtcNow.AddDays(-4000)); + var stored = await SeedFailedMessage(message); + await CompleteDatabaseOperation(); + + Assert.That((await ReadAll()).Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "the category has no window, because an operator is waiting on an unresolved message however old it is"); } [Test] @@ -101,29 +113,19 @@ public async Task Sends_only_the_migration_metadata_keys_across_the_seam() } [Test] - public async Task A_retry_issued_message_says_whether_the_source_still_holds_its_retry_claim() + public async Task An_unresolved_message_carries_no_claim_state_even_when_the_source_holds_a_claim_for_it() { - var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); - var released = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); using (var session = DocumentStore.OpenAsyncSession()) { - await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); await session.StoreAsync(new FailedMessageRetry { FailedMessageId = unresolved.Id, RetryBatchId = "RetryBatches/staging" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(unresolved.UniqueMessageId)); await session.SaveChangesAsync(); } await CompleteDatabaseOperation(); - var rows = (await ReadAll()).ToDictionary(row => ((FailedMessage)row.Document).UniqueMessageId); - - using (Assert.EnterMultipleScope()) - { - Assert.That(rows[claimed.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True); - Assert.That(rows[released.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.False); - Assert.That(rows[unresolved.UniqueMessageId].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.RetryClaimHeld), "only a retry-issued message gets a claim in SQL, even when the source holds one for another"); - } + Assert.That((await ReadAll()).Single().Metadata, Does.Not.ContainKey(MigrationMetadataKeys.RetryClaimHeld), "the claim holder is copied with the required category, and this copy finds it already present"); } [Test] @@ -182,9 +184,9 @@ public async Task Resuming_after_a_cursor_reads_only_the_unresolved_messages_aft using (Assert.EnterMultipleScope()) { - Assert.That(first.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { ids[0], ids[2] })); - Assert.That(first.Cursor, Is.EqualTo(ids[2])); - Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[3], ids[5] }), "a resume neither repeats a row nor skips one"); + Assert.That(first.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { ids[0], ids[3] })); + Assert.That(first.Cursor, Is.EqualTo(ids[3])); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[5] }), "a resume neither repeats a row nor skips one"); } } @@ -248,7 +250,7 @@ public async Task Reads_through_a_Corax_index_as_through_a_Lucene_one() await using var source = await OpenMigrationSource(); var batches = await CollectBatches(source, Category); - Assert.That(batches.Select(batch => (batch.Cursor, batch.Rows.Select(row => row.SourceId))), Is.EqualTo(new[] { (ids[2], new[] { ids[0], ids[2] }) }), "a full read would walk on to the resolved message and save its cursor past it"); + Assert.That(batches.Select(batch => (batch.Cursor, batch.Rows.Select(row => row.SourceId))), Is.EqualTo(new[] { (ids[0], new[] { ids[0] }) }), "a full read would walk on to the resolved message and save its cursor past it"); } [Test] @@ -257,7 +259,7 @@ public async Task Resumes_through_a_Corax_index_with_no_row_repeated_or_skipped( await UseCorax(); var statuses = Enumerable.Range(0, 60).Select(position => (FailedMessageStatus)((position % 4) + 1)).ToArray(); var ids = await SeedInIdOrder(statuses); - var expected = ids.Where((_, position) => statuses[position] is FailedMessageStatus.Unresolved or FailedMessageStatus.RetryIssued).ToList(); + var expected = ids.Where((_, position) => statuses[position] == FailedMessageStatus.Unresolved).ToList(); await using var source = await OpenMigrationSource(); var read = new List(); @@ -285,21 +287,12 @@ public async Task Reads_the_whole_collection_when_the_index_is_missing() { await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); - var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); - - using (var session = DocumentStore.OpenAsyncSession()) - { - await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); - await session.SaveChangesAsync(); - } + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); - using (Assert.EnterMultipleScope()) - { - Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { stored.Id, claimed.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); - Assert.That(rows.Single(row => row.SourceId == claimed.Id).Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True, "the whole-collection read must resolve claims too, or the writer stops on the first retry-issued row"); - } + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); } [Test] @@ -919,7 +912,7 @@ async Task> StreamWithWalkLimit(int batchSize, int maxDocum await foreach (var batch in RavenDocumentStream.ByPrefix( lifecycle, Category.Id, settings.DatabaseName, $"{FailedMessageIdGenerator.CollectionName}/", null, batchSize, maxDocumentsWalkedPerBatch, - new FailedMessageStream(lifecycle).Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses), TestContext.CurrentContext.CancellationToken)) + new FailedMessageStream(lifecycle).Project(MigrationCategoryRegistry.UnresolvedStatuses), TestContext.CurrentContext.CancellationToken)) { batches.Add(batch); } @@ -927,7 +920,7 @@ async Task> StreamWithWalkLimit(int batchSize, int maxDocum return batches; } - static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedFailedMessages); static readonly string IndexName = new FailedMessageViewIndex().IndexName; // The parsers read headers through IReadOnlyDictionary, so re-implementing its lookup makes every header read fail. diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs index 814f5565a3..62ce59904d 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs @@ -10,8 +10,11 @@ namespace ServiceControl.Persistence.Tests; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using NServiceBus.Extensibility; +using NServiceBus.Transport; using NUnit.Framework; using ServiceControl.MessageFailures; +using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DataMigration; @@ -37,6 +40,8 @@ class FailedMessagesMigrationTargetTests : PersistenceTestBase static readonly MigrationBody SmallTextBody = new(Encoding.UTF8.GetBytes("1"), "text/xml"); + static readonly MigrationCategory UnresolvedCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedFailedMessages); + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); [Test] @@ -480,9 +485,13 @@ public async Task A_changed_message_the_target_already_holds_is_not_reported_aga } [Test] - public async Task Count_takes_in_only_unresolved_and_retry_issued_messages() + public async Task Count_takes_in_retry_issued_messages_and_claim_holders_of_any_status() { - await Target.Write(FailedMessagesCategory, BatchOf(new IngestedFailure().ToFailedMessage(), new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + var holder = new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived); + await Target.Write(FailedMessagesCategory, BatchOf( + RowOf(new IngestedFailure().ToFailedMessage(), SmallTextBody), + RowOf(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), + WithClaimHeld(RowOf(holder, SmallTextBody), true)), Checkpoint()); await Query(db => { db.FailedMessages.AddRange( @@ -493,12 +502,198 @@ await Query(db => using (Assert.EnterMultipleScope()) { - Assert.That(await Target.Count(FailedMessagesCategory), Is.EqualTo(2), "the archived and resolved rows belong to another category"); - Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(4), "the filter is doing the work, not an empty table"); + Assert.That(await Target.Count(FailedMessagesCategory), Is.EqualTo(2), "an unresolved message without a claim and the archived and resolved rows without one belong to other copies"); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(5), "the filter is doing the work, not an empty table"); + } + } + + [Test] + public async Task Count_of_the_unresolved_copy_takes_in_only_unresolved_messages() + { + await Target.Write(UnresolvedCategory, BatchOf(new IngestedFailure().ToFailedMessage(), new IngestedFailure().ToFailedMessage()), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + await Query(db => + { + db.FailedMessages.AddRange( + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.RetryIssued, HeadersJson = "{}", FailingEndpointAddress = "error" }, + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.Archived, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + Assert.That(await Target.Count(UnresolvedCategory), Is.EqualTo(2)); + } + + [Test] + public async Task A_claim_holder_of_any_status_gets_a_retry_claim_and_a_message_without_one_does_not() + { + var unresolved = new IngestedFailure(); + var archived = new IngestedFailure(); + var resolved = new IngestedFailure(); + var withoutClaim = new IngestedFailure(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + WithClaimHeld(RowOf(unresolved.ToFailedMessage(), SmallTextBody), true), + WithClaimHeld(RowOf(archived.ToFailedMessage(FailedMessageStatus.Archived), SmallTextBody), true), + WithClaimHeld(RowOf(resolved.ToFailedMessage(FailedMessageStatus.Resolved), SmallTextBody), true), + WithClaimHeld(RowOf(withoutClaim.ToFailedMessage(FailedMessageStatus.Archived), SmallTextBody), false)), Checkpoint()); + + var rows = await Query(db => db.FailedMessages.AsNoTracking().ToDictionaryAsync(message => message.UniqueMessageId, message => message.Status)); + var claimed = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(4)); + Assert.That(rows[archived.UniqueMessageId], Is.EqualTo(FailedMessageStatus.Archived), "the row keeps the status it had in the source"); + Assert.That(rows[resolved.UniqueMessageId], Is.EqualTo(FailedMessageStatus.Resolved)); + Assert.That(claimed, Is.EquivalentTo(new[] { unresolved.UniqueMessageId, archived.UniqueMessageId, resolved.UniqueMessageId }), "a retry by id stays blocked for every holder, whatever its status"); + } + } + + [Test] + public async Task A_claim_holder_already_in_the_target_gets_no_retry_claim_from_the_copy() + { + var present = new IngestedFailure(); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = present.UniqueMessageId, Status = FailedMessageStatus.Unresolved, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(WithClaimHeld(RowOf(present.ToFailedMessage(), SmallTextBody), true)), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(await Query(db => db.FailedMessageRetries.CountAsync()), Is.Zero); + } + } + + [Test] + public async Task The_unresolved_copy_inserts_unresolved_messages_with_their_groups_and_bodies_and_no_claim() + { + var failure = new IngestedFailure { ContentType = "application/octet-stream" }; + var body = new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"); + + var result = await Target.Write(UnresolvedCategory, BatchOf(RowOf(failure.ToFailedMessage(), body)), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await Query(db => db.FailedMessageGroups.CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)), Is.EqualTo(failure.Groups.Count)); + Assert.That(await BodyStored(failure.UniqueMessageIdString), Is.True); + Assert.That(await Query(db => db.FailedMessageRetries.CountAsync()), Is.Zero); + } + } + + [Test] + public async Task A_row_ingestion_wrote_first_wins_over_the_unresolved_copy_with_its_attempt_count_payload_and_body() + { + var failure = new IngestedFailure { ContentType = "application/octet-stream" }; + var liveBody = new byte[] { 0x01, 0x02 }; + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity + { + UniqueMessageId = failure.UniqueMessageId, + Status = FailedMessageStatus.Unresolved, + ExceptionMessage = "Written by ingestion", + NumberOfProcessingAttempts = 1, + HeadersJson = "{}", + FailingEndpointAddress = "error", + BodyStoredExternally = true + }); + return db.SaveChangesAsync(); + }); + await BodyStore.WriteBody(failure.UniqueMessageIdString, liveBody, "application/octet-stream"); + + var result = await Target.Write(UnresolvedCategory, BatchOf(RowOf(failure.ToFailedMessage(numberOfAttempts: 3), new MigrationBody(new byte[] { 0x09, 0x09, 0x09 }, "application/octet-stream"))), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == failure.UniqueMessageId)); + var stored = await BodyStore.ReadBody(failure.UniqueMessageIdString); + using var memory = new MemoryStream(); + await stored.Stream.CopyToAsync(memory); + await stored.Stream.DisposeAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "an existing SQL row counts as already present, not as a skip"); + Assert.That(result.Skipped, Is.Zero); + Assert.That(row.ExceptionMessage, Is.EqualTo("Written by ingestion"), "SQL wins, so its payload stays even when the source holds a newer attempt"); + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(1), "SQL wins, so its attempt count omits the earlier RavenDB attempts"); + Assert.That(memory.ToArray(), Is.EqualTo(liveBody), "the migration never replaces a body a live row already holds"); + } + } + + [Test] + public async Task A_body_stored_after_the_present_check_and_before_the_body_write_is_not_replaced() + { + // The present check and the body write are not atomic, so a body that ingestion stores between them has to survive the write. + var failure = new IngestedFailure { ContentType = "application/octet-stream" }; + var liveBody = new byte[] { 0x01, 0x02 }; + await BodyStore.WriteBody(failure.UniqueMessageIdString, liveBody, "application/octet-stream"); + + await Target.Write(UnresolvedCategory, BatchOf(RowOf(failure.ToFailedMessage(), new MigrationBody(new byte[] { 0x09, 0x09, 0x09 }, "application/octet-stream"))), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + + var stored = await BodyStore.ReadBody(failure.UniqueMessageIdString); + using var memory = new MemoryStream(); + await stored.Stream.CopyToAsync(memory); + await stored.Stream.DisposeAsync(); + + Assert.That(memory.ToArray(), Is.EqualTo(liveBody)); + } + + // Known release blocker, not accepted behavior: docs/migration/failed-message-migration-split-plan.md describes the interleaving. + // The test pins today's behavior and changes no confirmation handling, so it has to be rewritten when the blocker is resolved. + [Test] + public async Task Known_release_blocker_a_retry_confirmation_for_a_message_the_unresolved_copy_has_not_inserted_is_consumed_and_the_message_arrives_unresolved() + { + var failure = new IngestedFailure(); + var confirmation = new MessageContext( + Guid.NewGuid().ToString(), + new Dictionary + { + ["ServiceControl.Retry.Successful"] = string.Empty, + ["ServiceControl.Retry.UniqueMessageId"] = failure.UniqueMessageIdString + }, + ReadOnlyMemory.Empty, + new TransportTransaction(), + "receiveAddress", + new ContextBag()); + var unitOfWork = await IngestionUnitOfWorkFactory.StartNew(); + + await new RetryConfirmationProcessor(new FakeDomainEvents()).Process([confirmation], unitOfWork); + + Assert.DoesNotThrowAsync(() => unitOfWork.Complete(TestContext.CurrentContext.CancellationToken), "a confirmation for a missing row succeeds today, which is how it is consumed"); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.Zero, "the confirmation records nothing for a row that is absent"); + + await Target.Write(UnresolvedCategory, BatchOf(failure.ToFailedMessage()), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == failure.UniqueMessageId)); + Assert.That(row.Status, Is.EqualTo(FailedMessageStatus.Unresolved), "the consumed confirmation no longer resolves the message"); + } + + [Test] + public void The_unresolved_copy_follows_the_required_copy_checkpoint_layout() + { + Assert.That(UnresolvedCategory.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + } + + [Test] + public async Task Writes_a_message_ingestion_has_not_stored_even_when_the_target_holds_a_claim_for_another() + { + var other = new IngestedFailure(); + var failure = new IngestedFailure(); + await Target.Write(FailedMessagesCategory, BatchOf(RowOf(other.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody)), Checkpoint()); + + var result = await Target.Write(UnresolvedCategory, BatchOf(failure.ToFailedMessage()), Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await Query(db => db.FailedMessageRetries.CountAsync()), Is.EqualTo(1), "the unresolved copy neither adds nor removes a claim"); } } - // PostgreSQL takes a fixed number of rows per statement whatever the entity, so only SQL Server can catch the wrong entity here. [Test] public async Task A_batch_is_one_statement_of_failed_message_rows() { @@ -706,6 +901,6 @@ sealed class UnwritableBodyStore : IBodyStoragePersistence FailedMessagesWriter Writer() => new(ServiceProvider.GetRequiredService(), BodyStore, ServiceProvider.GetRequiredService()); - static MigrationCheckpoint Checkpoint() => - new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + static MigrationCheckpoint Checkpoint(string categoryId = MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) => + new(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs index 9f50b13692..d932cbc893 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs @@ -99,7 +99,7 @@ public async Task A_comment_on_a_group_with_no_copied_message_is_reported_as_a_c { ("GroupComment/group-of-archived-messages", "The group 'group-of-archived-messages' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + - "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + "Its messages have not been copied yet, are archived or resolved, are gone from the source, or were skipped by the copy. " + $"The comment was: '{longComment}'") }), "a comment on a group with a copied message stays, so only the other is named, with the whole comment because the sweep deletes it"); Assert.That((first.Copied, first.Changed), Is.EqualTo((2, 1))); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index 3e942eace6..ae2043d82f 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -95,6 +95,7 @@ public async Task Every_writer_whose_rows_a_re_read_cannot_recognise_deletes_the [MigrationCategoryIds.KnownEndpoints] = nameof(KnownEndpointsMigrationTargetTests.A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), + [MigrationCategoryIds.UnresolvedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), [MigrationCategoryIds.LicensingEndpoints] = nameof(LicensingEndpointsMigrationTargetTests.A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.MessageRedirects] = nameof(MessageRedirectsMigrationTargetTests.A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), @@ -113,6 +114,7 @@ public async Task Every_writer_whose_rows_a_re_read_cannot_recognise_deletes_the [typeof(KnownEndpointsWriter)] = [typeof(KnownEndpointEntity)], [typeof(EndpointSettingsWriter)] = [typeof(EndpointSettingsEntity)], [typeof(FailedMessagesWriter)] = [typeof(FailedMessageEntity), typeof(FailedMessageGroupEntity)], + [typeof(UnresolvedFailedMessagesWriter)] = [typeof(FailedMessageEntity), typeof(FailedMessageGroupEntity)], [typeof(LicensingEndpointsWriter)] = [typeof(LicensingEndpointEntity)], [typeof(LicensingThroughputWriter)] = [typeof(LicensingEndpointThroughputEntity)], [typeof(LicensingReportMasksWriter)] = [typeof(SettingEntity)], diff --git a/src/ServiceControl.Persistence.Tests/EFCore/S3BodyStorageConditionalWriteTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/S3BodyStorageConditionalWriteTests.cs new file mode 100644 index 0000000000..51306bb304 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/S3BodyStorageConditionalWriteTests.cs @@ -0,0 +1,201 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Amazon.S3; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; + +// Runs the real S3 client against a local listener, so it needs neither Docker nor LocalStack. +[TestFixture] +class S3BodyStorageConditionalWriteTests +{ + FakeS3Endpoint endpoint; + + [SetUp] + public void StartEndpoint() => endpoint = new FakeS3Endpoint(); + + [TearDown] + public void StopEndpoint() => endpoint.Dispose(); + + S3BodyStoragePersistence Store() => new(new S3BodyStorageSettings + { + MinCompressionSize = 64, + ServiceUrl = endpoint.Url, + Region = "us-east-1", + Credentials = new S3StaticCredentials { AccessKeyId = "test", SecretAccessKey = "test" }, + BucketName = "bodies" + }); + + [Test] + public async Task The_upload_creates_the_object_only_if_none_exists_and_keeps_its_content_type_and_metadata() + { + await Store().WriteBody("body-1", new byte[] { 1, 2, 3 }, "text/plain"); + + var put = endpoint.Requests.Single(request => request.Method == "PUT"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(put.Headers["If-None-Match"], Is.EqualTo("*"), "without the condition a body stored after the existence check is replaced"); + Assert.That(put.Headers["Content-Type"], Is.EqualTo("text/plain")); + Assert.That(put.Headers["x-amz-meta-body-size"], Is.EqualTo("3")); + Assert.That(put.Headers["x-amz-meta-is-compressed"], Is.EqualTo("False")); + } + } + + [Test] + public async Task A_compressed_upload_is_conditional_too() + { + await Store().WriteBody("body-2", new byte[10_000], "application/octet-stream"); + + var put = endpoint.Requests.Single(request => request.Method == "PUT"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(put.Headers["If-None-Match"], Is.EqualTo("*")); + Assert.That(put.Headers["x-amz-meta-is-compressed"], Is.EqualTo("True")); + Assert.That(put.Headers["x-amz-meta-body-size"], Is.EqualTo("10000")); + } + } + + [Test] + public async Task An_existing_object_is_not_uploaded_over() + { + endpoint.ObjectExists = true; + + await Store().WriteBody("body-3", new byte[] { 1 }, "text/plain"); + + Assert.That(endpoint.Requests.Select(request => request.Method), Is.EqualTo(new[] { "HEAD" })); + } + + [Test] + public void An_object_stored_after_the_existence_check_is_left_alone_without_an_error() + { + endpoint.PutStatus = HttpStatusCode.PreconditionFailed; + + Assert.DoesNotThrowAsync(() => Store().WriteBody("body-4", new byte[] { 1 }, "text/plain")); + Assert.That(endpoint.Requests.Count(request => request.Method == "PUT"), Is.EqualTo(1)); + } + + [Test] + public void Any_other_storage_failure_still_fails_the_write() + { + endpoint.PutStatus = HttpStatusCode.Forbidden; + + var exception = Assert.ThrowsAsync(() => Store().WriteBody("body-5", new byte[] { 1 }, "text/plain")); + + Assert.That(exception.StatusCode, Is.EqualTo(HttpStatusCode.Forbidden)); + } + + [Test] + public void A_cancelled_write_is_cancelled() + { + using var cancelled = new CancellationTokenSource(); + cancelled.Cancel(); + + Assert.CatchAsync(() => Store().WriteBody("body-6", new byte[] { 1 }, "text/plain", cancelled.Token)); + } + + sealed class FakeS3Endpoint : IDisposable + { + readonly HttpListener listener = new(); + readonly object gate = new(); + readonly List requests = []; + + public FakeS3Endpoint() + { + // The listener needs a free port, which it cannot report before it starts, so probe for one. + for (var attempt = 0; ; attempt++) + { + var port = Random.Shared.Next(20000, 60000); + var candidate = new HttpListener(); + candidate.Prefixes.Add($"http://localhost:{port}/"); + + try + { + candidate.Start(); + listener = candidate; + Url = $"http://localhost:{port}"; + break; + } + catch (HttpListenerException) when (attempt < 20) + { + candidate.Close(); + } + } + + _ = Serve(); + } + + public string Url { get; } + public bool ObjectExists { get; set; } + public HttpStatusCode PutStatus { get; set; } = HttpStatusCode.OK; + + public IReadOnlyList Requests + { + get + { + lock (gate) + { + return [.. requests]; + } + } + } + + async Task Serve() + { + while (listener.IsListening) + { + HttpListenerContext context; + + try + { + context = await listener.GetContextAsync(); + } + catch (Exception ex) when (ex is HttpListenerException or ObjectDisposedException or InvalidOperationException) + { + return; + } + + var headers = context.Request.Headers.AllKeys.ToDictionary(key => key, key => context.Request.Headers[key], StringComparer.OrdinalIgnoreCase); + + lock (gate) + { + requests.Add(new RecordedRequest(context.Request.HttpMethod, headers)); + } + + // The body is not needed, but the client waits for the server to read it. + await context.Request.InputStream.CopyToAsync(Stream.Null); + + var status = context.Request.HttpMethod switch + { + "HEAD" => ObjectExists ? HttpStatusCode.OK : HttpStatusCode.NotFound, + "PUT" => PutStatus, + _ => HttpStatusCode.NotImplemented + }; + + context.Response.StatusCode = (int)status; + + if (context.Request.HttpMethod == "PUT" && status != HttpStatusCode.OK) + { + var code = status == HttpStatusCode.PreconditionFailed ? "PreconditionFailed" : "AccessDenied"; + var error = System.Text.Encoding.UTF8.GetBytes($"{code}{code}1"); + context.Response.ContentType = "application/xml"; + await context.Response.OutputStream.WriteAsync(error); + } + + context.Response.Close(); + } + } + + public void Dispose() => listener.Close(); + } + + sealed record RecordedRequest(string Method, Dictionary Headers); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs index 6670fcfbcb..fe22801861 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -17,7 +17,15 @@ public static class MigrationCategoryIds public const string LicensingThroughput = nameof(LicensingThroughput); public const string LicensingReportMasks = nameof(LicensingReportMasks); public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); + + /// + /// The retry-issued messages and the messages that hold a retry claim, copied before the host opens: every retry-issued message and + /// every other message that still holds a retry claim. + /// public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + + /// The unresolved failed messages, copied once the host is open, whatever their age. + public const string UnresolvedFailedMessages = nameof(UnresolvedFailedMessages); public const string PendingIntegrationEvents = nameof(PendingIntegrationEvents); public const string EventLog = nameof(EventLog); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs index e3db874daf..4f3e0492d8 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -12,10 +12,16 @@ namespace ServiceControl.Persistence.DataMigration; public static class MigrationCategoryRegistry { /// - /// The failed message statuses that are copied before the host opens. A message in one of these is waiting on - /// somebody, so an operator who cannot see it after the cutover has lost work. + /// The failed message status that is copied before the host opens, whether or not a retry claim is held. A retry + /// confirmation that arrives after the cutover resolves a message only if its row is already there. /// - public static readonly IReadOnlyList UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + public static readonly IReadOnlyList RetryIssuedStatuses = [FailedMessageStatus.RetryIssued]; + + /// + /// The failed message status that is copied in the background without a window, because an operator is waiting on + /// these however old they are. + /// + public static readonly IReadOnlyList UnresolvedStatuses = [FailedMessageStatus.Unresolved]; /// /// The failed message statuses that are copied in the background. These are history, so the instance is usable @@ -44,8 +50,9 @@ public static class MigrationCategoryRegistry new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Required, CarriesBodies: false, Order: 16, MustFollow: MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), // Optional, copied in the background once the host is open. - new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), - new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 2), + new(MigrationCategoryIds.UnresolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 1), + new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 2), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 3), ]; public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs index 8dee699d9f..5651af3531 100644 --- a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -11,7 +11,7 @@ namespace ServiceControl.Persistence.DataMigration; /// How long to wait between batches of an optional category, which is what keeps the background copy off the instance's back. Zero waits not at all. /// The share of skipped rows, as a percentage, that stops a category. /// The number of skipped rows that has to be passed before the percentage counts, so a handful of bad rows in a small category is not a halt. -/// The optional categories to copy. An optional category outside this list is never copied. +/// The optional categories to copy. An optional category outside this list is never copied. The unresolved failed messages are always in it. public sealed record MigrationEngineOptions( TimeSpan ThrottlePause, int HaltThresholdPercent, @@ -37,7 +37,7 @@ public sealed record MigrationEngineOptions( /// /// Reads the options from the instance's settings. Anything not set takes the default from /// , except the two windows, which default to the retention periods passed in. - /// An optional category is selected when its window is greater than zero. + /// A windowed optional category is selected when its window is greater than zero, and the unresolved failed messages are always selected. /// /// The settings root the migration keys are read under. /// The instance's event retention period, used as the event log window when none is set. @@ -51,7 +51,8 @@ public static MigrationEngineOptions FromSettings(SettingsRootNamespace settings var eventLogWindow = ReadWindow(settingsRootNamespace, MigrationSettings.EventLogWindowKey, eventRetentionPeriod); var archivedAndResolvedWindow = ReadWindow(settingsRootNamespace, MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey, errorRetentionPeriod); - var selectedIds = new List(); + // Not windowed: an unresolved message is waiting on somebody however old it is. + var selectedIds = new List { MigrationCategoryIds.UnresolvedFailedMessages }; if (eventLogWindow > TimeSpan.Zero) { diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt index 6c99c6b3e0..563a77e86d 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -14,5 +14,6 @@ Required 13: PendingIntegrationEvents Required 14: CustomChecks Required 15: FailedErrorImports, with bodies Required 16: GroupComments, after UnresolvedAndRetryIssuedFailedMessages -Optional 1: EventLog -Optional 2: ArchivedAndResolvedFailedMessages, with bodies \ No newline at end of file +Optional 1: UnresolvedFailedMessages, with bodies +Optional 2: EventLog +Optional 3: ArchivedAndResolvedFailedMessages, with bodies \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt index d0b5240bad..3338eba16c 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -1,4 +1,4 @@ Configured: EventLog Runs as: -Optional 1: EventLog -Never copied: ArchivedAndResolvedFailedMessages \ No newline at end of file +Optional 2: EventLog +Never copied: UnresolvedFailedMessages, ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs index 9e86ba2723..6566769991 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs @@ -64,6 +64,17 @@ public async Task Optional_work_does_not_hold_the_host_closed(MigrationCategoryS await new MigrationBeforeTheHostOpens(services).StartingAsync(); } + [Test] + public async Task The_required_failed_messages_with_the_unresolved_failed_messages_row_open_the_host_while_that_row_still_copies() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint(MigrationCategoryIds.UnresolvedFailedMessages, MigrationCategoryState.InProgress)); + using var services = Services(store); + + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + static ServiceProvider Services(IMigrationCheckpointStore store) => new ServiceCollection().AddSingleton(store).BuildServiceProvider(); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs index 214b13a2d2..cf786152dc 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -12,12 +12,12 @@ namespace ServiceControl.UnitTests.Migration; class MigrationCategoryRegistryTests { [Test] - public void Contains_all_eighteen_categories_with_unique_ids() + public void Contains_all_nineteen_categories_with_unique_ids() { using (Assert.EnterMultipleScope()) { - Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); - Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(19)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(19)); } } @@ -51,7 +51,7 @@ public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() Assert.That(optionalIds, Is.EqualTo(new[] { - "EventLog", "ArchivedAndResolvedFailedMessages" + "UnresolvedFailedMessages", "EventLog", "ArchivedAndResolvedFailedMessages" })); } @@ -76,23 +76,37 @@ public void Three_categories_declare_a_MustFollow() } [Test] - public void The_two_failed_message_categories_and_failed_imports_carry_bodies() + public void The_three_failed_message_categories_and_failed_imports_carry_bodies() { var carriesBodies = MigrationCategoryRegistry.All.Where(c => c.CarriesBodies).Select(c => c.Id).ToArray(); - Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); + Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "UnresolvedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); } [Test] - public void The_two_failed_message_categories_split_every_status_between_them() + public void The_unresolved_failed_messages_are_copied_after_the_host_opens_and_the_group_comments_do_not_wait_for_them() { - var split = MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses).ToArray(); + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedFailedMessages)?.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)?.MustFollow, Is.EqualTo(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)); + Assert.That(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)?.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + } + } + + [Test] + public void The_failed_message_statuses_are_split_between_the_retry_issued_unresolved_and_archived_and_resolved_copies() + { + var split = MigrationCategoryRegistry.RetryIssuedStatuses + .Concat(MigrationCategoryRegistry.UnresolvedStatuses) + .Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses) + .ToArray(); using (Assert.EnterMultipleScope()) { Assert.That(split, Is.Unique); Assert.That(split, Is.EquivalentTo(Enum.GetValues()), - "a status neither category claims is a failed message no migration copies"); + "a status no category claims is a failed message no migration copies"); } } diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs index 71e62dac7b..34ee9c7d63 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -35,7 +35,7 @@ public void Defaults_match_the_contract_when_nothing_is_configured() Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); - Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages }), "both optional categories are copied unless turned off"); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.UnresolvedFailedMessages, MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages }), "both windowed optional categories are copied unless turned off, and the unresolved failed messages always are"); Assert.That(options.EventLogWindow, Is.EqualTo(EventRetention), "the event log window defaults to the event retention period"); Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention), "the archived and resolved window defaults to the error retention period"); Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); @@ -60,7 +60,7 @@ public void Reads_configured_values_from_environment_variables() Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); Assert.That(options.EventLogWindow, Is.EqualTo(TimeSpan.FromDays(2))); Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(TimeSpan.FromDays(5.5))); - Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages })); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.UnresolvedFailedMessages, MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages })); } } @@ -72,7 +72,16 @@ public void A_zero_window_turns_its_category_off(string variable, string stillSe var options = Read(); - Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { stillSelected })); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.UnresolvedFailedMessages, stillSelected })); + } + + [Test] + public void The_unresolved_failed_messages_are_selected_when_both_windows_are_zero() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "0"); + + Assert.That(Read().SelectedOptionalCategoryIds, Is.EqualTo(new[] { MigrationCategoryIds.UnresolvedFailedMessages }), "unresolved messages have no window to turn off"); } [TestCase("a week")]