diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md index d1879cf889..dbcf16c25e 100644 --- a/docs/migration/ravendb-to-sql-migration-instructions.md +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -25,12 +25,11 @@ flowchart TD S2["2. Create an empty SQL database and run --setup"] S3["3. Choose the optional data windows"] S4["4. Run --migration-dry-run and book the outage"] - S5["5. Back up RavenDB"] - S1 --> S2 --> S3 --> S4 --> S5 + S1 --> S2 --> S3 --> S4 end subgraph outage["ServiceControl stopped: the outage"] - S6["6. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] + S6["5. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] Q1{"Did --start-data-migration succeed?"} Fix["Correct the reported cause"] RetryAbandon1["Run --migration-retry or --migration-abandon for the Failed category"] @@ -42,17 +41,17 @@ flowchart TD end subgraph online["ServiceControl open on SQL: no way back to RavenDB"] - S7["7. Start ServiceControl"] - S8["8. Watch the background copy with --migration-status"] + S7["6. Start ServiceControl"] + S8["7. Watch the background copy with --migration-status"] Q2{"Is an optional category Failed?"} RetryAbandon2["Stop ServiceControl, run --migration-retry or --migration-abandon, then start it"] - S9["9. Stop ServiceControl, run --migration-verify, then start it"] + S9["8. Stop ServiceControl, run --migration-verify, then start it"] S7 --> S8 --> Q2 Q2 -->|Yes| RetryAbandon2 --> S8 Q2 -->|"No: every category is Done or Abandoned"| S9 end - S5 --> S6 + S4 --> S6 Q1 -->|Yes| S7 ``` @@ -99,7 +98,7 @@ In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You > [!IMPORTANT] > > - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. -> - Do not start ServiceControl again until step 7. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. +> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: @@ -153,14 +152,7 @@ The dry run reads RavenDB and tells you what the migration will do, including an Book the outage from the estimate in the report, and add a margin. It is an estimate, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. -## Step 5: Back up RavenDB - -> [!IMPORTANT] -> Back up both RavenDB databases, primary and throughput, before you start the copy. Or turn off RavenDB expiration on them. - -RavenDB keeps deleting old failed messages and event log items during the migration and after it. Without a backup, the RavenDB database that you keep to go back to loses data every day. - -## Step 6: Run the offline command +## Step 5: Run the offline command 1. Stop ServiceControl and every ingestion worker. 2. Run `ServiceControl.exe --start-data-migration` from the instance installation folder. In a container deployment, run the command in a one-off container against the same databases. @@ -169,7 +161,7 @@ RavenDB keeps deleting old failed messages and event log items during the migrat The command exits without starting ServiceControl. -## Step 7: Start ServiceControl +## Step 6: Start ServiceControl After the offline command reports success, start the ServiceControl service or container. ServiceControl detects the migration from SQL checkpoints and copies optional categories in the background. @@ -180,7 +172,7 @@ Normal startup refuses an unfinished required copy and tells you to run `--start If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). -## Step 8: Watch the background copy +## Step 7: Watch the background copy ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: @@ -195,7 +187,7 @@ If the copy slows down your production work, raise `ServiceControl/Migration/Thr Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. -## Step 9: Finish the migration +## Step 8: Finish the migration 1. Stop ServiceControl. 2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md index 1b3c015e16..43c8f20f73 100644 --- a/docs/migration/ravendb-to-sql-migration-overview.md +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -3,7 +3,7 @@ > [!NOTE] > This build does not carry the whole migration yet. This page describes it as it will be when it ships. -This page describes the migration as designed, not what is built today. [The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. +[The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. ## Overview @@ -20,8 +20,8 @@ This page describes the migration as designed, not what is built today. [The ins - **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. - **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. - **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. -- **The main thing that makes the outage long is a big backlog of unresolved failures.** They almost never age out (see the [6.18 exception](#rows-removed-after-the-copy)), they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. -- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds the unresolved failures through a RavenDB index, so while ServiceControl is closed it reads only them. If the index is still catching up, the copy waits for it first. If the index is broken, or leaves out some failed messages, the copy reads every failed message instead, archived and resolved ones included, and logs a warning saying so. Then a big archive does make the outage longer. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They are required, rarely age out (see the [6.18 exception](#rows-removed-after-the-copy)), and can be months old. Keep them low by resolving and archiving. The *dry run* tells you where you stand. +- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds unresolved failures through a RavenDB index, and waits for it to catch up first. If the index is broken or incomplete, the copy reads every failed message instead and logs a warning. Then a big archive does make the outage longer. ```mermaid flowchart LR @@ -59,7 +59,7 @@ stateDiagram-v2 - **Short outage.** Only the required data copies while ServiceControl is closed. - **Support every RavenDB hosting shape.** Embedded, self-hosted or RavenDB Cloud, on one code path. -- **Never write to RavenDB.** But RavenDB's own expiration keeps deleting old failed messages and event log items, during the migration and afterwards. So the old database is a fallback that degrades from the moment you start, which is why [the instructions](ravendb-to-sql-migration-instructions.md#step-5-back-up-ravendb) have you back it up first. +- **Never write to RavenDB.** RavenDB's own expiration still deletes old failed messages and event log items. - **No duplicates and no gaps.** The copied rows and the *checkpoint* commit together, so a crash needs no clean-up. - **Every identifier anything points at is kept.** Only the event log, historic retry operations and pending integration events get new ids, because nothing points at theirs. - **Refuses rather than half-migrates.** Every check runs before the first row moves. @@ -72,7 +72,7 @@ stateDiagram-v2 - **Zero downtime.** There is a real, if short, outage. - **Rollback once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. - **Pausing or steering a running copy.** There is no pause, resume or abort. Decisions are two commands run with ServiceControl stopped, and every other change is a configuration edit and a restart. -- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine was design to be expandable in the future. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine is designed so it can be expanded in the future. ### The moving parts @@ -87,7 +87,7 @@ flowchart LR ``` - **Both persisters load into one ServiceControl process**, each in its own `AssemblyLoadContext`, and both are live during the copy. -- **The engine knows neither database.** It knows only `IMigrationSource` and `IMigrationTarget`, and deals in categories, *cursors* and counts. This allows it to be tested, and expanded in the future. +- **The engine knows neither database.** It works only through `IMigrationSource` and `IMigrationTarget`, in categories, *cursors* and counts, so it can be tested on its own and expanded in the future. - The source maps each category to what it reads, and describes itself as labelled facts. - The target maps each category to where it writes and how to count it. - Each side adds its own startup checks. @@ -100,13 +100,13 @@ flowchart LR [The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. - **The [free abort](#going-back-to-ravendb) window runs from the first `--start-data-migration` invocation until ServiceControl opens.** A successful command does not end this window; starting ServiceControl does. -- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command builds a persistence-only host and never starts it. Normal startup refuses unfinished required checkpoints before any hosted service starts. Optional categories never hold them back. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command never starts the host, and normal startup refuses while a required category is unfinished. ## Design ### Supported locations -The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no database-to-database transfer. RavenDB holds documents and SQL holds tables, so the rows have to be reshaped, and ServiceControl's own persisters already know how to read one and write the other. So what matters is whether the host can reach both ends, and how far the data travels. +The copy goes from RavenDB, through the ServiceControl host, to SQL, because documents must be reshaped into tables and ServiceControl's persisters already know both sides. So the host must reach both ends, and distance matters. | | Supported locations | | --- | --- | @@ -126,12 +126,8 @@ The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no - The host reaches the RavenDB source, the SQL target and the body store at the same time. - The SQL database is new and empty: only the schema `--setup` created. - Both RavenDB databases, primary and throughput, are on one server or cluster. The migration opens both through one `IDocumentStore`, and so does the licensing component. -- A SQL Server target has Full-Text Search installed, because message search needs it. - - The `AddFullTextSearch` EF Core migration checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent. - - An EF Core migration runs once, so this is checked the first time `--setup` brings a database up to that migration, not on every `--setup`. - - A stock SQL Server container image does not include Full-Text Search. - - PostgreSQL needs nothing extra, because its search index is a GIN index over `to_tsvector`. -- Transient failures on a managed target are already survivable. Retry on failure is on by default, and no setting turns it off. +- A SQL Server target has Full-Text Search installed, because message search needs it. `--setup` fails without it, and the stock SQL Server container image does not include it. PostgreSQL needs nothing extra. +- Transient failures on a managed target are retried by default. **Not supported:** @@ -171,14 +167,14 @@ There are eighteen *categories*: sixteen required and two optional. - RavenDB index definitions. - In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations` and `UnarchiveOperations`. - `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. -- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. +- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Nothing has read it since 6.2. - Broker and audit service version details, which refill on the next throughput collection. - Failed message edit locks, which stop one failed message being edited twice. See [Open decisions](#values-that-change-or-merge-in-sql). -- Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. +- Heartbeat state, which neither persister stores. It is rebuilt in memory, so monitoring after the move behaves as after any restart. **Written new by the copy rather than copied:** -- Retry claims in `FailedMessageRetries` are not copied. A claim ties a message to the retry that sent it, and while it exists, retrying the message from the failed messages list sends nothing. The copy writes a new claim for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). As on RavenDB, the claim goes when the retried message succeeds, when it fails again, or when it is resent from Pending Retries. +- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts, see [Open decisions](#values-that-change-or-merge-in-sql). ### Optional data windows @@ -203,7 +199,7 @@ Every skipped row is counted under a reason, and its id is written to the log. A skip is one of three kinds: - **Fault, a retry can fix it.** Something outside the data went wrong. Fix the cause and retry. -- **Fault, no retry can fix it.** The RavenDB document itself cannot become a SQL row. A retry reads the same document and skips it again, so only [abandoning](#retry-and-abandon) gets past it. It is a fault because the row is lost, not because anything is broken. +- **Fault, no retry can fix it.** The RavenDB document cannot become a SQL row, so a retry skips it again. Only [abandoning](#retry-and-abandon) gets past it. - **Harmless.** A row SQL would have removed anyway. A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Open decisions](#values-that-change-or-merge-in-sql). @@ -219,12 +215,11 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied | Reason | | --- | -| A message body that RavenDB does not hold at all, counted as `BodyMissing`. The whole message is skipped, for the same reason. The one exception is a message that expired during the copy (see below), which a retry does get past | +| A message body that RavenDB does not hold, counted as `BodyMissing`. The whole message is skipped. A retry does get past a message that expired during the copy | | A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | | A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | -| A subscription whose message type or address is over 200 characters, the SQL key limit | | A row missing a value SQL requires, such as a known endpoint with no name, or a failed message with no failing endpoint address to retry to | -| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters. Cutting it would change a key or send retries to the wrong address, so the row is skipped. The skip names the column and the length, counted the way the database counts (PostgreSQL by characters, SQL Server by UTF-16 units) | +| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters, or a subscription's message type or address. Cutting it would change a key or send retries to the wrong address. The skip names the column and the length | | A value of the wrong type, such as a `TimeSent` that is not a date. On PostgreSQL, also a text value holding the character U+0000, which PostgreSQL cannot store | **Harmless skips:** @@ -235,16 +230,13 @@ A value SQL can hold only after changing it is not a skip. Those rows are copied | An archived or resolved message, or event log item, already past its retention period | - **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. -- **A malformed document is skipped and counted, not a reason to stop the copy on its own.** The rest of the category carries on, so abandoning a category that read to the end and ended Failed this way gives up only those rows. But if the skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then also gives up every row it had not yet read. In that case raise the threshold or fix the cause, and retry. -- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them shortly after it opens, by the same rule it always applies, so the migration does not repeat that rule. +- **A malformed document does not stop the copy.** It is skipped and counted, and the rest of the category carries on. If skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then gives up every unread row too. Raise the threshold or fix the cause, and retry. +- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them itself shortly after it opens. - **Rows outside a window are not skips.** They are simply not part of the copy. -- **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. - - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). - - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. - - The exception is an unresolved message from version 6.18 or earlier, which can still expire during the copy. See [Open decisions](#rows-removed-after-the-copy). - - A failed message that expires after the copy reads it, but before the copy reads its body, is a skip. It counts as a body RavenDB does not hold, so it is a fault skip and the category ends Failed. For unresolved messages this happens only with a source last written by version 6.18 or earlier. A retry gets past it, because the expired message is no longer there to read. - - If the copy stops, and the message it stopped at expires before the next start, that start cannot carry on. The category stops with an error saying RavenDB holds no document with that id. If RavenDB is the right database, `--migration-retry` gets past it, because a retry reads the category from the start. - - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. +- **Rows RavenDB expires during the copy are not skips either.** Only archived or resolved failed messages and event log items expire, plus some unresolved messages from 6.18 or earlier (see [Open decisions](#rows-removed-after-the-copy)). + - A message that expires between the copy reading it and reading its body is a `BodyMissing` fault skip. A retry gets past it. + - If the message a stopped copy was at expires before the next start, the category stops with an error naming the missing id. `--migration-retry` gets past it. + - A window shorter than the retention period keeps the background copy clear of the expiry sweep. ### Startup checks @@ -252,7 +244,7 @@ In this order, cheapest first: 1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. `--start-data-migration` refuses a RavenDB target, because RavenDB is only ever the source. 2. The optional category windows are valid time spans of zero or more. -3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history, and no row count would ever show it. +3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history. 4. The SQL schema is current. This also refuses a database upgraded without `--setup`. 5. Until a required category has started copying: the SQL database holds no ServiceControl data. `--setup` writes none, but even one plain start of ServiceControl writes rows. Copying into a database that already served on SQL would mix the two. 6. Message body storage is writable. The probe body it writes is deleted again. @@ -265,7 +257,7 @@ In this order, cheapest first: If every required category is already Done or Abandoned, the offline command can succeed without opening the source. Normal startup does not need the source; only the online copier opens it. -Checks stop at the first one that refuses, and the refusal names it. Once open, the source can add checks of its own. The RavenDB source adds none. +Checks stop at the first refusal, which names the check. ```mermaid flowchart LR @@ -285,24 +277,17 @@ flowchart LR - **The source opens RavenDB read-only**: connect, check the version, stop. It never runs RavenDB's database setup, and it refuses any write request. - **Both source databases are opened through one `IDocumentStore`**, which is why they must be on the same server or cluster. -- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. A RavenDB start rewrites no stored document, so the copy reads exactly what this build's RavenDB persister would read. The only version check compares the RavenDB server to the RavenDB client, and only for an external source. -- **Distance to the source sets the pace.** The copier already holds each document from the stream, so each body costs one round trip rather than two. But it is one per message, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. +- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. +- **Distance to the source sets the pace.** Each message body is one round trip, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. ### Writing to SQL - **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). -- **A failed message keeps its failure groups and its last-modified time.** `LastModified` is RavenDB's own `@last-modified`, copied as it is, so failure groups, and lists filtered by when a message last changed, show the same messages as before. -- **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. -- **`StatusChangedAt` is rebuilt.** - - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. `@expires` is the status-change time plus `ServiceControl/ErrorRetentionPeriod`, so the copy subtracts the current value. A changed retention period makes this time wrong, see [Open decisions](#values-that-change-or-merge-in-sql). - - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. -- **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. -- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier decides inline or external storage with the same rule ingestion uses, because `IBodyStoragePersistence` does not apply it. The size limit for inline storage defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. -- **Throughput rows are written directly, not through the collector, and each day's count is set, not added.** - - Throughput is required, so it copies while ServiceControl is closed, before any collector has written to SQL. - - Setting makes the category safe to resume after a crash, where adding would double-count. The collector's own path does add (`LicensingDataStore.cs:198`). - - Copying the rows stops the audit and broker collectors gathering the same days again, because `LastCollectedDate` is worked out from the newest throughput row (`LicensingDataStore.cs:45`). - - The checkpoint stops a second pass overwriting days the collectors have written since. +- **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. +- **`UniqueMessageId` keeps its value** but becomes a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry key and the body key. +- **`StatusChangedAt` is rebuilt.** For archived and resolved messages it is `@expires` minus the current `ServiceControl/ErrorRetentionPeriod` (wrong if that changed, see [Open decisions](#values-that-change-or-merge-in-sql)). For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL`. +- **Bodies go through `IBodyStoragePersistence`**, which handles compression and the body store. The copier keeps a body inline when it is within `ServiceControl/MaxBodySizeToStore` (102,400 bytes by default), as ingestion does. +- **Throughput rows are written directly, and each day's count is set, not added**, so a resumed copy cannot double-count. The copied rows also stop the collectors gathering the same days again. ### Batch size and throttling @@ -313,7 +298,7 @@ flowchart LR ### Progress and checkpoints -The checkpoint is one row per category, kept in the SQL database and created by `--setup`. It holds the category's *state*, its cursor, its copied, skipped and already-present counts, how many of the already-present rows were merges, the count per skip reason, how many rows SQL held for the category when it settled, timings, the last error, the window it copies with, and a version number to catch a second writer. +The checkpoint is one row per category in the SQL database, created by `--setup`. It holds the category's *state*, cursor, counts, timings, last error and window, plus a version number that catches a second writer. ```mermaid sequenceDiagram @@ -331,21 +316,20 @@ sequenceDiagram - **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. - **A crash costs only the batch in flight.** Rerun `--start-data-migration` for required data, or restart ServiceControl for optional data. The copier resumes from the committed cursor. -- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. Unreadable bodies stay off the checkpoint until the write commits. -- **Every skip has a reason, or the save is refused.** The per-reason counts must add up exactly to the skipped count, and the target's counts must match what it committed. -- **The stored counts always describe the rows in SQL.** The target adds its own outcome and saves the result beside the rows, so nothing is saved later or separately. -- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts, or a command and a host, pointed at one database cannot interleave. -- **A copy stopped between two categories still shows the ones it never reached.** Before copying anything, a start saves a *not started* row for each category it is about to copy, so every gate sees them as unfinished. +- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. +- **Every skip has a reason, or the save is refused.** +- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts pointed at one database cannot interleave. +- **A copy stopped between two categories still shows the ones it never reached.** Each start first saves a *not started* row for every category it will copy. - **Only the copier and the two commands write the checkpoint.** The status and verify commands only read it. ### Failed categories -A category is *Failed* when something went wrong. Everything already copied stays committed. Nothing is retried in the background and nothing waits for a timer: the category stays Failed, with the reason written on it, until you run a [command](#retry-and-abandon). +A category is *Failed* when something went wrong. What it copied stays committed, and it stays Failed, with the reason, until you run a [command](#retry-and-abandon). **What makes a category Failed:** - **Any fault skip by the end of the category**, even one. -- **Too many fault skips in this run, which stops it early.** More than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Needing both means a tiny category does not stop on one bad row, and a huge one does not stop on its 101st. Under the threshold a category runs to the end, so you get the whole list of bad rows in one pass. +- **Too many fault skips in this run, which stops it early:** more than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Below that, a category runs to the end, so you see every bad row in one pass. - **The category's own counts not balancing:** every row read must be copied, skipped or already present. - **In a required category only, an exception or a stall of 30 minutes with nothing committed**, such as the body store unreachable, a certificate expired, the source or target down, or the disk full. The error and the cursor are recorded. @@ -353,21 +337,19 @@ A category is *Failed* when something went wrong. Everything already copied stay - **Harmless skips.** - **A shutdown or a crash.** The category stays Copying. The next `--start-data-migration` invocation resumes required data; the next ServiceControl start resumes optional data. -- **An exception or a stall in an optional category.** The error is written on the category, shown in status, and the next start carries on from the cursor. A RavenDB outage after opening is handled the same way. So a network blip never forces anything to be abandoned. If the error keeps coming back, retry or abandon the category. +- **An exception or a stall in an optional category**, including a RavenDB outage. The error shows in status, and the next start carries on from the cursor. If it keeps coming back, retry or abandon the category. **What a Failed category costs:** -- **The other categories still run.** Every required category gets its go in one start, so one fix-and-retry covers them all. A category that must follow a Failed one waits. +- **The other categories still run**, so one fix-and-retry covers them all. A category that depends on a Failed one waits. - **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. - **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. -**The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. - Some losses leave a category Done without any check noticing. They are listed under [Open decisions](#what-the-checks-do-not-catch). ### Retry and abandon -Stop ServiceControl before running either command. Rerun `--start-data-migration` for unfinished required data before starting the service again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. +Stop ServiceControl before either command. In a container, run it as a one-off `docker run --rm ` against the same database. | Command | Works on | What it does | | --- | --- | --- | @@ -383,36 +365,34 @@ Stop ServiceControl before running either command. Rerun `--start-data-migration **What a retry does:** -- **It re-reads the whole category from the start**, every time, which recovers from anything. Rows already in SQL are left alone, so only the rows that failed can change. Every message body is read again, so a retry takes about as long as the first copy. -- **Historic retry operations and pending integration events** get new keys from SQL on every insert, so a re-read could not recognise rows already copied. A retry of either first deletes what that category copied, in the same save as the reset. That is safe because both are required, so ServiceControl has not opened and the target started empty. +- **It re-reads the whole category from the start**, leaving rows already in SQL alone. Every body is read again, so a retry takes about as long as the first copy. +- **Historic retry operations and pending integration events** get new keys on every insert, so a retry first deletes what the category copied, in the same save as the reset. This is safe because both are required, so the target started empty. - **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. **To abandon:** -- **A required category can be abandoned once it is Failed or has started copying.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone before it starts, point back or start over: ServiceControl has not opened, so nothing being served is lost. +- **A required category can be abandoned once it has started.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone first, point back or start over. - **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. -- **Abandoning cascades to categories whose rows hang off it**, and the command names each: throughput history (it points at licensing endpoint records) and endpoint settings (it belongs to known endpoints). Group comments only waits for unresolved failed messages, so it is not abandoned with them. -- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a value too long for its column, a value of the wrong type, a missing required value, a body RavenDB does not hold. The refusal and the dry run say which, so abandon after the first failure rather than retrying. The exception is a category that stopped early at the threshold: abandoning it also gives up every row it had not yet read, so raise the threshold and retry first. +- **Abandoning cascades** to throughput history (from licensing endpoint records) and endpoint settings (from known endpoints), and the command names each. Group comments are not abandoned with unresolved failed messages. +- **A category that failed only on [skips no retry can fix](#skipped-rows) should be abandoned, not retried.** If it stopped early at the threshold, raise the threshold and retry first, because abandoning gives up every unread row too. > [!IMPORTANT] > Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. -**Nothing happens on a restart that you did not ask for.** A container or service restart policy restarts a refused host without anyone looking, and gets the same refusal each time, without reading anything. That is loud and loses nothing. - ### Ingestion workers -- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, as before, because only SQL can scale out. +- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, because only SQL can scale out. - **On SQL it starts only when every required category is Done or Abandoned**, based on SQL checkpoints. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. - **Every SQL writer runs that check**, because ingesting into a part-copied database would make going back a loss. - **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. -- **A worker or import that gets past the check marks the target as opened**, without an enable setting. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. +- **A worker or import that passes the check marks the target as opened**, which ends the free abort, because going back would lose what it wrote. > [!IMPORTANT] > **Stop workers before running `--start-data-migration`, and restart them only after the main host has opened.** A worker cannot detect a migration before its first checkpoint exists, see [Open decisions](#gaps-nothing-enforces). ### Going back to RavenDB -**While ServiceControl is closed**, only the copier has written to SQL and the migration has written nothing to RavenDB, which is still the source of truth. This is the *free abort*. RavenDB's own expiration still runs, though, unless you disabled it. +**While ServiceControl is closed**, RavenDB is still the source of truth, because the migration writes nothing to it. This is the *free abort*. RavenDB's own expiration still runs unless you disabled it. - **To go back:** point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. - **To try again later:** start from a new, empty SQL database (`--setup`). @@ -422,15 +402,9 @@ Stop ServiceControl before running either command. Rerun `--start-data-migration > > **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. -- **Every SQL start checks the checkpoint table.** A RavenDB instance has no checkpoint store and skips the check. -- **An unfinished required category keeps the host closed.** Run `--start-data-migration` again, after retrying or abandoning Failed categories. Startup never runs the required copy. -- **Outstanding optional checkpoints activate the online copier.** They do not prevent ServiceControl from opening. Failed optional categories remain Failed until an operator acts. -- **With no outstanding categories, ServiceControl does not open RavenDB.** Completed checkpoints remain as the migration record. -- **Before starting the copy, the offline command seeds required and selected optional checkpoints.** The online phase does not depend on a setting that an operator must remember to retain. - -Before ServiceControl opens, the other exits remain pointing back at RavenDB or starting over against a new, empty SQL database. +- **Every SQL start checks the checkpoint table.** An unfinished required category keeps the host closed until `--start-data-migration` finishes it. Outstanding optional categories start the online copier but do not stop ServiceControl opening. With nothing outstanding, ServiceControl does not open RavenDB. -**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl has opened, nothing can start without it. After, ServiceControl still opens, and the optional categories wait for RavenDB to come back, or can be abandoned. +**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl opens, nothing can start without it. After that, the optional categories wait for it, or can be abandoned. ### The dry run @@ -442,8 +416,8 @@ The dry run never writes to RavenDB, and can run again later against whatever is - An estimate of how long ServiceControl will be closed. - **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. - The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. -- How many failed error imports there are, with advice to run `--import-failed-errors` on RavenDB first. They are copied either way, but ones imported first arrive as ordinary failed messages. -- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure, unless that failure stopped it early at the threshold. +- How many failed error imports there are. They are copied either way, but running `--import-failed-errors` on RavenDB first is better. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. It says nothing about load on the source. @@ -453,22 +427,21 @@ It says nothing about load on the source. | --- | --- | --- | | `--migration-source-report` | Source facts and a document count per collection | Yes | | `--migration-dry-run` | The report above | Yes | -| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read, with skips and merges broken out. Exits 0 only when every category is Done or Abandoned, so a script can ask whether the migration is finished. Rows the copy never read are shown, not failed | Yes | +| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read. Exits 0 only when every category is Done or Abandoned | Yes | | `--migration-status` | Each category's state, progress, last error and the commands it can take | No, so it runs any time | - **On an embedded source, stop the ServiceControl service before any command that opens RavenDB.** A second RavenDB process cannot use a data directory the first one holds, so the dry run is part of the outage. -- **Verify reads every selected category in full**, though not the message bodies. On a big instance that takes a while, and on an embedded source that time is part of the outage too. +- **Verify reads every selected category in full**, though not the bodies, so it takes a while on a big instance. - **On an external source** they all run against a live instance. - **In a container**, run each as a one-off `docker run` of the same image against an external RavenDB server. **What verify compares:** -- **RavenDB against the rows the copy read, never against SQL as it is now.** Once ServiceControl opens, normal use changes most categories' SQL rows within minutes: ingestion, heartbeats, retries, archiving, the licensing collectors, ServiceControl's own custom checks and the retention sweep. A SQL count then measures that use, not the copy. -- **RavenDB holding more rows than the copy read means rows were never read.** Nothing writes to RavenDB once the copy starts, so it can only shrink. Verify flags these rows. They are not in SQL, so keep RavenDB. The same flag shows if something other than this migration wrote to RavenDB since, such as an old instance still running on it. -- **RavenDB holding fewer is shown, not judged.** RavenDB's expiry keeps deleting archived and resolved messages, event log items and a few unresolved messages from 6.18 or earlier, after the copy has read them. -- **The same expiry can hide rows that were never read**, see [Open decisions](#what-the-checks-do-not-catch). -- **A windowed category is counted inside the window it started with**, on RavenDB only. -- **Verify also prints how many rows SQL held for each category when it settled.** Before ServiceControl opens nothing else writes, so for a required category that is exactly what the copy left in SQL. For the two optional categories it includes what ServiceControl wrote while they copied. +- **RavenDB against the rows the copy read, never against SQL now**, because normal use changes SQL rows within minutes of opening. +- **RavenDB holding more rows than the copy read means rows were never read.** Keep RavenDB. The same flag shows if something else wrote to RavenDB, such as an old instance still running on it. +- **RavenDB holding fewer is shown, not judged**, because expiry keeps deleting rows after the copy reads them. Expiry can also hide rows that were never read, see [Open decisions](#what-the-checks-do-not-catch). +- **A windowed category is counted inside the window it started with.** +- **Verify also prints how many rows SQL held for each category when it settled.** ### Monitoring and settings @@ -482,30 +455,29 @@ It says nothing about load on the source. - **The source needs no new settings.** It reads the instance's existing RavenDB settings. - **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. -- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. Finished checkpoints deactivate copying automatically. There is no migration custom check, because custom checks are being removed from ServiceControl. -- **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. -- **Stopping a copy takes a restart.** +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** There is no migration custom check instead migration status will be surfaced with a custom ServicePulse UI. +- **There is no HTTP API, pause, resume or abort.** Stopping a copy takes a restart. ## Open decisions -The design currently accepts each case below. Each one changes, merges or drops data, or lets a loss pass without failing a category. None is settled: each needs a decision before the migration ships. Where the design gives a reason for accepting a case, it is stated. +Each case below changes, merges or drops data, or lets a loss pass without failing a category. The design accepts each one for now, but each needs a decision before the migration ships. ### Values that change or merge in SQL -- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so it keeps the newest attempt and the number of attempts. A message that failed five times still shows five attempts and still shows as a repeated failure, but only the newest attempt keeps its headers and exception. This applies to every failed message, whatever its status. -- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. -- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. - - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. - - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. - - The copy logs a warning naming both endpoints and the one it kept. - - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **Attempt history collapses to the newest attempt.** SQL keeps the newest attempt and the attempt count, so only the newest attempt keeps its headers and exception. +- **Message redirects, subscriptions and pending retry operations merge when their keys match in SQL.** The copy keeps one row, counts the other as already present, and logs a warning naming both. + - Redirects merge on source addresses the key column treats as equal. On SQL Server, by default, that includes addresses that differ only in case. + - Subscriptions also merge when they differ only in message-type version, because the SQL key leaves the version out. + - Pending retry operations with the same retry type and request id merge, keeping the latest. +- **On SQL Server, endpoint settings whose names differ only in case merge into one row**, because the name column compares without case by default. PostgreSQL keeps both. + - The copy keeps the setting whose name exactly matches a known endpoint, because ServiceControl deletes the other. Otherwise it keeps the first in RavenDB order, which is effectively arbitrary. + - The choice is made within one batch of up to a thousand. Across batches, the first one met wins. + - The copy logs a warning, and the dry run lists each pair. - **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. -- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row.** SQL Server ignores trailing spaces when it compares the key. PostgreSQL keeps both. - - The copy keeps the endpoint whose RavenDB document id sorts first in ordinal order. - - A day that both endpoints counted holds the sum of their counts, as it would if SQL Server had recorded both endpoints from the start. - - This is not counted as a merge. The copy logs no warning, and the dry run does not list them. The second endpoint and its share of each shared day count as already present. - - The copy adds counts only for names that differ in case and trailing spaces. A key column given a collation that also ignores, for example, accents still makes one row of other names, and that row keeps one endpoint's count. The other endpoint's counts are lost. -- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** The copy subtracts the current value from `@expires`, so the rebuilt time is off by the difference. The message still ages out of SQL when RavenDB would have deleted it, because SQL adds the same current value back when it checks the row against the retention period. +- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row**, and a day both counted holds the sum. PostgreSQL keeps both. + - This is not reported as a merge: the copy logs no warning, and the dry run does not list it. + - A key column whose collation also ignores, for example, accents still makes one row, but keeps only one endpoint's counts. The other's counts are lost. +- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** It is off by the difference, but the message still ages out of SQL when RavenDB would have deleted it. - **Every retry-issued message gets a retry claim, even one whose claim RavenDB had already released.** While the claim exists, retrying the message from the failed messages list sends nothing. - **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. @@ -513,21 +485,21 @@ The dry run counts the subscription and endpoint settings merges before anything ### Rows removed after the copy -- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens.** ServiceControl's usual clean-up removes it, because that group has no failed messages in SQL until the background copy brings the archived and resolved ones. -- **Unresolved messages from version 6.18 or earlier can expire during the copy.** A message that failed again after being archived or resolved on 6.18 or earlier kept its old expiry, and upgrading does not remove it. One that RavenDB deletes before the copy reads it is not copied and not counted as a skip. +- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. +- **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. ### What the checks do not catch -- **Nothing counts RavenDB during the copy.** A read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read. -- **Rows never read on a Done category are reported, not fixed.** Verify prints them, but its exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before ServiceControl opens. The current reasoning: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. -- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Only tests on each reader cover those. -- **Expiry can hide rows that were never read.** In verify, a deleted row and a missed row cancel out, so a flagged number is a floor. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. -- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. -- **A crash during a batch that carries bodies leaves orphan bodies in the body store.** External bodies are written before the transaction opens, so a crash leaves bodies no row points at, never rows pointing at missing bodies. +- **Nothing counts RavenDB during the copy.** A read that ends early still balances and ends Done. `--migration-verify` finds it afterwards. +- **Rows never read on a Done category are reported, not fixed.** Verify's exit code ignores them, and no command can recopy a Done category. Keeping RavenDB is the remedy. +- **Verify uses the same reader as the copy**, so it cannot catch a reader that always misses the same rows, or a source pointed at the wrong database. +- **Expiry can hide rows that were never read**, because in verify a deleted row and a missed row cancel out. +- **A write that silently stores fewer rows still ends Done**, because already present is what is left after copied and skipped. +- **A crash during a batch with bodies leaves orphan bodies** in the body store, never rows pointing at missing bodies. ### Gaps nothing enforces -- **An ingestion worker started at the same moment as the first migration start can slip in before the copy writes its first checkpoint.** The ingestion gate cannot see a copy with no checkpoint row, so the worker ingests into a part-copied database. Only the instructions prevent this. +- **An ingestion worker started at the same moment as the first migration start can slip in before the first checkpoint exists**, and ingest into a part-copied database. Only the instructions prevent this. - **Reusing an old copy after going back to RavenDB cannot be detected**, because it looks exactly like a normal resume. The copy skips every category already finished and misses anything RavenDB received since. ## Glossary diff --git a/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs new file mode 100644 index 0000000000..207bfd706f --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +[TestFixture] +[NonParallelizable] +class MessageRedirectsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_three_redirects_from_one_document_into_three_rows() + { + var seeded = new[] + { + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = new DateTime(2026, 3, 1, 12, 0, 1, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = new DateTime(2026, 3, 1, 12, 0, 2, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old3", ToPhysicalAddress = "new3", LastModified = new DateTime(2026, 3, 1, 12, 0, 3, DateTimeKind.Utc) } + }; + await SeedSourceMessageRedirects(seeded); + + var result = await CopyCategory("MessageRedirects"); + + Assert.That(result.Copied, Is.EqualTo(3)); + + var redirects = await MessageRedirectsDataStore.GetRedirects(); + Assert.That(redirects.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified)), + Is.EquivalentTo(seeded.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified))), + "each redirect keeps its own destination and the time it was last changed"); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!), Is.EqualTo(3), "--migration-verify reads this count"); + } + + [TestCase("old1", "old1")] + [TestCase("Sales@HOST", "sales@host")] + public async Task A_repeated_address_is_copied_or_already_present_as_the_database_compares_it(string firstAddress, string secondAddress) + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = firstAddress, ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = secondAddress, ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + var stored = await MessageRedirectsDataStore.GetRedirects(); + + Assert.That(result.Skipped, Is.Zero, "a repeat loses no row the target lacks, so it is never a skip"); + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(result.Copied, Is.EqualTo(stored.Count), + "Copied is the rows the database stored: one where the key treats the addresses as the same, two where it does not"); + Assert.That(stored.Select(redirect => (redirect.FromPhysicalAddress, redirect.ToPhysicalAddress)), Does.Contain((firstAddress, "new1")), + "the first redirect in document order is copied on every database"); + } + + [Test] + public async Task A_redirect_with_no_destination_is_skipped_as_a_required_value_missing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = null!, LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = null!, ToPhysicalAddress = "new3", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "old1", "messageredirects#2" }), "a redirect with no source address is named by its place in the document"); + + var checkpoint = await ReadCheckpoint("MessageRedirects"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs index bcc073ccfc..f93b77210c 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -15,8 +15,11 @@ namespace ServiceControl.Migration.AcceptanceTests; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; using NServiceBus.Extensibility; using NServiceBus.Transport; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; using NUnit.Framework; using Particular.LicensingComponent.BrokerThroughput; using Particular.LicensingComponent.Contracts; @@ -29,6 +32,8 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Infrastructure.WebApi; using ServiceControl.MessageFailures; using ServiceControl.Migration.Checks; +using ServiceControl.Monitoring; +using ServiceControl.Notifications; using ServiceControl.Operations; using ServiceControl.Operations.BodyStorage; using ServiceControl.Persistence; @@ -36,10 +41,13 @@ namespace ServiceControl.Migration.AcceptanceTests; using ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Tests; using ServiceControl.Persistence.UnitOfWork; +using ServiceControl.Recoverability; using ServiceControl.Transports.BrokerThroughput; using TestHelper; // Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. @@ -224,6 +232,43 @@ protected Task SeedThroughputReportMasks(IReadOnlyList maskedStrings) => protected Task SeedThroughputLicensedEndpointDetails(LicensedEndpointDetails details) => SeedSource(Source.ThroughputDatabase, ("LicensedEndpointDetails", details)); + protected Task SeedSourceMessageRedirects(params MessageRedirect[] redirects) => + SeedSource(("messageredirects", new + { + Redirects = redirects.Select(redirect => new { redirect.FromPhysicalAddress, redirect.ToPhysicalAddress, LastModifiedTicks = redirect.LastModified.Ticks }).ToList() + })); + + // Stores the type as the string MessageTypeConverter writes, because that is what the persister reads. + protected Task SeedSourceSubscriptions(params SubscriptionRow[] subscriptions) => + SeedSource(("Subscriptions/All", new + { + All = subscriptions + .GroupBy(subscription => new MessageType(subscription.MessageType, subscription.Version).ToString()) + .ToDictionary(group => group.Key, group => new + { + Id = group.Key, + MessageType = group.Key, + Subscribers = group.Select(subscription => new { subscription.TransportAddress, subscription.Endpoint }).ToList() + }) + })); + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + protected async Task SeedSourceNotificationSettings(EmailNotifications emailNotifications, string collection = "NotificationsSettingsDocuments") + { + var document = new { Email = emailNotifications }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, "NotificationsSettings/All"); + session.Advanced.GetMetadataFor(document)["@collection"] = collection; + await session.SaveChangesAsync(); + } + + protected Task SeedSourceTrialEndDate(DateOnly endDate) => + SeedSource(("metadata/trialinformation", new TrialMetadata { TrialEndDate = endDate })); + + protected Task SeedSourceRetryHistory(RetryHistory history) => + SeedSource(("RetryOperations/History", history)); + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => SeedSourceWithBodies((document, body, contentType)); @@ -442,11 +487,60 @@ protected static async Task> CollectBatches(IMigrationSourc return batches; } + SyncPassCountingLogger syncPasses; + + protected void CountSyncPasses(IServiceCollection services) + { + syncPasses = null; + services.AddSingleton>(provider => + syncPasses = new SyncPassCountingLogger(provider.GetRequiredService().CreateLogger())); + } + + protected async Task WaitForSyncPass(FakeTimeProvider time) + { + await WaitUntil(() => Task.FromResult(syncPasses?.Started >= 1), "the heartbeat settings sync started a pass"); + + time.Advance(TimeSpan.FromHours(6) + TimeSpan.FromSeconds(1)); + + await WaitUntil(() => Task.FromResult(syncPasses.Started >= 2), "the heartbeat settings sync finished its pass and reached its next tick"); + } + + sealed class SyncPassCountingLogger(ILogger inner) : ILogger + { + int started; + + public int Started => started; + + public IDisposable BeginScope(TState state) where TState : notnull => inner.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (state is IReadOnlyList> values + && values.Any(value => value.Key == "{OriginalFormat}" && Equals(value.Value, "Performing sync for {ServiceName}"))) + { + Interlocked.Increment(ref started); + } + + inner.Log(logLevel, eventId, state, exception, formatter); + } + } + protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService(); protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService(); protected IRetryBatchStore RetryBatchStore => targetServices.GetRequiredService(); protected IRetryStagingStore RetryStagingStore => targetServices.GetRequiredService(); protected IFailedMessageRetryDataStore FailedMessageRetryDataStore => targetServices.GetRequiredService(); + protected IMessageRedirectsDataStore MessageRedirectsDataStore => targetServices.GetRequiredService(); + protected INotificationsDataStore NotificationsStore => targetServices.GetRequiredService(); + protected ITrialLicenseDataProvider TrialLicenseDataProvider => targetServices.GetRequiredService(); + protected IRetryHistoryDataStore RetryHistoryStore => targetServices.GetRequiredService(); + + // Built by hand, because its DI constructor needs NServiceBus settings the target services do not have. + // A zero cache duration makes every read go to the table. + protected ISubscriptionStorage SubscriptionStorage => + new SubscriptionStorage(targetServices.GetRequiredService(), "Particular.ServiceControl", "Particular.ServiceControl", [], TimeSpan.Zero, TimeProvider.System); protected async Task CopyCategory(string categoryId, CancellationToken cancellationToken = default) { @@ -583,6 +677,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; diff --git a/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs new file mode 100644 index 0000000000..14ba89749c --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class NotificationSettingsMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory NotificationSettingsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.NotificationSettings)!; + + // Every field differs from its default, so a field the copy drops cannot match by accident. + static EmailNotifications Seeded => new() + { + Enabled = true, + SmtpServer = "smtp.contoso.com", + SmtpPort = 587, + AuthenticationAccount = "servicecontrol", + AuthenticationPassword = "not-a-real-password", + EnableTLS = true, + To = "ops@contoso.com", + From = "servicecontrol@contoso.com" + }; + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + [TestCase("NotificationsSettingsDocuments", "NotificationsSettings")] + [TestCase("NotificationsSettings", "NotificationsSettingsDocuments")] + public async Task Copies_notification_settings_as_one_row_in_settings_from_either_collection(string collection, string otherCollection) + { + var seeded = Seeded; + await SeedSourceNotificationSettings(seeded, collection); + + var result = await CopyCategory("NotificationSettings"); + + var collections = (await SourceStore.Maintenance.ForDatabase(Source.PrimaryDatabase).SendAsync(new GetCollectionStatisticsOperation())).Collections; + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(collections, Does.ContainKey(collection).And.Not.ContainKey(otherCollection), "the test proves nothing unless the seed landed in the collection this case names"); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(seeded)), "LoadSettings reads through GetSetting, so every field must survive that read"); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } + + [Test] + public async Task A_document_with_no_email_settings_copies_nothing() + { + await SeedSourceNotificationSettings(null); + + var result = await CopyCategory("NotificationSettings"); + + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.Zero, + "a row may not carry a null document, and the product reads a null value and a missing row as the same default"); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(new EmailNotifications()))); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs new file mode 100644 index 0000000000..da6e68e86e --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +[TestFixture] +[NonParallelizable] +class RetryOperationsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_one_document_into_both_target_tables() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-10), + CompletionTime = CompletedAt.AddMinutes(-9), + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + }); + history.HistoricOperations.Add(HistoricOperation("group-3", CompletedAt.AddMinutes(-20))); + history.UnacknowledgedOperations.Add(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-5), + CompletionTime = CompletedAt.AddMinutes(-4), + Last = DateTime.MaxValue, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + }); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(3)); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)), Is.EqualTo(3), "--migration-verify counts both tables, and a count of either table alone would report 2 or 1"); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(Columns), Is.EqualTo(history.HistoricOperations.Select(Columns)), "every column of the historic operation arrives as the source holds it"); + Assert.That(stored.UnacknowledgedOperations.Select(Columns), Is.EqualTo(history.UnacknowledgedOperations.Select(Columns)), "every column of the pending operation arrives as the source holds it, even a Last of DateTime.MaxValue, which InMemoryRetry records when it has no last time"); + } + + [Test] + public async Task Duplicate_pending_operations_with_one_key_copy_once_and_count_the_rest_as_already_present() + { + var history = new RetryHistory(); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt, originator: "First in the document")); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt.AddMinutes(1), originator: "Second in the document")); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the second lands on the key the first fills, so it loses no row the target lacks and is neither a skip nor a primary-key violation"); + Assert.That(result.Skipped, Is.Zero); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.UnacknowledgedOperations.Single().Originator, Is.EqualTo("Second in the document"), "RavenDB appends a later retry of the same group, and the SQL store keeps the latest one, so the copy keeps the last in document order"); + } + + [Test] + public async Task Copies_more_operations_than_one_statement_could_carry() + { + var history = new RetryHistory(); + + foreach (var index in Enumerable.Range(0, OperationsPerList)) + { + history.HistoricOperations.Add(HistoricOperation($"historic-{index}", CompletedAt.AddMinutes(index))); + history.UnacknowledgedOperations.Add(PendingOperation($"pending-{index}", CompletedAt)); + } + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(2 * OperationsPerList)); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.HistoricOperations.Select(operation => operation.RequestId))); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.UnacknowledgedOperations.Select(operation => operation.RequestId))); + } + + // More rows than SQL Server's 2,000 parameters could carry at even one column each. + const int OperationsPerList = 2_100; + + // Whole seconds, because PostgreSQL stores microseconds and would not give back a time with more precision. + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + static (string, RetryType, DateTime, DateTime, string, bool, int) Columns(HistoricRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Originator, operation.Failed, operation.NumberOfMessagesProcessed); + + static (string, RetryType, DateTime, DateTime, DateTime, string, string, bool, int) Columns(UnacknowledgedRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Last, operation.Originator, operation.Classifier, operation.Failed, operation.NumberOfMessagesProcessed); + + internal static HistoricRetryOperation HistoricOperation(string requestId, DateTime completedAt) => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + internal static UnacknowledgedRetryOperation PendingOperation(string requestId, DateTime completedAt, string originator = "Shipping") => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Last = completedAt, + Originator = originator, + Classifier = "Shipping-Failures", + NumberOfMessagesProcessed = 1 + }; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs new file mode 100644 index 0000000000..6d07667274 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Recoverability; + +[TestFixture] +[NonParallelizable] +class RetryResetMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_retry_of_retry_operations_leaves_both_tables_as_one_copy_left_them() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var copied = await Target.Count(RetryOperations); + + var saved = await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.RetryOperations))); + var stored = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + var second = await CopyCategory(MigrationCategoryIds.RetryOperations); + + using (Assert.EnterMultipleScope()) + { + Assert.That(saved.Version, Is.EqualTo(stored.Version), "the returned checkpoint carries the version the reset saved"); + Assert.That(saved.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(copied, Is.EqualTo(4), "the first copy fills both tables"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(copied), "a historic row gets a new key on every insert, so a retry that kept them would double the history"); + Assert.That(second.AlreadyPresent, Is.Zero, "the pending table was emptied too, so the re-read finds none of its rows already present"); + } + } + + [Test] + public async Task A_retry_of_a_category_whose_keys_the_copy_supplies_deletes_nothing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = ChangedAt }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = ChangedAt }); + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.MessageRedirects); + await CopyCategory(MigrationCategoryIds.RetryOperations); + + await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.MessageRedirects))); + var second = await CopyCategory(MigrationCategoryIds.MessageRedirects); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.AlreadyPresent, Is.EqualTo(2), "rows already in SQL are left alone, so the re-read finds both"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the reset deletes only for the category it names, so another category's copied rows stay"); + } + } + + [Test] + public async Task A_stale_reset_deletes_nothing() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var stale = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(stale)); + + Assert.ThrowsAsync(() => Target.ResetForRetry(ResetForRetry(stale))); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the delete shares the reset's transaction, so a refused reset leaves every copied row"); + } + + // The same reset --migration-retry makes. It keeps StartedAt and leaves SourceTotal alone, because the engine does not write it. + static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0, + SkipReasons = null, + SettledAt = null, + LastError = null + }; + + static RetryHistory TwoHistoricAndTwoPending() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-1", ChangedAt)); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-2", ChangedAt.AddMinutes(-10))); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-3", ChangedAt)); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-4", ChangedAt.AddMinutes(-10))); + return history; + } + + static MigrationCategory RetryOperations => MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime ChangedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs new file mode 100644 index 0000000000..87455299ac --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Extensibility; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class SubscriptionsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Two_message_type_versions_for_the_same_subscriber_merge_onto_one_row_and_the_merge_is_already_present() + { + await SeedSourceSubscriptions( + new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), + new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); + + var result = await CopyCategory("Subscriptions"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a version merge loses no row the target lacks, and a skip would leave the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a merge is not a skip, so the category is Done"); + + var subscribers = await SubscriptionStorage.GetSubscriberAddressesForMessage( + [new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0))], new ContextBag(), TestContext.CurrentContext.CancellationToken); + + Assert.That(subscribers.Count(), Is.EqualTo(1)); + Assert.That(subscribers.Select(subscriber => (subscriber.TransportAddress, subscriber.Endpoint)), Is.EqualTo(new[] { ("Billing@BILLING01", "Billing") }), + "the copied row holds the subscriber's address and endpoint as the source has them"); + } + + [Test] + public async Task A_legacy_subscriber_with_no_endpoint_gets_the_endpoint_the_product_stores_for_one() + { + var copiedType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var storedType = new MessageType("MyMessages.OrderShipped", new Version(1, 0, 0)); + var token = TestContext.CurrentContext.CancellationToken; + + await SeedSourceSubscriptions(new SubscriptionRow(copiedType.TypeName, copiedType.Version, "Billing@BILLING01", Endpoint: null)); + + var result = await CopyCategory("Subscriptions"); + + // Another type, so the product's upsert cannot overwrite the copied row it is compared with. + await SubscriptionStorage.Subscribe(new Subscriber("Billing@BILLING01", null), storedType, new ContextBag(), token); + + var copied = await SubscriptionStorage.GetSubscriberAddressesForMessage([copiedType], new ContextBag(), token); + var stored = await SubscriptionStorage.GetSubscriberAddressesForMessage([storedType], new ContextBag(), token); + + Assert.That(result.Copied, Is.EqualTo(1), "Endpoint is NOT NULL on the target, so a row built without a fallback fails the batch and leaves the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(copied.Single().TransportAddress, Is.EqualTo("Billing@BILLING01")); + Assert.That(copied.Single().Endpoint, Is.EqualTo(stored.Single().Endpoint), + "the copy must store the endpoint SubscriptionStorage itself stores for the same legacy subscriber"); + Assert.That(copied.Single().Endpoint, Is.EqualTo("Billing"), "the endpoint is the part of the address before the '@'"); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs index fd07ff535b..a30e2caa57 100644 --- a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs +++ b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs @@ -67,6 +67,8 @@ public Task Write(MigrationCategory category, MigrationBat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; @@ -115,6 +117,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; diff --git a/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs new file mode 100644 index 0000000000..a280f2d58a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class TrialEndDateMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory TrialEndDateCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.TrialEndDate)!; + + [Test] + public async Task Copies_the_trial_end_date_as_one_row_in_settings() + { + var endDate = new DateOnly(2027, 3, 15); + await SeedSourceTrialEndDate(endDate); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.Null, "the target holds no trial end date before the copy, so the date read afterwards came from the copy"); + + var result = await CopyCategory("TrialEndDate"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.EqualTo(endDate), "the product's own trial licence store must read back the seeded date"); + Assert.That(await Target.Count(TrialEndDateCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs new file mode 100644 index 0000000000..6802772bce --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class When_the_heartbeat_sync_runs_after_a_migration : MigrationAcceptanceTest +{ + [Test] + public async Task Migrated_endpoint_settings_survive_the_sync_under_the_name_their_known_endpoint_uses() + { + var time = new FakeTimeProvider(); + + var billing = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = "BILLING01" }, Monitored = true }; + + await SeedSource(($"KnownEndpoints/{billing.EndpointDetails.GetDeterministicId()}", billing)); + await SeedSourceKnownEndpoints("SmokeSales"); + await SeedSourceEndpointSettings(("Billing", true), ("smokesales", true), ("SmokeSales", false)); + + await RunRequiredCopy(customize: services => + { + services.AddSingleton(time); + CountSyncPasses(services); + }); + + time.Advance(TimeSpan.FromSeconds(21)); + await WaitForSyncPass(time); + + var settings = await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(settings.Select(setting => setting.Name), Does.Contain(string.Empty), + "The heartbeat sync did not create the default row, which it does after its delete loop. The pass did not finish, so the other checks prove nothing."); + Assert.That(settings.Select(setting => setting.Name), Does.Contain("Billing"), + "The heartbeat sync deleted the migrated Billing setting. It only does that when no Billing known endpoint was copied."); + Assert.That(settings.Where(setting => setting.Name == "SmokeSales").Select(setting => setting.TrackInstances), Is.EqualTo(new[] { false }), + "SmokeSales must keep the operator's Do Not Track. On a target whose name column ignores case, the two settings become one row. If the copy keeps 'smokesales', the sync deletes it and SmokeSales gets the default."); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs new file mode 100644 index 0000000000..0e805ce94a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure; + +[TestFixture] +[NonParallelizable] +class When_the_offline_migration_logs : MigrationAcceptanceTest +{ + [Test] + public void It_logs_only_through_the_providers_the_instance_is_configured_with() + { + string[] registered = null; + + Assert.ThrowsAsync(async () => await MigrateCommand.Run(Settings, builder => + { + registered = ProviderTypes(builder.Services); + throw new ProvidersCaptured(); + })); + + var configured = ProviderTypes(new ServiceCollection().AddLogging(logging => logging.ConfigureLogging(Settings.LoggingSettings.LogLevel))); + + Assert.That(registered, Is.EquivalentTo(configured), + "an extra default provider prints every line twice and prints EF Core's SQL, which the configured providers filter out"); + } + + static string[] ProviderTypes(IServiceCollection services) => + [.. services + .Where(service => service.ServiceType == typeof(ILoggerProvider)) + .Select(service => (service.ImplementationType ?? service.ImplementationInstance?.GetType() ?? service.ImplementationFactory?.Method.ReturnType)?.FullName ?? "unknown")]; + + sealed class ProvidersCaptured : Exception; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs index e0b683b1c8..066d6e8a2c 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -30,7 +30,12 @@ sealed class EFCoreMigrationTarget( new LicensingEndpointsWriter(migrationDialect), new LicensingThroughputWriter(migrationDialect), new LicensingReportMasksWriter(migrationDialect), - new LicensedEndpointDetailsWriter(migrationDialect) + new LicensedEndpointDetailsWriter(migrationDialect), + new MessageRedirectsWriter(migrationDialect), + new SubscriptionsWriter(migrationDialect), + new NotificationSettingsWriter(migrationDialect), + new TrialEndDateWriter(migrationDialect), + new RetryOperationsWriter(migrationDialect) }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => @@ -113,6 +118,36 @@ internal static int AlreadyPresentIn(MigrationCategory category, MigrationBatch public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => ExecuteWithDbContext((dbContext, token) => WriterFor(category).Count(dbContext, token), cancellationToken); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, resetToken) => + { + var storeKeyed = writers.GetValueOrDefault(reset.CategoryId) as IStoreKeyedCategoryWriter; + + var (deleted, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + var deletedRows = storeKeyed is null ? 0 : await storeKeyed.DeleteCopiedRows(dbContext, token); + var stored = await dbContext.UpsertCheckpoint(reset, token); + await transaction.CommitAsync(token); + + return (deletedRows, stored); + }, resetToken); + + if (storeKeyed is null) + { + logger.LogInformation("Reset category {CategoryId} for a retry and kept the rows it copied", reset.CategoryId); + } + else + { + logger.LogInformation("Reset category {CategoryId} for a retry and deleted the {DeletedRowCount} rows it copied, because a re-read would add them again under new keys", reset.CategoryId, deleted); + } + + return saved; + }, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => writers.Keys; public IReadOnlyDictionary DocumentTypes => writers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs index 1e81a7fff2..a5f241096e 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs @@ -193,7 +193,7 @@ static DateTime ReadResolvedDate(MigrationRow source, string key) => // Npgsql only writes UTC to a timestamp with time zone column, and a value whose kind is Unspecified is already UTC, so it must not be shifted. // Kept apart from the RavenDB source's copy because this project does not reference it. - static DateTime AsUtc(DateTime value) => + internal static DateTime AsUtc(DateTime value) => value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); // Uses the same length limit that ingestion applies to the MessageType column. diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs new file mode 100644 index 0000000000..726c80ad1e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; + +/// +/// A category writer whose rows get their keys from the database, so a re-read would add every row again. +/// The target deletes those rows when it resets the category for a retry. +/// The delete empties whole tables, so it is only safe for a category copied while ServiceControl is stopped. +/// +interface IStoreKeyedCategoryWriter +{ + /// + /// Deletes every row in this category's tables and returns how many it deleted. + /// Runs in the same transaction as the checkpoint reset. + /// + Task DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs new file mode 100644 index 0000000000..b0db77217d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.MessageRedirects; + +/// +/// Turns a redirect from the source into the row the product's own save would write. +/// +static class MessageRedirectRowMapper +{ + public static MessageRedirectEntity Map(MessageRedirect redirect) => new() + { + FromPhysicalAddress = redirect.FromPhysicalAddress, + ToPhysicalAddress = redirect.ToPhysicalAddress, + LastModified = redirect.LastModified + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs new file mode 100644 index 0000000000..400af82e6c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Recoverability; + +/// +/// Turns a retry operation from the source into the row the target stores. +/// Times are stored as UTC, and a time with no kind is taken as UTC already. +/// +static class RetryOperationRowMapper +{ + public static HistoricRetryOperationEntity Map(HistoricRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Originator = operation.Originator, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; + + public static UnacknowledgedRetryOperationEntity Map(UnacknowledgedRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Last = operation.Last, + Originator = operation.Originator, + Classifier = operation.Classifier, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs new file mode 100644 index 0000000000..734283aae7 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +/// +/// Turns a subscription from the source into the row the target stores. +/// The version is dropped, because the row has no column for it. +/// +static class SubscriptionRowMapper +{ + public static SubscriptionEntity Map(SubscriptionRow row) => new() + { + MessageType = row.MessageType, + TransportAddress = row.TransportAddress, + // The value SubscriptionStorage.Subscribe stores for a subscriber with no endpoint. + Endpoint = row.Endpoint ?? row.TransportAddress.Split('@').First() + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs index 72db9db495..b13a6e8e43 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -7,10 +7,9 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.DataMigration; /// -/// Writes every per-endpoint setting as stored. A setting for an endpoint the target does not know is copied too, -/// and the heartbeat settings sync removes it by its own rule once the instance opens. On a target whose name -/// column ignores case, two names that differ only in case are one row, and the writer reports each setting the -/// insert drops as a merge. +/// Copies every per-endpoint setting as stored. +/// Where the name column ignores case, names that differ only in case become one row, and the dropped ones are +/// reported as merges. The row kept is the one whose name matches a known endpoint exactly. /// sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter { @@ -24,8 +23,7 @@ public override Task Count(ServiceControlDbContext dbContext, Cancellation protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) { var insert = MigrationInsert.For(dbContext); - List rows = []; - List rowSourceIds = []; + List<(string SourceId, EndpointSettingsEntity Row)> rows = []; List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; foreach (var (row, settings) in documents) @@ -38,47 +36,32 @@ protected override async Task PrepareDocuments(ServiceControlDbCo continue; } - rows.Add(entity); - rowSourceIds.Add(row.SourceId); + rows.Add((row.SourceId, entity)); } - // The source yields document-id order, and the statement keeps the first of any keys the database treats as one. + rows = await KnownEndpointNamesFirst(dbContext, rows, cancellationToken); + + // The insert keeps the first of any names the database treats as equal. return new PreparedBatch( - async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, - await MergesIn(dbContext, rows, rowSourceIds, cancellationToken)); + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); } - // Mirrors the insert's de-duplication, so an operator can see which settings a case-insensitive name column dropped. - async Task> MergesIn(ServiceControlDbContext dbContext, List rows, List rowSourceIds, CancellationToken cancellationToken) + async Task> KnownEndpointNamesFirst(ServiceControlDbContext dbContext, List<(string SourceId, EndpointSettingsEntity Row)> rows, CancellationToken cancellationToken) { - var sameName = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); - string[] rowNames = [.. rows.Select(row => row.Name)]; - - // The database compares under the column's collation, so this also returns a stored name that differs only in case. - var keptNames = await dbContext.EndpointSettings.AsNoTracking() - .Select(stored => stored.Name) - .Where(name => rowNames.Contains(name)) - .ToListAsync(cancellationToken); - - var merges = new List<(string SourceId, string Detail)>(); + var nameComparer = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); + string[] mergingNames = [.. rows.GroupBy(row => row.Row.Name, nameComparer).Where(group => group.Count() > 1).SelectMany(group => group.Select(row => row.Row.Name))]; - for (var index = 0; index < rows.Count; index++) + if (mergingNames.Length == 0) { - var name = rows[index].Name; - var keptName = keptNames.FirstOrDefault(kept => sameName.Equals(kept, name)); - - if (keptName is null) - { - keptNames.Add(name); - } - else if (!string.Equals(keptName, name, StringComparison.Ordinal)) - { - merges.Add((rowSourceIds[index], $"the name column treats '{name}' and '{keptName}' as one endpoint, so it kept '{keptName}' and dropped the settings of '{name}'")); - } + return rows; } - return merges; + var knownNames = (await dbContext.KnownEndpoints.Where(endpoint => mergingNames.Contains(endpoint.Name)).Select(endpoint => endpoint.Name).ToListAsync(cancellationToken)).ToHashSet(StringComparer.Ordinal); + + // OrderBy is stable, so only the known names move to the front. + return [.. rows.OrderBy(row => knownNames.Contains(row.Row.Name) ? 0 : 1)]; } } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs new file mode 100644 index 0000000000..8116cac289 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Infrastructure; +using Microsoft.EntityFrameworkCore; + +/// +/// Finds the rows the insert will drop because another row, earlier in the batch or already in the table, has the same +/// key under the column's collation. A row the table already holds exactly is not a merge. +/// Returns each dropped row's source id with a line naming the key that was kept. +/// +static class KeyMerges +{ + public static async Task> In(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, CancellationToken cancellationToken = default) where TEntity : class + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; + var keyProperties = entityType.FindPrimaryKey()!.Properties; + var queryColumn = keyProperties.FirstOrDefault(property => property.ClrType == typeof(string)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no text key column, so no collation can join two of its keys."); + Func[] columnMatches = [.. keyProperties.Select(property => property.ClrType == typeof(string) ? TextMatch(dialect.KeyComparer(typeof(TEntity), property.Name)) : object.Equals)]; + + object?[] KeyOf(TEntity entity) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(entity))]; + bool SameKey(object?[] first, object?[] second) => columnMatches.Select((matches, column) => matches(first[column], second[column])).All(same => same); + + var stored = new List(); + string[] queryValues = [.. rows.Select(row => (string)queryColumn.GetGetter().GetClrValue(row.Row)!).Distinct(StringComparer.Ordinal)]; + + foreach (var chunk in queryValues.Chunk(dialect.RowsPerStatementFor(dbContext))) + { + stored.AddRange(await dbContext.Set().AsNoTracking().Where(entity => chunk.Contains(EF.Property(entity, queryColumn.Name))).ToListAsync(cancellationToken)); + } + + // A stored key wins over every row of the batch, and an earlier row over a later one, as in the insert. + var kept = stored.Select(entity => (Key: KeyOf(entity), Stored: true)).ToList(); + var merges = new List<(string SourceId, string Detail)>(); + + foreach (var (sourceId, row) in rows) + { + var key = KeyOf(row); + var match = kept.FindIndex(entry => SameKey(entry.Key, key)); + + if (match < 0) + { + kept.Add((key, false)); + } + else if (kept[match].Stored && kept[match].Key.SequenceEqual(key)) + { + // Claimed by this row, so a later row of the batch with the same key is a merge. + kept[match] = (key, false); + } + else + { + merges.Add((sourceId, $"{entityType.GetTableName()} treats {Describe(key)} and {Describe(kept[match].Key)} as one key, so it kept {Describe(kept[match].Key)} and dropped the row for {Describe(key)}")); + } + } + + return merges; + } + + static Func TextMatch(IEqualityComparer comparer) => (first, second) => comparer.Equals((string?)first, (string?)second); + + static string Describe(object?[] key) => string.Join(", ", key.Select(value => $"'{value}'")); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs index 9bcf8ec909..1e9e702dc9 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs @@ -1,9 +1,6 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; -using DbContexts; -using Entities; using Infrastructure; -using Microsoft.EntityFrameworkCore; using Particular.LicensingComponent.Contracts; using ServiceControl.Persistence.DataMigration; @@ -11,21 +8,7 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; /// Writes the licensed endpoint details file as one row in Settings. A row the target already holds is left as it is /// and counted as already present. /// -sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.LicensedEndpointDetails) { public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; - - const string SettingKey = SettingKeys.LicensedEndpointDetails; - - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); - - protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, LicensedEndpointDetails Document)> documents, CancellationToken cancellationToken = default) - { - List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; - - return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); - } - - public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => - dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs index 18823e068c..6abd48b3d3 100644 --- a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs @@ -1,30 +1,13 @@ namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; -using DbContexts; -using Entities; using Infrastructure; -using Microsoft.EntityFrameworkCore; using ServiceControl.Persistence.DataMigration; /// /// Writes the report masks as one row in Settings. A row the target already holds is left as it is and counted as /// already present. /// -sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter> +sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter>(migrationDialect, SettingKeys.ReportMasks) { public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; - - const string SettingKey = SettingKeys.ReportMasks; - - public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); - - protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, List Document)> documents, CancellationToken cancellationToken = default) - { - List rows = [.. documents.Select(document => SettingRowMapper.Map(SettingKey, document.Document))]; - - return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); - } - - public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => - dbContext.Settings.LongCountAsync(setting => setting.Key == SettingKey, cancellationToken); } diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs new file mode 100644 index 0000000000..9544efd9d9 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +/// +/// Copies the message redirects, one row per redirect. A redirect whose source address is already taken is reported as a merge. +/// A redirect that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class MessageRedirectsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.MessageRedirects.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, MessageRedirect Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var rows = new List<(string SourceId, MessageRedirectEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, redirect) in documents) + { + if (redirect.FromPhysicalAddress is null || redirect.ToPhysicalAddress is null) + { + var column = redirect.FromPhysicalAddress is null ? nameof(MessageRedirectEntity.FromPhysicalAddress) : nameof(MessageRedirectEntity.ToPhysicalAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"MessageRedirects.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = MessageRedirectRowMapper.Map(redirect); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs new file mode 100644 index 0000000000..85c0842e91 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Infrastructure; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the notification email settings as one Settings row. +/// +sealed class NotificationSettingsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.NotificationEmails) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs new file mode 100644 index 0000000000..e84dd7d149 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs @@ -0,0 +1,124 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +/// +/// Copies the retry history into two tables. A pending operation whose key is already taken is reported as a merge. +/// Historic operations get new keys, so a retry's reset empties both tables before the history is read again. +/// An operation that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class RetryOperationsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override async Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.LongCountAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.LongCountAsync(cancellationToken); + + // Whole tables, because the target is refused unless it held no ServiceControl data before the first copy. + public async Task DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.ExecuteDeleteAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.ExecuteDeleteAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, RetryOperationRow Document)> documents, CancellationToken cancellationToken = default) + { + var pendingInsert = MigrationInsert.For(dbContext); + var historicText = HistoricTextColumns(dbContext); + List historic = []; + List<(string SourceId, UnacknowledgedRetryOperationEntity Row)> unacknowledged = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, operation) in documents) + { + if (operation.Historic is { } historicOperation) + { + if (historicOperation.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it")); + } + else if (UnstorableHistoricValue(historicText, row, RetryOperationRowMapper.Map(historicOperation)) is { } unstorable) + { + skips.Add(unstorable); + } + else + { + historic.Add(historicOperation); + } + + continue; + } + + var pending = operation.Unacknowledged!; + + if (pending.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it")); + continue; + } + + var entity = RetryOperationRowMapper.Map(pending); + + if (UnstorableValueSkip(pendingInsert, migrationDialect, row, entity) is { } unstorablePending) + { + skips.Add(unstorablePending); + continue; + } + + unacknowledged.Add((row.SourceId, entity)); + } + + // Reversed, because the insert keeps the first of a repeated key and the latest retry must win. + unacknowledged.Reverse(); + + return new PreparedBatch( + async (context, token) => + { + var inserted = await migrationDialect.InsertMissing(context, [.. unacknowledged.Select(pending => pending.Row)], token); + + // Fresh entities per attempt, because a failed save can leave generated ids on the previous ones. + // Reversed, so on a tie in completion time the first in the document still wins, as in RavenDB. + context.HistoricRetryOperations.AddRange(Enumerable.Reverse(historic).Select(RetryOperationRowMapper.Map)); + await context.SaveChangesAsync(token); + + return historic.Count + inserted.Count; + }, + historic.Count + unacknowledged.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, unacknowledged, cancellationToken)); + } + + // Read from the model, because MigrationInsert.For refuses a table whose key the store generates. + static (IProperty Property, string TableAndColumn)[] HistoricTextColumns(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(HistoricRetryOperationEntity))!; + + return [.. entityType.GetProperties().Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{entityType.GetTableName()}.{property.GetColumnName()}"))]; + } + + (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableHistoricValue((IProperty Property, string TableAndColumn)[] text, MigrationRow row, HistoricRetryOperationEntity entity) + { + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(entity) is string { Length: var units } value && property.GetMaxLength() is { } maxLength && units > maxLength && migrationDialect.TextLength(value) is var length && length > maxLength) + { + return (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}, on document '{row.SourceId}'"); + } + } + + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(entity) is string value && migrationDialect.RejectedCharacterIn(value) is { } rejected) + { + return (row.SourceId, MigrationSkipReason.MalformedValue, $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains, on document '{row.SourceId}'"); + } + } + + return null; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs new file mode 100644 index 0000000000..8d285689bb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes a category that the target stores as one JSON row in Settings. +/// A row already under that key is left alone and counted as already present. +/// +abstract class SettingRowWriter(IMigrationSqlDialect migrationDialect, string settingKey) : MigrationCategoryWriter where TDocument : class +{ + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Settings.LongCountAsync(setting => setting.Key == settingKey, cancellationToken); + + /// + /// Builds the Settings row for one document, storing the whole document by default. + /// Override it when the document holds the value rather than being it. + /// + protected virtual SettingEntity RowFor(string key, TDocument document) => SettingRowMapper.Map(key, document); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => RowFor(settingKey, document.Document))]; + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs new file mode 100644 index 0000000000..a748509255 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Copies the subscriptions. The key leaves out the message type's version, so two versions of one type for one +/// subscriber become one row, and the second is reported as a merge. +/// A subscription that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// +sealed class SubscriptionsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor(dbContext); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Subscriptions.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, SubscriptionRow Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert.For(dbContext); + var rows = new List<(string SourceId, SubscriptionEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, subscription) in documents) + { + if (subscription.MessageType is null || subscription.TransportAddress is null) + { + var column = subscription.MessageType is null ? nameof(SubscriptionEntity.MessageType) : nameof(SubscriptionEntity.TransportAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"Subscriptions.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = SubscriptionRowMapper.Map(subscription); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs new file mode 100644 index 0000000000..5e0ace65ad --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Entities; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the trial end date as one Settings row, storing only the date, as the product does. +/// +sealed class TrialEndDateWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter(migrationDialect, SettingKeys.TrialEndDate) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override SettingEntity RowFor(string key, TrialMetadata document) => SettingRowMapper.Map(key, document.TrialEndDate); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs index cfd20d1582..18266881e0 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -31,7 +31,12 @@ sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMig new LicensingEndpointsReader(lifecycle), new LicensingThroughputReader(lifecycle), new LicensingReportMasksReader(lifecycle), - new LicensedEndpointDetailsReader(lifecycle) + new LicensedEndpointDetailsReader(lifecycle), + new MessageRedirectsReader(lifecycle), + new SubscriptionsReader(lifecycle), + new NotificationSettingsReader(lifecycle), + new TrialEndDateReader(lifecycle), + new RetryOperationsReader(lifecycle) }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs index 477edb9b3b..f5b8d3f66e 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs @@ -2,11 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; -using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Runtime.CompilerServices; -using System.Threading; using Particular.LicensingComponent.Contracts; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB.Throughput; @@ -16,32 +12,11 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; /// whole document, or no rows when the document does not exist. The batch's cursor is the document's id, and a read /// resumed after that id gives nothing. /// -sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.LicencedEndpointDetailsDocumentId) { - const string DocumentId = LicensingDataStore.LicencedEndpointDetailsDocumentId; - public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; - public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - ReadLicensedEndpointDetails(resumeAfter, cancellationToken); - - async IAsyncEnumerable ReadLicensedEndpointDetails(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (resumeAfter == DocumentId) - { - yield break; - } - - if (resumeAfter is not null) - { - throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); - } - - using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); - var document = await session.LoadAsync(DocumentId, cancellationToken); - - yield return new MigrationBatch( - document is null ? [] : [new MigrationRow(DocumentId, document, ReadOnlyDictionary.Empty)], - DocumentId); - } + protected override IEnumerable<(string SourceId, LicensedEndpointDetails Document)> RowsFrom(LicensedEndpointDetails document) => + [(DocumentId, document)]; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs index 6b74a4e20a..47597f08cd 100644 --- a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs @@ -2,11 +2,7 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; -using System; using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Runtime.CompilerServices; -using System.Threading; using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.RavenDB.Throughput; using ServiceControl.Persistence.RavenDB.Throughput.Models; @@ -16,32 +12,11 @@ namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; /// holding the list, or no rows when the document does not exist, and a list stored as null reads as an empty one. /// The batch's cursor is the document's id, and a read resumed after that id gives nothing. /// -sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader>(lifecycle) +sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader>(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.ReportMasksDocumentId) { - const string DocumentId = LicensingDataStore.ReportMasksDocumentId; - public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; - public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => - ReadReportMasks(resumeAfter, cancellationToken); - - async IAsyncEnumerable ReadReportMasks(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) - { - if (resumeAfter == DocumentId) - { - yield break; - } - - if (resumeAfter is not null) - { - throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); - } - - using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); - var document = await session.LoadAsync(DocumentId, cancellationToken); - - yield return new MigrationBatch( - document is null ? [] : [new MigrationRow(DocumentId, document.MaskedStrings ?? [], ReadOnlyDictionary.Empty)], - DocumentId); - } + protected override IEnumerable<(string SourceId, List Document)> RowsFrom(ReportConfigurationDocument document) => + [(DocumentId, document.MaskedStrings ?? [])]; } diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs new file mode 100644 index 0000000000..12795ddbba --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs @@ -0,0 +1,22 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.RavenDB.MessageRedirects; + +/// +/// Reads the message redirects document as one row per redirect, in the document's order. +/// +sealed class MessageRedirectsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, MessageRedirectsDataStore.CollectionId) +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + // The writer skips a redirect with no source address, and that skip still needs an id to name. + protected override IEnumerable<(string SourceId, MessageRedirect Document)> RowsFrom(MessageRedirectsCollection document) => + document.ToRedirects().Select((redirect, index) => (redirect.FromPhysicalAddress ?? $"{DocumentId}#{index}", redirect)); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs new file mode 100644 index 0000000000..9a0ae42aea --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Editing; + +/// +/// Reads the notification settings document, whose email settings become one Settings row. +/// +sealed class NotificationSettingsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, NotificationsDataStore.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; + + protected override IEnumerable<(string SourceId, EmailNotifications Document)> RowsFrom(NotificationsSettingsDocument document) => + document.Email is { } email ? [(DocumentId, email)] : []; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs new file mode 100644 index 0000000000..72280d9697 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs @@ -0,0 +1,24 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; +using ServiceControl.Recoverability; + +/// +/// Reads the retry history document as rows, historic operations first, then unacknowledged ones. +/// +sealed class RetryOperationsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, RetryHistoryDataStore.DocumentId) +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + protected override IEnumerable<(string SourceId, RetryOperationRow Document)> RowsFrom(RetryHistory document) => + [ + .. document.HistoricOperations.Select(operation => ($"historic|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(operation, null))), + .. document.UnacknowledgedOperations.Select(operation => ($"unacknowledged|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(null, operation))) + ]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs new file mode 100644 index 0000000000..7f61ffa043 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs @@ -0,0 +1,27 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the subscriptions document as one row per subscriber of each message type and version. +/// +sealed class SubscriptionsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, Subscriptions.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + protected override IEnumerable<(string SourceId, SubscriptionRow Document)> RowsFrom(Subscriptions document) + { + foreach (var subscription in document.All.Values) + { + foreach (var subscriber in subscription.Subscribers) + { + yield return ($"{subscription.Id}|{subscriber.TransportAddress}", new SubscriptionRow(subscription.MessageType.TypeName, subscription.MessageType.Version, subscriber.TransportAddress, subscriber.Endpoint)); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs new file mode 100644 index 0000000000..3724963f67 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the trial metadata document, whose end date becomes one Settings row. +/// +sealed class TrialEndDateReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader(lifecycle, lifecycle.Settings.DatabaseName, TrialLicenseDataProvider.TrialMetadataId) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override IEnumerable<(string SourceId, TrialMetadata Document)> RowsFrom(TrialMetadata document) => [(DocumentId, document)]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs new file mode 100644 index 0000000000..aac5d66c70 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs @@ -0,0 +1,54 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads a category that RavenDB keeps in one document with a fixed id. +/// The document is one batch, and its id is the cursor. A missing document gives a batch with no rows. +/// +abstract class SingleDocumentReader(RavenReadOnlySourceLifecycle lifecycle, string databaseName, string documentId) : MigrationCategoryReader(lifecycle) + where TStored : class + where TDocument : class +{ + /// + /// The id of the one document this category reads, and the cursor of its only batch. + /// + protected string DocumentId { get; } = documentId; + + public sealed override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadDocument(resumeAfter, cancellationToken); + + /// + /// Turns the loaded document into rows, in the order the writer gets them. Called only when the document exists. + /// Leave out a missing value rather than return a null document. + /// + protected abstract IEnumerable<(string SourceId, TDocument Document)> RowsFrom(TStored document); + + async IAsyncEnumerable ReadDocument(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (resumeAfter == DocumentId) + { + yield break; + } + + if (resumeAfter is not null) + { + throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); + } + + using var session = Lifecycle.OpenSession(databaseName); + var document = await session.LoadAsync(DocumentId, cancellationToken); + + yield return new MigrationBatch( + document is null ? [] : [.. RowsFrom(document).Select(row => new MigrationRow(row.SourceId, row.Document, ReadOnlyDictionary.Empty))], + DocumentId); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs index c801d834fa..1aefbe8e67 100644 --- a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Persistence.RavenDB.Editing class NotificationsDataStore(IRavenSessionProvider sessionProvider) : INotificationsDataStore { - const string SingleDocumentId = "NotificationsSettings/All"; + internal const string SingleDocumentId = "NotificationsSettings/All"; public async Task LoadSettings(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs index 93f55c7ef1..2df9d1af09 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs @@ -9,7 +9,7 @@ class RetryHistoryDataStore(IRavenSessionProvider sessionProvider) : IRetryHistoryDataStore { - const string DocumentId = "RetryOperations/History"; + internal const string DocumentId = "RetryOperations/History"; public async Task> GetRetryHistory(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs index f0feaa2aaf..a3d51910a1 100644 --- a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs @@ -23,6 +23,6 @@ public async Task StoreTrialEndDate(DateOnly trialEndDate, CancellationToken can await session.SaveChangesAsync(cancellationToken); } - static string TrialMetadataId = "metadata/trialinformation"; + internal const string TrialMetadataId = "metadata/trialinformation"; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs index a2a7b1c7e3..fa228033bb 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs @@ -59,11 +59,9 @@ public async Task An_absent_document_gives_one_empty_batch_that_still_records_th } [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] - [TestCase(MigrationCategoryIds.LicensedEndpointDetails, "LicensedEndpointDetails")] public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string categoryId, string documentId) { await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); - await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); var batches = await Read(categoryId, documentId); @@ -71,11 +69,9 @@ public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string } [TestCase(MigrationCategoryIds.LicensingReportMasks)] - [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] public async Task A_read_resumed_after_a_cursor_the_reader_never_issued_is_refused(string categoryId) { await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); - await Seed("LicensedEndpointDetails", new LicensedEndpointDetails { LicenseId = "abc-123" }); var exception = Assert.ThrowsAsync(() => Read(categoryId, "Sales.Orders/Broker")); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs new file mode 100644 index 0000000000..ebd380e1bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using RavenMessageRedirectsDataStore = Persistence.RavenDB.MessageRedirects.MessageRedirectsDataStore; + +class MessageRedirectsSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects); + + [Test] + public async Task A_document_whose_last_redirect_was_removed_gives_one_empty_batch() + { + var redirect = new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }; + await MessageRedirectsDataStore.AddRedirect(redirect); + await MessageRedirectsDataStore.RemoveRedirect(redirect); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { RavenMessageRedirectsDataStore.CollectionId })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs index 463c008a2f..67a81ea63f 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -4,11 +4,16 @@ namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.Unicast.Subscriptions; using NUnit.Framework; using Particular.LicensingComponent.Contracts; using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Notifications; using ServiceControl.Operations; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.RavenDB.Throughput.Models; using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; @@ -133,13 +138,22 @@ public async Task Reading_every_category_creates_no_index() [MigrationCategoryIds.LicensingEndpoints] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingThroughput] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), [MigrationCategoryIds.LicensingReportMasks] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso"] }), - [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }) + [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }), + [MigrationCategoryIds.MessageRedirects] = tests => tests.MessageRedirectsDataStore.AddRedirect(new MessageRedirect { FromPhysicalAddress = "Sales@OLDHOST", ToPhysicalAddress = "Sales@NEWHOST", LastModified = DateTime.UtcNow }), + [MigrationCategoryIds.Subscriptions] = tests => tests.SeedSubscription(), + [MigrationCategoryIds.NotificationSettings] = tests => tests.NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + [MigrationCategoryIds.TrialEndDate] = tests => tests.ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + [MigrationCategoryIds.RetryOperations] = tests => tests.RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10) }; string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; + // Both databases, because the licensing readers read the throughput database. async Task IndexNames() => - await DocumentStore.Maintenance.SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); + [.. await IndexNamesIn(DatabaseName), .. await IndexNamesIn(ThroughputDatabaseName)]; + + Task IndexNamesIn(string database) => + DocumentStore.Maintenance.ForDatabase(database).SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); const int IndexNamePageSize = 1024; @@ -152,6 +166,21 @@ async Task SeedThroughputDocument(string id, object document) await session.SaveChangesAsync(); } + async Task SeedSubscription() + { + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)), + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using var session = await SessionProvider.OpenSession(); + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + async Task SeedEndpointSettings(int count) { for (var index = 0; index < count; index++) diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs new file mode 100644 index 0000000000..a45824203a --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Unicast.Subscriptions; +using NUnit.Framework; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +class SubscriptionsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Flattens_the_dictionary_of_subscriptions_into_one_row_per_subscriber() + { + var eventV1 = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var eventV2 = new MessageType("MyMessages.OrderPlaced", new Version(2, 0, 0)); + + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = eventV1, + Subscribers = [ + new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }, + new SubscriptionClient { Endpoint = "Shipping", TransportAddress = "Shipping@SHIP01" } + ] + }; + document.All["v2"] = new Subscription + { + Id = "v2", + MessageType = eventV2, + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + + var batches = await Read(); + + Assert.That(batches.Last().Cursor, Is.EqualTo(Subscriptions.SingleDocumentId)); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => (row.SourceId, (SubscriptionRow)row.Document)), Is.EquivalentTo(new[] + { + ("v1|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("v1|Shipping@SHIP01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping")), + ("v2|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")) + }), "each row carries every value its subscriber holds, under an id naming the subscription and the address"); + } + + async Task> Read() + { + await using var source = await OpenMigrationSource(); + return await CollectBatches(source, MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions)); + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs index 8a90e4965c..9a081a0b2b 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -30,7 +30,12 @@ public async Task A_case_merge_inside_one_batch_logs_both_names_and_the_one_kept await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); - Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + using (Assert.EnterMultipleScope()) + { + Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/2 ").And.Contains("kept 'Sales'"), "the operator finds the dropped row by this source id"); + Assert.That(targetWarnings.Messages, Has.None.StartsWith("Merged EndpointSettings/1 "), "the kept row was not dropped"); + } } [Test] @@ -44,6 +49,26 @@ public async Task A_case_merge_with_a_row_an_earlier_batch_wrote_is_logged_and_a Assert.That(targetWarnings.Messages, Has.Exactly(1).Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); } + [Test] + public async Task A_case_merge_keeps_the_setting_whose_name_a_known_endpoint_uses_even_when_it_comes_second() + { + await OpenACaseInsensitiveTarget("Sales"); + + await Write(("EndpointSettings/1", "sales", true), ("EndpointSettings/2", "Sales", false)); + + List stored; + using (var scope = ServiceProvider.CreateScope()) + { + stored = await scope.ServiceProvider.GetRequiredService().EndpointSettings.AsNoTracking().ToListAsync(); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Select(setting => (setting.Name, setting.TrackInstances)), Is.EqualTo(new[] { ("Sales", false) }), "the heartbeat sync deletes a setting no known endpoint has the exact name of, so the orphan must be the one dropped"); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/1 ").And.Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + } + async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) { using (var scope = ServiceProvider.CreateScope()) @@ -70,11 +95,13 @@ async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) await ServiceProvider.GetRequiredService().Open(); } - async Task Write(params (string SourceId, string Name)[] rows) + Task Write(params (string SourceId, string Name)[] rows) => Write([.. rows.Select(row => (row.SourceId, row.Name, true))]); + + async Task Write(params (string SourceId, string Name, bool TrackInstances)[] rows) { var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); var batch = new MigrationBatch( - [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = true }, new Dictionary()))], + [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = row.TrackInstances }, new Dictionary()))], rows[^1].SourceId); var checkpoint = await ServiceProvider.GetRequiredService().Read(category.Id) ?? new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs deleted file mode 100644 index 1c6cba5d3d..0000000000 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensedEndpointDetailsMigrationTargetTests.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ServiceControl.Persistence.Tests; - -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using NUnit.Framework; -using Particular.LicensingComponent.Contracts; -using ServiceControl.Persistence.DataMigration; - -class LicensedEndpointDetailsMigrationTargetTests : PersistenceTestBase -{ - [SetUp] - public Task OpenTarget() => Target.Open(); - - IMigrationTarget Target => ServiceProvider.GetRequiredService(); - - static readonly MigrationCategory LicensedEndpointDetailsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensedEndpointDetails); - - [Test] - public async Task Count_is_the_licensed_endpoint_details_row_alone() - { - await LicensingDataStore.SaveReportMasks(["contoso"]); - Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.Zero, "the report masks row is not a licensed endpoint details row"); - - await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); - - Assert.That(await Target.Count(LicensedEndpointDetailsCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the report masks row and every other row in the table"); - } -} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs deleted file mode 100644 index 45faa38e2e..0000000000 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingReportMasksMigrationTargetTests.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace ServiceControl.Persistence.Tests; - -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using NUnit.Framework; -using Particular.LicensingComponent.Contracts; -using ServiceControl.Persistence.DataMigration; - -class LicensingReportMasksMigrationTargetTests : PersistenceTestBase -{ - [SetUp] - public Task OpenTarget() => Target.Open(); - - IMigrationTarget Target => ServiceProvider.GetRequiredService(); - - static readonly MigrationCategory LicensingReportMasksCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingReportMasks); - - [Test] - public async Task Count_is_the_report_masks_row_alone() - { - await LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); - Assert.That(await Target.Count(LicensingReportMasksCategory), Is.Zero, "the licensed endpoint details row is not a report masks row"); - - await LicensingDataStore.SaveReportMasks(["contoso"]); - - Assert.That(await Target.Count(LicensingReportMasksCategory), Is.EqualTo(1), "counted as the whole Settings table, this would also count the licensed endpoint details row and every other row in the table"); - } -} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs new file mode 100644 index 0000000000..a6e99bb5bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var entity = MessageRedirectRowMapper.Map(new MessageRedirect + { + FromPhysicalAddress = "Sales@OLDHOST", + ToPhysicalAddress = "Sales@NEWHOST", + LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) + }); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs new file mode 100644 index 0000000000..467cedecc8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory MessageRedirectsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + + [Test] + public async Task A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var batch = BatchOf( + ("Redirect/1", Redirect(new string('f', 450), new string('t', 450))), + ("Redirect/2", Redirect(new string('f', 451), "Sales@NEWHOST")), + ("Redirect/3", Redirect("Billing@OLDHOST", new string('t', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new MessageRedirectsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(MessageRedirectsCategory, batch, CheckpointAfter("Redirect/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "Redirect/2", "Redirect/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await MessageRedirectsDataStore.GetRedirects()).Select(redirect => redirect.FromPhysicalAddress), Is.EqualTo(new[] { new string('f', 450) })); + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details[0], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/2'")); + Assert.That(details[1], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/3'")); + } + } + + [Test] + public async Task A_redirect_whose_address_the_key_treats_as_an_earlier_ones_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("Redirect/1", Redirect("Sales@HOST", "Sales@NEWHOST")), + ("Redirect/2", Redirect("Sales@HOST", "Sales@OTHERHOST")), + ("Redirect/3", Redirect("sales@host", "Sales@NEWHOST"))); + var keyIgnoresCase = dialect.KeyComparer(typeof(MessageRedirectEntity), nameof(MessageRedirectEntity.FromPhysicalAddress)).Equals("Sales@HOST", "sales@host"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new MessageRedirectsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresCase ? new[] { "Redirect/2", "Redirect/3" } : new[] { "Redirect/2" }), + "a second copy of an address is dropped on every database, and a case-only variant only where the key column ignores case"); + Assert.That(merges[0].Detail, Does.Contain("kept 'Sales@HOST'"), "the target logs this line, so it has to name the address it kept"); + } + } + + static MessageRedirect Redirect(string from, string to) => + new() { FromPhysicalAddress = from, ToPhysicalAddress = to, LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) }; + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs index 0cf8345618..3237b5700e 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -1,13 +1,17 @@ namespace ServiceControl.Persistence.Tests; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; @@ -47,8 +51,30 @@ public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_ Assert.That(model.FindEntityType(typeof(SettingEntity))!.FindProperty(nameof(SettingEntity.Value))!.GetMaxLength(), Is.Null, "NoValueCanBeTooLong exempts the settings categories because of this; give them a value-too-long test if the column gains a limit"); } + [Test] + public async Task Every_writer_whose_rows_get_their_key_from_the_store_deletes_them_for_a_retry_and_no_other_writer_does() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)); + + Assert.That(writers, Is.EquivalentTo(EntitiesEachWriterWrites.Keys), "list the entities a new writer writes here, so this test can tell whether a retry must delete its rows"); + + using (Assert.EnterMultipleScope()) + { + foreach (var (writer, entities) in EntitiesEachWriterWrites) + { + var storeKeyed = entities.Any(entity => model.FindEntityType(entity)!.FindPrimaryKey()!.Properties.Any(property => property.ValueGenerated != ValueGenerated.Never)); + + Assert.That(typeof(IStoreKeyedCategoryWriter).IsAssignableFrom(writer), Is.EqualTo(storeKeyed), + $"{writer.Name}: a retry keeps the rows of a writer that is not an IStoreKeyedCategoryWriter, so a re-read adds rows with store keys a second time, and a writer whose keys the copy supplies must not delete"); + } + } + } + // Each is one Settings row under a constant key whose value column has no maximum length, so no source value can be too long for it. - static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails]; + static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. static readonly Dictionary ValueTooLongTests = new() @@ -57,6 +83,26 @@ public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_ [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), [MigrationCategoryIds.LicensingEndpoints] = nameof(LicensingEndpointsMigrationTargetTests.A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), - [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.MessageRedirects] = nameof(MessageRedirectsMigrationTargetTests.A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.Subscriptions] = nameof(SubscriptionsMigrationTargetTests.A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy), + [MigrationCategoryIds.RetryOperations] = nameof(RetryOperationsMigrationTargetTests.A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + }; + + // The writers do not say which entities they write, so this list does. + static readonly Dictionary EntitiesEachWriterWrites = new() + { + [typeof(KnownEndpointsWriter)] = [typeof(KnownEndpointEntity)], + [typeof(EndpointSettingsWriter)] = [typeof(EndpointSettingsEntity)], + [typeof(FailedMessagesWriter)] = [typeof(FailedMessageEntity), typeof(FailedMessageGroupEntity)], + [typeof(LicensingEndpointsWriter)] = [typeof(LicensingEndpointEntity)], + [typeof(LicensingThroughputWriter)] = [typeof(LicensingEndpointThroughputEntity)], + [typeof(LicensingReportMasksWriter)] = [typeof(SettingEntity)], + [typeof(LicensedEndpointDetailsWriter)] = [typeof(SettingEntity)], + [typeof(MessageRedirectsWriter)] = [typeof(MessageRedirectEntity)], + [typeof(SubscriptionsWriter)] = [typeof(SubscriptionEntity)], + [typeof(NotificationSettingsWriter)] = [typeof(SettingEntity)], + [typeof(TrialEndDateWriter)] = [typeof(SettingEntity)], + [typeof(RetryOperationsWriter)] = [typeof(HistoricRetryOperationEntity), typeof(UnacknowledgedRetryOperationEntity)] }; } diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs index e04ebaee93..8e43f3c55f 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs @@ -23,6 +23,17 @@ public static async Task> DetailsFrom(IServiceProvider ser return [.. prepared.Skips.Select(skip => skip.Detail)]; } + /// + /// Prepares a batch without writing it, and returns the rows the writer reports as merges with their log detail. + /// + public static async Task> MergesFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService(), batch); + + return prepared.Merges; + } + /// /// The table and column a property is stored in, as this provider names them. /// diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs new file mode 100644 index 0000000000..6763d6d562 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Recoverability; + +class RetryOperationRowMapperTests : PersistenceTestBase +{ + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task Sets_every_mapped_property_of_a_historic_and_an_unacknowledged_operation() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + using (Assert.EnterMultipleScope()) + { + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + })); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs new file mode 100644 index 0000000000..58adff0d22 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs @@ -0,0 +1,244 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Recoverability; + +class RetryOperationsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory RetryOperationsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var dialect = ServiceProvider.GetRequiredService(); + var fourHundredCodePoints = char.ConvertFromUtf32(0x1F600) + new string('r', 399); + var countsCodePoints = dialect.TextLength(fourHundredCodePoints) < fourHundredCodePoints.Length; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(new string('r', 400)), null)), + ("historic/2", new RetryOperationRow(Historic(new string('r', 401)), null)), + ("historic/3", new RetryOperationRow(Historic(fourHundredCodePoints), null)), + ("pending/1", new RetryOperationRow(null, Pending(new string('p', 401), "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", new string('c', 451)))), + ("pending/3", new RetryOperationRow(null, Pending(new string('p', 400), new string('c', 450))))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/3")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + var historicRequestId = PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(HistoricRetryOperationEntity.RequestId)); + string[] expectedDetails = + [ + $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/2'", + .. countsCodePoints ? Array.Empty() : new[] { $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/3'" }, + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.RequestId))} takes at most 400 characters and the value holds 401, on document 'pending/1'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Classifier))} takes at most 450 characters and the value holds 451, on document 'pending/2'" + ]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(fourHundredCodePoints, Has.Length.EqualTo(401), "the emoji request id is one UTF-16 unit over the column, and 400 code points"); + Assert.That(result.Copied, Is.EqualTo(countsCodePoints ? 3 : 2), "a value exactly as long as its column fits, in either table"); + Assert.That(result.SkippedIds, Is.EqualTo(countsCodePoints ? new[] { "historic/2", "pending/1", "pending/2" } : new[] { "historic/2", "historic/3", "pending/1", "pending/2" }), + "400 code points fit a column that counts code points, and a skip there would lose a retry operation the database could hold"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = countsCodePoints ? 3 : 4 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(countsCodePoints ? new[] { new string('r', 400), fourHundredCodePoints } : new[] { new string('r', 400) })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.Classifier)), Is.EqualTo(new[] { (new string('p', 400), new string('c', 450)) })); + Assert.That(details, Is.EqualTo(expectedDetails)); + } + } + + [Test] + public async Task A_retry_operation_holding_a_character_the_database_refuses_is_skipped_as_malformed_value_where_the_database_refuses_it() + { + var dialect = ServiceProvider.GetRequiredService(); + var refused = dialect.RejectedCharacterIn("\0") is not null; + var historic = Historic("group-1"); + historic.Originator = "Sales\0Orders"; + var pending = Pending("group-2", "Shipping-Failures"); + pending.Originator = "Ship\0ping"; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("historic/2", new RetryOperationRow(Historic("group-3"), null)), + ("pending/1", new RetryOperationRow(null, pending)), + ("pending/2", new RetryOperationRow(null, Pending("group-4", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.SkippedIds, Is.EqualTo(refused ? new[] { "historic/1", "pending/1" } : Array.Empty()), "only a database that refuses the character fails the statement over it"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-3" } : new[] { "group-1", "group-3" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-4" } : new[] { "group-2", "group-4" })); + + if (refused) + { + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.MalformedValue] = 2 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(HistoricRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'historic/1'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'pending/1'" + })); + } + } + } + + [Test] + public async Task A_retry_operation_with_no_request_id_is_skipped_naming_the_table_and_the_rest_copy() + { + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(null!), null)), + ("historic/2", new RetryOperationRow(Historic("group-1"), null)), + ("pending/1", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it", + "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the table the operation was bound for"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "historic/1", "pending/1" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-1" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_second_pending_operation_with_the_same_key_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic("group-2"), null)), + ("pending/skipped", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Billing-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress)))); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2", "pending/1" } : new[] { "pending/1" }), + "the last pending operation with one key is kept and the earlier ones are dropped on every database, a case-only request id counts as the same key only where the column ignores case, one request retried as another type never merges, and a historic or skipped row never merges"); + Assert.That(merges[0].Detail, Does.Contain(requestIdIgnoresCase ? "kept 'GROUP-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'" : "kept 'group-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'"), + "the target logs this line, so it has to name the key it kept and the key it dropped without swapping them"); + } + } + + [Test] + public async Task A_pending_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService(); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + var first = await Target.Write(RetryOperationsCategory, BatchOf(("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("pending/2", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress))), + ("historic/1", new RetryOperationRow(Historic("group-2"), null))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), second); + var result = await Target.Write(RetryOperationsCategory, second, first.Saved with { Cursor = second.Cursor }); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2" } : Array.Empty()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another retry type never"); + + if (requestIdIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'group-2', 'FailureGroup' and dropped the row for 'GROUP-2', 'FailureGroup'")); + } + + Assert.That(result.Copied, Is.EqualTo(requestIdIgnoresCase ? 2 : 3), "the historic row and the other retry type are copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(requestIdIgnoresCase ? 2 : 1)); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.RetryType)), Is.EquivalentTo(requestIdIgnoresCase + ? new[] { ("group-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) } + : new[] { ("group-2", RetryType.FailureGroup), ("GROUP-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) })); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_retry_operation_time_of_local_or_unspecified_kind_is_copied_as_the_same_instant() + { + var historic = Historic("group-1"); + historic.StartTime = CompletedAt.AddMinutes(-1).ToLocalTime(); + historic.CompletionTime = DateTime.SpecifyKind(CompletedAt, DateTimeKind.Unspecified); + var pending = Pending("group-2", "Shipping-Failures"); + pending.StartTime = DateTime.SpecifyKind(CompletedAt.AddMinutes(-1), DateTimeKind.Unspecified); + pending.CompletionTime = CompletedAt.ToLocalTime(); + pending.Last = CompletedAt.AddMinutes(1).ToLocalTime(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("pending/1", new RetryOperationRow(null, pending))); + + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/1")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "Npgsql refuses a time that is not UTC for a timestamptz column, and that would fail the whole retry history"); + Assert.That(stored.HistoricOperations.Select(operation => (operation.StartTime, operation.CompletionTime)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt) }), + "an Unspecified time is already UTC and a Local time is converted, so both keep their instant"); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.StartTime, operation.CompletionTime, operation.Last)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt, CompletedAt.AddMinutes(1)) })); + } + } + + static HistoricRetryOperation Historic(string requestId) => new() + { + RequestId = requestId, + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + static UnacknowledgedRetryOperation Pending(string requestId, string classifier, RetryType retryType = RetryType.FailureGroup) => new() + { + RequestId = requestId, + RetryType = retryType, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = classifier, + NumberOfMessagesProcessed = 1 + }; + + static MigrationBatch BatchOf(params (string Id, RetryOperationRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.RetryOperations, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..98c6ec145d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +class SettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly string[] SettingsCategoryIds = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; + + [TestCase(MigrationCategoryIds.LicensingReportMasks)] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] + [TestCase(MigrationCategoryIds.NotificationSettings)] + [TestCase(MigrationCategoryIds.TrialEndDate)] + public async Task Count_is_the_row_under_the_categorys_own_key_alone(string categoryId) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == categoryId); + + foreach (var otherCategoryId in SettingsCategoryIds.Where(id => id != categoryId)) + { + await StoreSettingFor(otherCategoryId); + } + + Assert.That(await Target.Count(category), Is.Zero, "the other settings rows are not rows of this category"); + + await StoreSettingFor(categoryId); + + Assert.That(await Target.Count(category), Is.EqualTo(1), "counted as the whole Settings table, this would also count the other settings rows"); + } + + Task StoreSettingFor(string categoryId) => categoryId switch + { + MigrationCategoryIds.LicensingReportMasks => LicensingDataStore.SaveReportMasks(["contoso"]), + MigrationCategoryIds.LicensedEndpointDetails => LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }), + MigrationCategoryIds.NotificationSettings => NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + MigrationCategoryIds.TrialEndDate => ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + _ => throw new ArgumentOutOfRangeException(nameof(categoryId), categoryId, "This test stores no settings row for that category.") + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs new file mode 100644 index 0000000000..5334eef99d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class SubscriptionRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var row = SubscriptionRowMapper.Map(new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, row); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs new file mode 100644 index 0000000000..0b437aece1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs @@ -0,0 +1,154 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SubscriptionsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory SubscriptionsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions); + + [Test] + public async Task A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow(new string('t', 200), version, new string('a', 200), new string('e', 450))), + ("s/2", new SubscriptionRow(new string('t', 201), version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, new string('a', 201), "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", new string('e', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/4")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/2", "s/3", "s/4" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.ValueTooLong] = 3 }), + "an over-long value would fail the whole insert and halt the category"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.MessageType))} takes at most 200 characters and the value holds 201, on document 's/2'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.TransportAddress))} takes at most 200 characters and the value holds 201, on document 's/3'", + $"{PreparedSkips.TableAndColumnOf(ServiceProvider, nameof(SubscriptionEntity.Endpoint))} takes at most 450 characters and the value holds 451, on document 's/4'" + })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { (new string('t', 200), new string('a', 200), new string('e', 450)) }), "the row whose values all fit is stored as it was read"); + Assert.That(await Target.Count(SubscriptionsCategory), Is.EqualTo(1), "Count is the rows the Subscriptions table holds"); + } + } + + [Test] + public async Task A_second_version_of_a_type_for_one_subscriber_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService(); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderShipped", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2", "s/3" } : new[] { "s/2" }), + "the key has no version, so a second version is dropped on every database, a case-only address only where the column ignores case, and another type never"); + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01'"), "the target logs this line, so it has to name the key it kept"); + + if (addressIgnoresCase) + { + Assert.That(merges[1].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'"), + "the line must not swap the key kept and the key dropped"); + } + } + } + + [Test] + public async Task A_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService(); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + var first = await Target.Write(SubscriptionsCategory, BatchOf(("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))), CheckpointAfter("s/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), second); + var result = await Target.Write(SubscriptionsCategory, second, first.Saved with { Cursor = second.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2" } : Array.Empty()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another address never"); + + if (addressIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'")); + } + + Assert.That(result.Copied, Is.EqualTo(addressIgnoresCase ? 1 : 2), "the other address is copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(addressIgnoresCase ? 2 : 1)); + Assert.That(await StoredSubscriptions(), Is.EquivalentTo(addressIgnoresCase + ? new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") } + : new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "billing@billing01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + [Test] + public async Task A_subscription_missing_a_key_column_is_skipped_naming_the_column_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", version, TransportAddress: null!, "Billing")), + ("s/2", new SubscriptionRow(MessageType: null!, version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", "Shipping"))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "Subscriptions.TransportAddress is NOT NULL and the document has no value for it", + "Subscriptions.MessageType is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the column that is missing"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/1", "s/2" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + async Task> StoredSubscriptions() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var stored = await scope.ServiceProvider.GetRequiredService().Subscriptions.AsNoTracking().ToListAsync(); + + return [.. stored.Select(subscription => (subscription.MessageType, subscription.TransportAddress, subscription.Endpoint))]; + } + + static MigrationBatch BatchOf(params (string Id, SubscriptionRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.Subscriptions, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs index e2b93060ab..449c4cf85d 100644 --- a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -11,22 +11,19 @@ namespace ServiceControl.Persistence.DataMigration; public interface IMigrationTarget { /// - /// Makes the target ready to write. The host calls it before the copy and after the target's own checks have passed. + /// Makes the target ready to write. Called once, before the copy starts. /// Task Open(CancellationToken cancellationToken = default); /// - /// The most rows a source may return in one batch for this category. - /// The target picks it because its own database sets the limit, and a bigger batch fails the write. + /// The most rows this target can write in one batch for the category. A bigger batch fails. /// Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default); /// /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. - /// Add this batch's own outcome with and save that, so the stored counts are never a guess. - /// For a category that carries bodies, every row must have its body. - /// Bodies that go to external storage are written first, outside the transaction. So a failed write can leave a - /// stored body that no row points to, but never a row that points to a body that was not stored. + /// Build the checkpoint with . + /// Bodies kept outside the database are written first, so a failure can leave an unused body but never a row without one. /// Task Write( MigrationCategory category, @@ -35,26 +32,30 @@ Task Write( CancellationToken cancellationToken = default); /// - /// How many rows the target holds for one category, for progress and verify. - /// Counts only that category, even where two categories share a table. + /// How many rows the target holds for one category, even where categories share a table. /// Task Count(MigrationCategory category, CancellationToken cancellationToken = default); /// - /// The categories this target can write. A category outside this set is never handed to the engine. - /// Answers before Open and does not change across it. + /// Saves a reset checkpoint so its category can be copied again, and returns it with its new version. + /// Where the database makes the row keys, it also deletes the rows already copied, so none are copied twice. + /// Throws if another writer saved the checkpoint first. + /// + Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default); + + /// + /// The categories this target can write. Known before Open and never changes. /// IReadOnlyCollection SupportedCategoryIds { get; } /// - /// The type of this target takes, for each category in - /// . A row holding any other type fails its batch with an - /// . Answers before Open and does not change across it. + /// The type this target expects for each supported category. + /// A row of any other type fails its batch. Known before Open and never changes. /// IReadOnlyDictionary DocumentTypes { get; } } /// -/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +/// What the target did with one batch, and the checkpoint it saved with it. Every skipped row has a reason in SkipReasons. /// public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs new file mode 100644 index 0000000000..aa5939175e --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Recoverability; + +/// +/// One retry operation from the retry history document. Exactly one of Historic and Unacknowledged is set. +/// +public sealed record RetryOperationRow(HistoricRetryOperation? Historic, UnacknowledgedRetryOperation? Unacknowledged); diff --git a/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs new file mode 100644 index 0000000000..3441fd2fe6 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// +/// One subscriber of one message type, on its way from source to target. +/// +public sealed record SubscriptionRow(string MessageType, Version Version, string TransportAddress, string? Endpoint); diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs index b6ef99b0a1..927b8be477 100644 --- a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -176,6 +176,13 @@ async Task WriteNow( public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + /// + /// Saves the reset through the checkpoint store, so a stale Version throws as on the real target. + /// It deletes no rows, because no unit test needs that. + /// + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs new file mode 100644 index 0000000000..8843923d12 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs @@ -0,0 +1,18 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryStateExtensionsTests +{ + [Test] + public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsBenign()), + Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), + "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs index 4053d7d71b..7726a173f1 100644 --- a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -204,6 +204,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); @@ -223,6 +225,8 @@ public async Task Write(MigrationCategory category, Migrat public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + public Task ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); diff --git a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs index feedfd181d..8681aa14e9 100644 --- a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs +++ b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs @@ -6,6 +6,7 @@ namespace ServiceControl.Hosting.Commands; using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Particular.ServiceControl.Hosting; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; @@ -47,6 +48,7 @@ internal static async Task Run(Settings settings, Action var builder = WebApplication.CreateBuilder(); // Persistence also registers runtime-only services whose dependencies this command does not need. builder.Host.UseDefaultServiceProvider(options => options.ValidateOnBuild = false); + builder.Logging.ClearProviders(); builder.Logging.ConfigureLogging(settings.LoggingSettings.LogLevel); builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddPersistence(settings);