diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b276a4134..0af212e3f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: # Categories are declared by the property in each test project. A category can # span several projects that share infrastructure, so that the container is provisioned once # and the union of their build closures is compiled once. - test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] + test-category: [ DefaultCore, DefaultMonitoring, Migration, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] include: - os: windows-latest os-name: Windows @@ -96,7 +96,7 @@ jobs: id: select run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }} - name: Download RavenDB Server - if: matrix.test-category == 'DefaultCore' + if: contains(fromJSON('["DefaultCore", "Migration"]'), matrix.test-category) run: ./tools/download-ravendb-server.ps1 - name: Build id: build diff --git a/README.md b/README.md index d9a8edeb65..356a0cbc18 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ It's also possible to [locally test containers built from PRs in GitHub Containe If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile. +## Migrating from RavenDB to SQL Server or PostgreSQL + +- [Migrate from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md): what an operator can run today, and what is not built yet. + ## Secrets Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**. diff --git a/docs/README.md b/docs/README.md index 4c6ebc1521..3c8cd29743 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,6 +26,7 @@ This section points to sources that explain why ServiceControl is designed the w - [Message body storage](message-body-storage.md) — where the Audit SQL Server and PostgreSQL persisters keep message bodies, how retention removes them, and which settings are optional - [Bulk retries design](bulk-retries-design.md) — how ServiceControl retries failed messages in bulk - [Retries over Azure Storage Queues transport](retries-asq-transport.md) — how retries behave with one or several storage accounts +- [RavenDB-to-SQL migration design](migration/ravendb-to-sql-migration-overview.md) — why required data copies offline and persisted checkpoints drive online completion - [Data versioning design](data-versioning-design.md) — which invariant keeps a cached API response from going stale - [Event log design](eventlog-design.md) — what the event log is and what it records - [Multiple ServiceControl instances communication](multipleservicecontrolinstancescommunication.md) — how primary, audit, and monitoring instances talk to each other diff --git a/docs/migration/migration-system-design-diagram.png b/docs/migration/migration-system-design-diagram.png new file mode 100644 index 0000000000..85636355e7 Binary files /dev/null and b/docs/migration/migration-system-design-diagram.png differ diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md new file mode 100644 index 0000000000..27e456c9c0 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -0,0 +1,269 @@ +# Migration Engine: Instructions + +> [!NOTE] +> Production copying remains blocked until the complete migration ships. `--migration-source-report` works; `--start-data-migration` reports that copying is unavailable. + +This page tells you how to move the data of a ServiceControl error instance from RavenDB to SQL Server or PostgreSQL. The ServiceControl executable provides the offline command and the online copier, so you do not need a separate tool. [The migration overview](ravendb-to-sql-migration-overview.md) explains how the migration works and why. + +The migration does not move the audit instance or the monitoring instance. The audit instance has no migration path and the monitoring instance keeps its data in memory, so it has nothing to move. + +## How the migration runs + +Run `--start-data-migration` with ServiceControl stopped to copy the data it needs to open. **This part is your outage**. After the command succeeds, start ServiceControl to copy the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. + +> [!IMPORTANT] +> Until ServiceControl opens, you can go back to RavenDB and lose nothing. After it opens, you cannot go back. + +The migration copies the data in categories, and each category is Copying, Done, Failed or Abandoned. [The overview](ravendb-to-sql-migration-overview.md#when-something-goes-wrong-you-decide-what-happens-next) explains the categories and the states. + +The steps below follow this path: + +```mermaid +flowchart TD + subgraph prepare["ServiceControl runs on RavenDB"] + S1["1. Upgrade and run --migration-source-report"] + S2["2. Create an empty SQL database and run --setup"] + S3["3. Choose the optional data windows"] + S4["4. Run --migration-dry-run and book the outage"] + S1 --> S2 --> S3 --> S4 + end + + subgraph outage["ServiceControl stopped: the outage"] + S6["5. Stop ServiceControl and every ingestion worker, then run --start-data-migration"] + Q1{"Did --start-data-migration succeed?"} + Fix["Correct the reported cause"] + RetryAbandon1["Run --migration-retry or --migration-abandon for the Failed category"] + Back["Go back: set PersistenceType to RavenDB and start ServiceControl"] + S6 --> Q1 + Q1 -->|"No: a check failed or the command stopped"| Fix --> S6 + Q1 -->|"No: a required category is Failed"| RetryAbandon1 --> S6 + Q1 -.->|"Give up, nothing is lost"| Back + end + + subgraph online["ServiceControl open on SQL: no way back to RavenDB"] + S7["6. Start ServiceControl"] + S8["7. Watch the background copy with --migration-status"] + Q2{"Is an optional category Failed?"} + RetryAbandon2["Stop ServiceControl, run --migration-retry or --migration-abandon, then start it"] + S9["8. Stop ServiceControl, run --migration-verify, then start it"] + S7 --> S8 --> Q2 + Q2 -->|Yes| RetryAbandon2 --> S8 + Q2 -->|"No: every category is Done or Abandoned"| S9 + end + + S4 --> S6 + Q1 -->|Yes| S7 +``` + +## Before you start + +Make sure that your setup is supported: + +- The RavenDB source is one of these: + - embedded on the host (Windows installation only) + - self-hosted on your network, or RavenDB Cloud +- The primary database and the throughput database of RavenDB are on the same server or cluster. +- The SQL target is SQL Server or PostgreSQL. +- A SQL Server target has Full-Text Search installed. Message search needs it, so `--setup` fails without it. The stock SQL Server container image does not have it. PostgreSQL needs nothing extra. +- The ServiceControl host can connect to RavenDB, to the SQL database, and to the message body storage at the same time. +- If ServiceControl runs in a container, RavenDB is an external server. The container image does not contain the RavenDB server, so it cannot open an embedded database. + +Get the data ready: + +- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. The copy reads archived and resolved messages only after ServiceControl opens. The exception is a RavenDB index that is broken or leaves out some failed messages: then the copy reads every failed message during the outage, logs a warning, and a large archive makes the outage longer too. +- Run `--import-failed-errors` while the instance is still on RavenDB. A failed error import is a message that ServiceControl took off the error queue but failed to store. The migration copies these either way, but imported ones arrive as ordinary failed messages. + +Keep these settings as they are: + +- Keep `ServiceControl/RetryHistoryDepth` above 0. If it is 0 or less, the migration refuses to start. At that value, the first completed retry after the move deletes all the copied retry history. +- Do not change `ServiceControl/ErrorRetentionPeriod` or `ServiceControl/EventRetentionPeriod` until the migration is finished. The migration uses them to decide which rows are too old to copy. If one changes after an optional category has started copying with its default window, ServiceControl refuses to start and names the category. + +## Keep the RavenDB settings + +The migration reads RavenDB through the RavenDB settings that the instance already has. You do not add new settings for the source. Keep these settings in the configuration of the instance, also after you change `ServiceControl/PersistenceType` to SQL Server or PostgreSQL. + +| Setting | Environment variable | What it is | +| --- | --- | --- | +| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | The address of an external RavenDB server. Leave it unset for an embedded database. | +| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The data directory of the embedded database. | +| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database. The default is `primary`. | +| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database. The default is `throughput`. | +| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | The client certificate for a secured external server. | +| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. ServiceControl does not start without it. | + +In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You cannot leave out the `LICENSINGCOMPONENT_` prefix. + +## Step 1: Upgrade ServiceControl and make the source report + +> [!IMPORTANT] +> +> - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. +> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database before `--start-data-migration`, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. + +Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: + +```powershell +# Windows installation. Start this from the installation folder of the instance. +.\ServiceControl.exe --migration-source-report +``` + +```shell +# Container, against an external RavenDB server. +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-source-report +``` + +Where you run the report depends on the source: + +- External server: run the report while ServiceControl runs. The report only reads. +- Embedded database: stop the ServiceControl service first. Run the report, then start the service again. The report starts its own RavenDB process on the data directory, and it cannot do that while the instance holds the directory. + +The report prints: + +- The source persistence that it read. +- The RavenDB server version. +- Whether the source is embedded or external, and where it is. +- The name of each of the two databases, and the setting that each name came from. +- A row count for every collection in both databases. + +The report exits with 0 when it prints a report, and with 1 when it cannot. If the report fails, read [Errors from the RavenDB source](#errors-from-the-ravendb-source). + +## Step 2: Create an empty SQL database + +1. Create a new database on SQL Server or PostgreSQL. +2. In the configuration of the instance, set `ServiceControl/PersistenceType` to `SQLServer` or `PostgreSQL`. +3. Set `ServiceControl/Database/ConnectionString` to the new database. +4. Set `ServiceControl/MessageBody/StorageType` to `FileSystem`, `AzureBlob` or `S3`, and set the settings for that storage type. +5. Run `ServiceControl.exe --setup`. This creates the SQL schema, which includes the table where the migration saves its progress. + +## Step 3: Choose the optional data + +If you want less optional data, set a shorter window for the optional categories. A window is how far back an optional category copies. Set it to `0` to turn the category off. The SQL checkpoints record the migration. + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/EventLogWindow` | The value of `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | The value of `ServiceControl/ErrorRetentionPeriod` | How far back the copy of archived and resolved messages goes. | + +By default, the migration copies all the optional data that RavenDB still holds (current retension period setting). A window longer than the retention period copies nothing extra. After a category starts copying, you cannot change its window. To stop it early, abandon it. + +## Step 4: Run the dry run + +The dry run reads RavenDB and tells you what the migration will do, including an estimate of how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. + +Book the outage from the estimate in the report, and add a margin. It is an estimate, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. + +## Step 5: Run the offline command + +1. Stop ServiceControl and every ingestion worker. +2. Run `ServiceControl.exe --start-data-migration` from the instance installation folder. In a container deployment, run the command in a one-off container against the same databases. +3. Wait for the command to finish. It runs the startup checks, copies every required category, and saves progress in SQL. It never starts the API, ingestion, or other hosted services. +4. If the command fails, correct the reported cause. Run `--start-data-migration` again to resume. A Failed category still needs retry or abandon first. + +The command exits without starting ServiceControl. + +## Step 6: Start ServiceControl + +After the offline command reports success, start the ServiceControl service or container. ServiceControl detects the migration from SQL checkpoints and copies optional categories in the background. + +Normal startup refuses an unfinished required copy and tells you to run `--start-data-migration`; it never resumes that copy itself. + +> [!IMPORTANT] +> Do not start any `--error-ingestion-only` worker until the offline command succeeds and the main ServiceControl instance opens. A worker cannot detect a migration before its first checkpoint exists. + +If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). + +## Step 7: Watch the background copy + +ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: + +- The ServicePulse activity feed. It shows when a category fails and when RavenDB cannot be reached. +- The ServiceControl log. +- The output of `--migration-status`. This command does not open RavenDB, so you can run it at any time. + +If the copy slows down your production work, raise `ServiceControl/Migration/ThrottlePauseMilliseconds` and restart. This is the pause between two background batches. The default is 100 milliseconds. + +> [!IMPORTANT] +> To give up an optional category, use [the abandon command](#to-abandon-a-category). + +Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. + +## Step 8: Finish the migration + +1. Stop ServiceControl. +2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped rows. Merged and changed rows are not shown here. Each one is logged as a warning by the copy run that merged or changed it, and that run's summary counts them. It exits with 0 only when every category is Done or Abandoned. +3. Start ServiceControl. Finished checkpoints remain in SQL, but no background copy runs and RavenDB is not opened. + +Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time. + +If verify shows rows that the copy did not read, keep RavenDB. Those rows are only in RavenDB. RavenDB can show fewer rows than the copy read when nothing is wrong, because RavenDB deletes old rows. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) explains why. + +No configuration change is needed to finish. Once every category is Done or Abandoned, subsequent starts leave RavenDB unopened. + +## If something goes wrong + +### A startup check fails + +ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then run `--start-data-migration` again. The dry run runs the same checks, so you can use it to test your correction. + +### A category is Failed + +A category becomes Failed when it cannot copy rows, or when an error stops it. The rows that it already copied stay in SQL. A restart does not change a Failed category. You must run one of two commands. + +#### To retry a category + +1. Stop ServiceControl. +2. Read why the category failed. The reason is in the startup refusal and in the output of `--migration-status`. +3. Correct the cause. It is usually outside the migration, for example the body storage cannot be reached, a certificate expired, a database is down, or a disk is full. +4. Run `--migration-retry ` to reset the migration progress for that category. +5. For a required category, run `--start-data-migration` again before starting ServiceControl. For an optional category, start ServiceControl. The category copies again from the start. + +#### To abandon a category + +1. Stop ServiceControl. +2. Run `--migration-abandon `. When you abandon a category, the categories that depend on it are abandoned too. +3. If any required category remains unfinished, run `--start-data-migration` again. Otherwise, start ServiceControl. + +> [!IMPORTANT] +> **Abandon is final**. The rows that the category already copied stay in SQL, and the rest are not copied. Abandon a category when a retry cannot fix it, or when the data is not worth the time. You cannot retry the event log, so you can only abandon it. + +You can abandon a required category only after it starts copying. You can abandon an optional category at any time, for example to stop a large copy early. + +### You want to go back to RavenDB + +You can go back to RavenDB without losing data only while ServiceControl is still closed: + +1. Set `ServiceControl/PersistenceType` back to RavenDB. +2. Start ServiceControl. + +You lose only the copy. To try the migration again later, start from step 2 with a new, empty SQL database. + +> [!IMPORTANT] +> Do not use the old copy again. The migration skips the categories that it finished, and misses everything that RavenDB received after that. + +After ServiceControl opens on SQL, you cannot go back. You can only finish the migration, or abandon what is left. + +### Errors from the RavenDB source + +The source report, the dry run and the migration show these errors when they cannot read RavenDB: + +- `has no database named ...`: the database name in the setting that the message names is wrong. Correct that setting. +- `refused its client certificate access ...`: give that certificate Read access to the database, or use a certificate that already has this access. +- `is secured but no client certificate is configured`: the connection string starts with `https://`. Set `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`. +- `is valid from ... to ..., which does not include now`: the client certificate expired, or it is not valid yet. Use a current certificate. +- `ServiceControl expects RavenDB Server version ... or higher`: the external RavenDB server is older than this version of ServiceControl supports. Upgrade the RavenDB server. +- `could not start a server for the embedded database ...`: the rest of the line gives the reason. Usually a ServiceControl instance still holds the data directory. Stop that instance and try again. The other reasons are a port that is already in use, and a missing RavenDB server. +- `did not finish loading within 5 minutes`: another process holds the embedded data directory, or the directory is damaged. You cannot change the 5 minutes. + +## Commands and settings + +On an embedded source, stop ServiceControl before you run a command that opens RavenDB. **Only one RavenDB process can use the data directory**. The overview lists [every command and when it can run](ravendb-to-sql-migration-overview.md#the-dry-run), [the retry and abandon commands](ravendb-to-sql-migration-overview.md#retry-and-abandon), and [every migration setting](ravendb-to-sql-migration-overview.md#monitoring-and-settings). + +### Run a command in a container + +In a container, run each command as a one-off container of the same image, against the same databases: + +```shell +docker run --rm -it --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --start-data-migration +``` + +The RavenDB source must be an external server. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md new file mode 100644 index 0000000000..36c4d99b4d --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -0,0 +1,540 @@ +# Migration Engine: Design Overview + +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. + +[The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. + +## Overview + +### What this is + +- The migration moves an **error instance's** data from RavenDB to SQL Server or PostgreSQL, so a customer can switch persisters and keep their data. +- The ServiceControl executable provides an interactive offline command and an online background copier. There is no separate tool. +- **The audit instance is out of scope.** There is no migration path for the audit instance. +- **The monitoring instance is out of scope.** It keeps its data in memory, so there is nothing to move. + +### TL;DR + +- **Retention does most of the work.** Error retention is 5 to 45 days and event retention defaults to 14 days, so most of RavenDB ages out on its own within weeks. +- **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. +- **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. +- **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They are required, rarely age out (see the [6.18 exception](#rows-removed-after-the-copy)), and can be months old. Keep them low by resolving and archiving. The *dry run* tells you where you stand. +- **A big archive does not make the outage longer, as long as RavenDB has indexed every failed message.** The copy finds unresolved failures through a RavenDB index, and waits for it to catch up first. If the index is broken or incomplete, the copy reads every failed message instead and logs a warning. Then a big archive does make the outage longer. + +```mermaid +flowchart LR + A["--start-data-migration: checks"] --> B["Copy required data
ServiceControl stopped"] + B --> V["Required all Done
or Abandoned"] --> P["Command exits:
start ServiceControl"] + P --> C["ServiceControl detects checkpoints
optional data copies
in the background"] + C --> D["Everything Done or
Abandoned: no further copying"] +``` + +### When something goes wrong, you decide what happens next + +The end goal is simple: **either all the data comes across, or you explicitly give up on a named part of it.** + +- **Each *category* of data is in one of four states:** Copying, Done, Failed or Abandoned. +- **A category goes Failed when something went wrong**, for example a row it could not copy. It stays Failed, and the status says why. +- **You act with one of two commands**, with ServiceControl stopped: + - `--migration-retry `, after fixing the cause: rerun `--start-data-migration` for a required category, or restart ServiceControl for an optional category. + - `--migration-abandon `: give up on what it has not copied, and keep what it has. +- **A restart never decides anything for you.** A Failed category stays Failed until you run a command, so a restart policy cannot loop or lose data. +- **ServiceControl opens once every required category is Done or Abandoned**, and the migration is finished once every category is. + +```mermaid +stateDiagram-v2 + [*] --> Copying + Copying --> Done + Copying --> Failed: something went wrong + Failed --> Copying: --migration-retry + Failed --> Abandoned: --migration-abandon + Copying --> Abandoned: --migration-abandon + Done --> [*] + Abandoned --> [*] +``` + +### Goals + +- **Short outage.** Only the required data copies while ServiceControl is closed. +- **Support every RavenDB hosting shape.** Embedded, self-hosted or RavenDB Cloud, on one code path. +- **Never write to RavenDB.** RavenDB's own expiration still deletes old failed messages and event log items. +- **No duplicates and no gaps.** The copied rows and the *checkpoint* commit together, so a crash needs no clean-up. +- **Every identifier anything points at is kept.** Only the event log, historic retry operations and pending integration events get new ids, because nothing points at theirs. +- **Refuses rather than half-migrates.** Every check runs before the first row moves. +- **No silent loss.** Every skipped row is counted with a reason, and a migration cannot finish with a category unfinished unless someone deliberately *abandons* it. +- **Gentle on a live instance.** The copy is streamed, so memory does not grow with the database, and there is a pause between background batches. +- **Known before it starts, visible while it runs.** The dry run predicts the outage, and progress shows up as it happens. + +### Non-goals + +- **Zero downtime.** There is a real, if short, outage. +- **Rollback once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. +- **Pausing or steering a running copy.** There is no pause, resume or abort. Decisions are two commands run with ServiceControl stopped, and every other change is a configuration edit and a restart. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine is designed so it can be expanded in the future. + +### The moving parts + +```mermaid +flowchart LR + subgraph host["One ServiceControl process"] + src["RavenDB
source"] --> eng["Migration
engine"] --> tgt["SQL
target"] + end + old[("RavenDB
read only")] --> src + tgt --> sql[("SQL Server or PostgreSQL
plus the checkpoint table")] + tgt --> bod[("Body store
filesystem, Blob or S3")] +``` + +- **Both persisters load into one ServiceControl process**, each in its own `AssemblyLoadContext`, and both are live during the copy. +- **The engine knows neither database.** It works only through `IMigrationSource` and `IMigrationTarget`, in categories, *cursors* and counts, so it can be tested on its own and expanded in the future. + - The source maps each category to what it reads, and describes itself as labelled facts. + - The target maps each category to where it writes and how to count it. + - Each side adds its own startup checks. + - The engine passes the cursor from one side to the other without looking inside it. + - RavenDB to SQL is the only supported pair, but the engine does not depend on it. +- **The source reuses the instance's existing RavenDB settings**, so an existing customer configures nothing new for it. Leave them in place when switching `ServiceControl/PersistenceType`. + +### A migration from start to finish + +[The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. + +- **The [free abort](#going-back-to-ravendb) window runs from the first `--start-data-migration` invocation until ServiceControl opens.** A successful command does not end this window; starting ServiceControl does. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The offline command never starts the host, and normal startup refuses while a required category is unfinished. + +## Design + +### Supported locations + +The copy goes from RavenDB, through the ServiceControl host, to SQL, because documents must be reshaped into tables and ServiceControl's persisters already know both sides. So the host must reach both ends, and distance matters. + +| | Supported locations | +| --- | --- | +| RavenDB source | Embedded on the host (Windows installations only), self-hosted on the network (container, VM or bare metal), or RavenDB Cloud | +| SQL target | SQL Server or PostgreSQL on the host, on the network, in a container, or managed (Azure SQL, Amazon RDS, Google Cloud SQL) | +| Body store | Wherever the target is configured to put bodies: filesystem, Azure Blob or S3. Small text bodies stay in the database. This choice is made at the same time as the database move | + +| Route | What to expect | +| --- | --- | +| On-prem to on-prem | The common case, and the fastest: embedded or self-hosted RavenDB to SQL on the same host or network | +| On-prem to cloud | Works. Each batch is a round trip, so write latency adds up per batch | +| Cloud to on-prem | Works. The customer pays egress on everything copied, mostly archived messages and their bodies | +| Cloud to cloud | Works, but only sensible when ServiceControl runs next to one of them. A host on-prem between two clouds pulls every byte down and pushes it straight back up | + +**Requirements:** + +- The host reaches the RavenDB source, the SQL target and the body store at the same time. +- The SQL database is new and empty: only the schema `--setup` created. +- Both RavenDB databases, primary and throughput, are on one server or cluster. The migration opens both through one `IDocumentStore`, and so does the licensing component. +- A SQL Server target has Full-Text Search installed, because message search needs it. `--setup` fails without it, and the stock SQL Server container image does not include it. PostgreSQL needs nothing extra. +- Transient failures on a managed target are retried by default. + +**Not supported:** + +- Any server-to-server copy: backup and restore, RavenDB ETL or replication, or an external pipeline. +- A host that can reach only one database at a time, so no staged move through an offline copy. +- An embedded RavenDB source when ServiceControl runs in a container. Reading an embedded database means starting a RavenDB server process, and the container image does not include one. +- Primary and throughput RavenDB databases in different places. +- Any source other than RavenDB, or any target other than a ServiceControl SQL persister. + +### What data moves + +There are eighteen *categories*: sixteen required and two optional. + +| Category | Kind | Why | +| --- | --- | --- | +| Unresolved and retry-issued failed messages, with bodies | Required | The point of the instance. Issuing a retry removes the expiry, so retry-issued messages never age out. Leave one behind and the retry confirmation finds no row to resolve, so the message stays missing even though the retry succeeded | +| Message redirects | Required | | +| Known endpoints, with the monitored flag | Required | The flag is part of the endpoint row | +| Endpoint settings | Required | Copied after known endpoints | +| Notification settings | Required | | +| Licence trial end date | Required | | +| Licensing endpoint records | Required | The per-endpoint rows in the throughput database | +| Throughput history | Required | Copied after licensing endpoint records, because each day's row hangs off one | +| Retry operations, unacknowledged and historic | Required | One category, because RavenDB holds both lists in one document | +| Licensing report masks | Required | | +| Uploaded licensed endpoint details file | Required | Nothing recomputes it. Losing it means downloading it again from the licence portal and uploading it again | +| Subscriptions | Required | | +| Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been, except a `MessageFailed` event for a message archived or resolved before the move (see [Rows removed after the copy](#rows-removed-after-the-copy)). RavenDB sends them in no particular order anyway, so no ordering is lost | +| Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts. Results of the six checks the RavenDB persister runs about its own database, matched on id and category, are left out, as harmless skips | +| Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | +| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. Some comments are removed after ServiceControl opens, see [Open decisions](#rows-removed-after-the-copy) | +| Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | +| Archived and resolved failed messages, with bodies | Optional | The biggest category by far, and most of the copying time | + +**Never copied, because nothing needs them:** + +- RavenDB index definitions. +- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `ArchiveOperations` and `UnarchiveOperations`. +- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. +- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Nothing has read it since 6.2. +- Broker and audit service version details, which refill on the next throughput collection. +- Failed message edit locks, which stop one failed message being edited twice. See [Open decisions](#values-that-change-or-merge-in-sql). +- Heartbeat state, which neither persister stores. It is rebuilt in memory, so monitoring after the move behaves as after any restart. + +**Written new by the copy rather than copied:** + +- Retry claims in `FailedMessageRetries`. A claim ties a message to the retry that sent it. The copy writes one for each retry-issued message it inserts that RavenDB still held a claim for, so a retry by id sends or holds back a message just as it did before the move. + +### Optional data windows + +- **Each optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. +- **The window defaults to the instance's own retention period**: `ServiceControl/EventRetentionPeriod` for the event log, `ServiceControl/ErrorRetentionPeriod` for archived and resolved messages. So by default everything RavenDB still holds is copied. +- **`0` turns a category off before it starts.** A longer window than the retention period copies nothing extra, because older rows would only be deleted by SQL's clean-up. +- **The window counts back from when the category first started copying**, which the checkpoint records, so a restart copies the same window rather than one that has slid forward. +- **The window is fixed once the category starts.** A changed value, including a changed retention period behind a default window, is refused at startup, naming the category. To stop a started category short, abandon it. + +### Data that changes shape + +- **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. For historic retry operations and pending integration events, the log says so once per category, in an information line. +- **Other values change or merge on the way into SQL.** Each is listed under [Open decisions](#values-that-change-or-merge-in-sql), with the line the log shows for it. + - A row copied with a changed value logs a warning starting `Changed in category on the way into SQL:`, followed by what changed and what was kept. + - A row merged into another logs a warning starting `Merged in category into a row with the same key:`, naming both keys, the source row kept, or the row SQL already held, and the source row dropped. + - The category's summary at the end of the copy counts both, apart from the rows already present, for example `Subscriptions: 14 copied (0 changed), 0 already present, 1 merged, nothing skipped`. The summary is a warning when anything was changed or merged. The counts cover only the current run: a category resumed after a crash counts only the changes and merges of the run that finished it, and counts earlier merges as already present. + - The summary's already present count leaves out the merged rows. If the copy reports more merged rows than SQL found already present in total, the summary adds a warning naming both counts, because the copy's prediction of which keys SQL treats as equal then disagrees with SQL. + - The per-row lines are written after each batch commits. If the process stops between a batch's commit and its log lines, those lines are lost. A resumed copy does not repeat them, because it finds those rows already present. + +### Skipped rows + +Every skipped row is counted under a reason, and its id is written to the log. + +> [!IMPORTANT] +> The log is the only record of each skipped row. Look them up in RavenDB while you still have it. + +A skip is one of three kinds: + +- **Fault, a retry can fix it.** Something outside the data went wrong. Fix the cause and retry. +- **Fault, no retry can fix it.** The RavenDB document cannot become a SQL row, so a retry skips it again. Only [abandoning](#retry-and-abandon) gets past it. +- **Harmless.** A row SQL would have removed anyway, or a result of a check the source's own storage runs about itself (the RavenDB source flags its own checks, and the target skips flagged rows). + +A value SQL can hold only after changing it is not a skip. Those rows are copied, and are listed under [Open decisions](#values-that-change-or-merge-in-sql). + +**Fault skips a retry can fix:** + +| Reason | +| --- | +| A message body that cannot be read after three tries. The whole message is skipped, because a message with no body is worse than none | +| An error from the SQL target | + +**Fault skips no retry can fix:** + +| Reason | +| --- | +| A message body that RavenDB does not hold, counted as `BodyMissing`. The whole message is skipped. A retry does get past a message that expired during the copy | +| A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | +| A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | +| A row missing a value SQL requires, such as a known endpoint with no name, or a failed message with no failing endpoint address to retry to | +| A text value too long for its SQL column, such as a failed message id or failing endpoint address over 450 characters, or a subscription's message type or address. Cutting it would change a key or send retries to the wrong address. The skip names the column and the length | +| A value of the wrong type, such as a `TimeSent` that is not a date. On PostgreSQL, also a text value holding the character U+0000, which PostgreSQL cannot store | + +**Harmless skips:** + +| Reason | +| --- | +| A blank group comment, which SQL never stores | +| An archived or resolved message, or event log item, already past its retention period | +| A result of one of the six checks the RavenDB persister runs about its own database, such as index lag or free disk space, matched on the check's id and category. Only a RavenDB primary instance reports these checks, so nothing updates them after the move. A check that reuses one of these ids under another category is copied, such as a RavenDB audit instance's dirty memory check | + +- **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. +- **A malformed document does not stop the copy.** It is skipped and counted, and the rest of the category carries on. If skips pass the [early-stop threshold](#failed-categories), the category stops where it is, and abandoning it then gives up every unread row too. Raise the threshold or fix the cause, and retry. +- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them itself shortly after it opens. +- **Rows outside a window are not skips.** They are simply not part of the copy. +- **Rows RavenDB expires during the copy are not skips either.** Only archived or resolved failed messages and event log items expire, plus some unresolved messages from 6.18 or earlier (see [Open decisions](#rows-removed-after-the-copy)). + - A message that expires between the copy reading it and reading its body is a `BodyMissing` fault skip. A retry gets past it. + - If the message a stopped copy was at expires before the next start, the category stops with an error naming the missing id. `--migration-retry` gets past it. + - A window shorter than the retention period keeps the background copy clear of the expiry sweep. + +### Startup checks + +In this order, cheapest first: + +1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. `--start-data-migration` refuses a RavenDB target, because RavenDB is only ever the source. +2. The optional category windows are valid time spans of zero or more. +3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history. +4. The SQL schema is current. This also refuses a database upgraded without `--setup`. +5. Until a required category has started copying: the SQL database holds no ServiceControl data. `--setup` writes none, but even one plain start of ServiceControl writes rows. Copying into a database that already served on SQL would mix the two. +6. Message body storage is writable. The probe body it writes is deleted again. +7. The SQL target opens, and no optional category's window has changed since that category started. +8. The RavenDB source opens: + - an embedded server starts; + - on an external server, the client certificate is in date and an `https://` address has one, checked before the first request; + - an external server is at least the RavenDB client's version; + - both databases load, within 5 minutes each on an embedded source. + + If every required category is already Done or Abandoned, the offline command can succeed without opening the source. Normal startup does not need the source; only the online copier opens it. + +Checks stop at the first refusal, which names the check. + +```mermaid +flowchart LR + A["--start-data-migration: checks pass?"] -->|No| X["Command fails
ServiceControl stays stopped"] + A -->|Yes| B["Copy every required
category"] + B --> C["All Done
or Abandoned?"] + C -->|No| X + C -->|Yes| D["Command succeeds:
prompt operator to start ServiceControl"] + D --> E["Start ServiceControl:
copy optional categories"] +``` + +- **The host also stays closed** if a second instance is writing the same checkpoints. +- **When ServiceControl opens**, the target is marked as opened, which ends the [free abort](#going-back-to-ravendb). +- **Once it has opened, a RavenDB outage does not stop it.** Only the background copy needs the source then, so ServiceControl opens anyway, and the optional categories [wait for the source](#failed-categories) to come back. + +### Reading from RavenDB + +- **The source opens RavenDB read-only**: connect, check the version, stop. It never runs RavenDB's database setup, and it refuses any write request. +- **Both source databases are opened through one `IDocumentStore`**, which is why they must be on the same server or cluster. +- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. +- **Distance to the source sets the pace.** Each message body is one round trip, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. + +### Writing to SQL + +- **A failed message is written as one row**, with its status unchanged. Its older attempts are dropped, see [Open decisions](#values-that-change-or-merge-in-sql). +- **A failed message keeps its failure groups and its last-modified time**, so filtered lists show the same messages as before. A failure group with no id is the exception, see [Open decisions](#values-that-change-or-merge-in-sql). +- **`UniqueMessageId` keeps its value** but becomes a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry key and the body key. +- **`StatusChangedAt` is rebuilt.** For archived and resolved messages it is `@expires` minus the current `ServiceControl/ErrorRetentionPeriod` (wrong if that changed, see [Open decisions](#values-that-change-or-merge-in-sql)). For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL`. +- **Bodies go through `IBodyStoragePersistence`**, which handles compression and the body store. The copier keeps a body inline when it is within `ServiceControl/MaxBodySizeToStore` (102,400 bytes by default), as ingestion does. +- **Throughput rows are written directly, and each day's count is added to the stored count**, as ServiceControl's own throughput write adds it. Endpoints whose names the SQL key treats as one have their counts for a day summed into one row: SQL Server compares names under the key column's own collation, and PostgreSQL compares them as its deterministic collations do, character for character. A resumed copy cannot double-count, because each batch and its checkpoint commit together. The copied rows also stop the collectors gathering the same days again. + +### Batch size and throttling + +- **Batch size comes from the database.** SQL Server divides its parameter budget by the column count, PostgreSQL uses a flat 50 rows. +- **Background batches have a pause between them**, 100 ms by default, set by `ServiceControl/Migration/ThrottlePauseMilliseconds`. The first batch of each category is not paused. +- **Raise the pause if the copy competes with production.** To stop an optional category permanently, abandon it. There is no enable switch. +- **The required copy is never paused**, because ServiceControl is closed and nothing competes with it. + +### Progress and checkpoints + +The checkpoint is one row per category in the SQL database, created by `--setup`. It holds the category's *state*, cursor, counts, timings, last error and window, plus a version number that catches a second writer. + +```mermaid +sequenceDiagram + participant E as Engine + participant S as RavenDB + participant T as SQL + loop each batch + E->>S: Read rows after the cursor, and their bodies + E->>T: Write rows + new counts + new cursor + Note over T: One transaction + E->>E: Stop as Failed if too much of this run was skipped + end + E->>T: Settle as Done or Failed +``` + +- **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. +- **A crash costs only the batch in flight.** Rerun `--start-data-migration` for required data, or restart ServiceControl for optional data. The copier resumes from the committed cursor. +- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. +- **Every skip has a reason, or the save is refused.** +- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts pointed at one database cannot interleave. +- **A copy stopped between two categories still shows the ones it never reached.** Each start first saves a *not started* row for every category it will copy. +- **Only the copier and the two commands write the checkpoint.** The status and verify commands only read it. + +### Failed categories + +A category is *Failed* when something went wrong. What it copied stays committed, and it stays Failed, with the reason, until you run a [command](#retry-and-abandon). + +**What makes a category Failed:** + +- **Any fault skip by the end of the category**, even one. +- **Too many fault skips in this run, which stops it early:** more than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Below that, a category runs to the end, so you see every bad row in one pass. +- **The category's own counts not balancing:** every row read must be copied, skipped or already present. +- **In a required category only, an exception or a stall of 30 minutes with nothing committed**, such as the body store unreachable, a certificate expired, the source or target down, or the disk full. The error and the cursor are recorded. + +**What does not make it Failed:** + +- **Harmless skips.** +- **A shutdown or a crash.** The category stays Copying. The next `--start-data-migration` invocation resumes required data; the next ServiceControl start resumes optional data. +- **An exception or a stall in an optional category**, including a RavenDB outage. The error shows in status, and the next start carries on from the cursor. If it keeps coming back, retry or abandon the category. + +**What a Failed category costs:** + +- **The other categories still run**, so one fix-and-retry covers them all. A category that depends on a Failed one waits. +- **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. +- **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. + +Some losses leave a category Done without any check noticing. They are listed under [Open decisions](#what-the-checks-do-not-catch). + +### Retry and abandon + +Stop ServiceControl before either command. In a container, run it as a one-off `docker run --rm ` against the same database. + +| Command | Works on | What it does | +| --- | --- | --- | +| `--migration-retry ` | A Failed category, or an optional one still copying with an error | The next `--start-data-migration` invocation copies a required category again; the next ServiceControl start copies an optional category again | +| `--migration-abandon ` | A Failed or started required category, or any optional one | Gives up on what it has not copied, and on any category whose rows hang off it. What they copied stays in SQL. Final | + +**To retry:** + +1. Read the reason in the refusal, the log or `--migration-status`. It names the skipped rows by reason, or the error. +2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or target down, or the disk full. +3. Run `--migration-retry `. +4. For a required category, run `--start-data-migration` again before starting ServiceControl. For an optional category, start ServiceControl. + +**What a retry does:** + +- **It re-reads the whole category from the start**, leaving rows already in SQL alone. Every body is read again, so a retry takes about as long as the first copy. +- **Historic retry operations, pending integration events and throughput history** are deleted first. The first two get new keys on every insert, and throughput history adds each day's count to the stored one, so a re-read would copy them a second time. The retry deletes what the category copied, in the same save as the reset. This is safe because all three are required, so the target started empty and ServiceControl has not opened on it. Once ServiceControl has opened on the database, a retry of any of them is refused and deletes nothing. +- **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. + +**To abandon:** + +- **A required category can be abandoned once it has started.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone first, point back or start over. +- **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. +- **Abandoning cascades** to throughput history (from licensing endpoint records) and endpoint settings (from known endpoints), and the command names each. Group comments are not abandoned with unresolved failed messages. +- **A category that failed only on [skips no retry can fix](#skipped-rows) should be abandoned, not retried.** If it stopped early at the threshold, raise the threshold and retry first, because abandoning gives up every unread row too. + +> [!IMPORTANT] +> Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. + +### Ingestion workers + +- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, because only SQL can scale out. +- **On SQL it starts only when every required category is Done or Abandoned**, based on SQL checkpoints. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. +- **Every SQL writer runs that check**, because ingesting into a part-copied database would make going back a loss. +- **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. +- **A worker or import that passes the check marks the target as opened**, which ends the free abort, because going back would lose what it wrote. + +> [!IMPORTANT] +> **Stop workers before running `--start-data-migration`, and restart them only after the main host has opened.** A worker cannot detect a migration before its first checkpoint exists, see [Open decisions](#gaps-nothing-enforces). + +### Going back to RavenDB + +**While ServiceControl is closed**, RavenDB is still the source of truth, because the migration writes nothing to it. This is the *free abort*. RavenDB's own expiration still runs unless you disabled it. + +- **To go back:** point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. +- **To try again later:** start from a new, empty SQL database (`--setup`). + +> [!IMPORTANT] +> Never reuse the old copy. It would skip every category already finished and miss anything RavenDB received since. Nothing detects the reuse, see [Open decisions](#gaps-nothing-enforces). +> +> **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. + +- **Every SQL start checks the checkpoint table.** An unfinished required category keeps the host closed until `--start-data-migration` finishes it. Outstanding optional categories start the online copier but do not stop ServiceControl opening. With nothing outstanding, ServiceControl does not open RavenDB. + +**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl opens, nothing can start without it. After that, the optional categories wait for it, or can be abandoned. + +### The dry run + +The dry run never writes to RavenDB, and can run again later against whatever is outstanding. It reports: + +- Whether the source is embedded or external, and which server. +- Both RavenDB database names, and the setting each came from. +- Rows and body volume per category, counting only what is inside each window. +- An estimate of how long ServiceControl will be closed. +- **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. +- The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. +- How many failed error imports there are. They are copied either way, but running `--import-failed-errors` on RavenDB first is better. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. + +It says nothing about load on the source. + +**When each read-only command can run:** + +| Command | What it does | Opens RavenDB? | +| --- | --- | --- | +| `--migration-source-report` | Source facts and a document count per collection | Yes | +| `--migration-dry-run` | The report above | Yes | +| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read. Exits 0 only when every category is Done or Abandoned | Yes | +| `--migration-status` | Each category's state, progress, last error and the commands it can take | No, so it runs any time | + +- **On an embedded source, stop the ServiceControl service before any command that opens RavenDB.** A second RavenDB process cannot use a data directory the first one holds, so the dry run is part of the outage. +- **Verify reads every selected category in full**, though not the bodies, so it takes a while on a big instance. +- **On an external source** they all run against a live instance. +- **In a container**, run each as a one-off `docker run` of the same image against an external RavenDB server. + +**What verify compares:** + +- **RavenDB against the rows the copy read, never against SQL now**, because normal use changes SQL rows within minutes of opening. +- **RavenDB holding more rows than the copy read means rows were never read.** Keep RavenDB. The same flag shows if something else wrote to RavenDB, such as an old instance still running on it. +- **RavenDB holding fewer is shown, not judged**, because expiry keeps deleting rows after the copy reads them. Expiry can also hide rows that were never read, see [Open decisions](#what-the-checks-do-not-catch). +- **A windowed category is counted inside the window it started with.** +- **Verify also prints how many rows SQL held for each category when it settled.** + +### Monitoring and settings + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/EventLogWindow` | `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. `0` turns it off | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | `ServiceControl/ErrorRetentionPeriod` | How far back the archived and resolved copy goes. `0` turns it off | +| `ServiceControl/Migration/ThrottlePauseMilliseconds` | `100` | Pause between background batches | +| `ServiceControl/Migration/HaltThresholdPercent` | `5` | Percentage part of the early-stop threshold | +| `ServiceControl/Migration/HaltThresholdMinimum` | `100` | Floor part of the early-stop threshold | + +- **The source needs no new settings.** It reads the instance's existing RavenDB settings. +- **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** There is no migration custom check instead migration status will be surfaced with a custom ServicePulse UI. +- **There is no HTTP API, pause, resume or abort.** Stopping a copy takes a restart. + +## Open decisions + +Each case below changes, merges or drops data, or lets a loss pass without failing a category. The design accepts each one for now, but each needs a decision before the migration ships. + +### Values that change or merge in SQL + +- **Attempt history collapses to the newest attempt.** SQL keeps the newest attempt and the attempt count, so only the newest attempt keeps its headers and exception. This is not logged and not counted as a change, because SQL keeps only the newest attempt for every message that failed more than once, by design. +- **A failure group with no id is dropped from its message, and a group with no title or type is stored with an empty one.** SQL keys a group by its id, so the message is missing from that group in ServicePulse, and a group retry or archive leaves it out. ServiceControl's classifiers always set an id, so this needs hand-made data. The log names each message, in a warning whose detail starts `Dropped the failure groups with no id` or `Stored an empty title or type for the failure groups that had none`. +- **Two failure groups of one message whose ids SQL treats as one key become one**, keeping the first group's title and type. ServiceControl never gives a message the same group twice. The log names each message, in a warning whose detail starts `Kept the failure group '' titled '' and dropped`. +- **Message redirects, subscriptions, pending retry operations, group comments and failed error imports merge when their keys match in SQL.** The copy keeps one row, counts the other as already present, and logs a warning starting `Merged <source id> in category <category> into a row with the same key:` naming both. On SQL Server the copy predicts these merges from the key column's collation settings for case, accents, kana type and width, plus SQL Server's trailing-space rule, and reports the merges those settings make. It compares text with .NET's culture rules rather than SQL Server's own tables, so an unusual character the two treat differently can merge without a line. + - Redirects merge on source addresses the key column treats as equal. On SQL Server, by default, that includes addresses that differ only in case. + - Subscriptions also merge when they differ only in message-type version, because the SQL key leaves the version out. The warning names the version and endpoint of the row kept and of the row dropped. + - Pending retry operations with the same retry type and request id merge, keeping the latest. The warning names the source row kept and the one dropped, and each source id ends with that operation's completion time. + - On SQL Server, group comments merge when their group ids differ only in trailing spaces. ServiceControl makes every group id from a GUID, so only a comment written on a hand-made group id can merge. PostgreSQL keeps both. + - Failed error imports of the same message merge. Version 6.18 and earlier stored each failed import attempt as its own document, and SQL keeps one row per message. The copy keeps the first it reads, and for those documents RavenDB order is effectively random, so it can keep an older attempt's exception. The warning names both documents, or the stored row's failure time and this one's. +- **On SQL Server, endpoint settings whose names differ only in case, kana type or width merge into one row**, because the name column's default collation ignores all three. A column whose collation also ignores accents merges those too. PostgreSQL keeps both. + - The copy keeps the setting whose name exactly matches a known endpoint, because ServiceControl deletes the other. Otherwise it keeps the first in RavenDB order, which is effectively arbitrary. + - The choice is made within one batch of up to a thousand. Across batches, the first one met wins. + - The copy logs a warning starting `Merged EndpointSettings/<name>` that names both names and the one kept, and the dry run lists each pair. +- **A message type over 450 characters is cut to its last 450 characters**, as ingestion does. The row is copied, not skipped. The log names each message, in a warning whose detail starts `Cut the message type from <length> characters to its last 450`. +- **On SQL Server, licensing endpoints whose names differ only in trailing spaces become one row**, and a day both counted holds the sum. PostgreSQL keeps both. + - The copy keeps the endpoint it meets first, counts the other as already present and logs a warning starting `Merged <source id> in category LicensingEndpoints` naming both. Within one batch the endpoint met first is the one whose source id sorts first, and an endpoint SQL already holds is met before any in the batch. The dry run does not list it. + - A key column whose collation also ignores, for example, accents, kana type or width makes one row in the same way, a day holds the sum of every endpoint the column treats as one, and the copy logs the same warning for it. +- **`StatusChangedAt` is wrong for an archived or resolved message if `ServiceControl/ErrorRetentionPeriod` changed since RavenDB stamped it.** It is off by the difference, but the message still ages out of SQL when RavenDB would have deleted it. Archived and resolved messages are not copied yet, so no log line flags these. +- **Failed message edit locks are not copied.** On RavenDB, an edited message that fails again can never be edited again. After the move it can be edited once more. No log line lists these messages. +- **A failed message's sending and receiving endpoints are rebuilt from its headers** when RavenDB hands back the stored endpoint without its type, following ingestion's own rules on the same headers. A message whose headers name no endpoint gets none. No log line names these messages. +- **A failed error import's failure time is when RavenDB last modified its document**, because RavenDB stores no failure time for an import. ServiceControl never changes an import after storing it, so this is when the import was stored. No log line names imports for this. A missing exception or body is stored empty, and the log names each such import, in a warning whose detail starts `Stored an empty exception`, `Stored an empty body` or `Stored an empty exception and an empty body`. + +The dry run counts the subscription and endpoint settings merges before anything moves. + +### Rows removed after the copy + +- **A group comment on a group whose messages are all archived or resolved is removed soon after ServiceControl opens**, because that group has no failed messages in SQL until the background copy brings them. The same happens when the group's messages are gone from RavenDB or were skipped by the copy. The copy names each such comment, in a warning starting `Changed GroupComment/<group id> in category GroupComments on the way into SQL: The group '<group id>' has no copied failed message`, which also quotes the whole comment. +- **A `MessageFailed` event still waiting to be sent for a message archived or resolved before the move is never sent.** ServiceControl looks the message up when it sends the event. Archived and resolved messages arrive only with the background copy, so the lookup finds nothing, and the event is deleted with no log line. RavenDB would have sent it. +- **Unresolved messages from version 6.18 or earlier can expire during the copy.** On those versions, a message that failed again after being archived or resolved kept its expiry. One deleted before the copy reads it is not copied and not counted. No log line names these messages yet. + +### What the checks do not catch + +- **Nothing counts RavenDB during the copy.** A read that ends early still balances and ends Done. `--migration-verify` finds it afterwards. +- **Rows never read on a Done category are reported, not fixed.** Verify's exit code ignores them, and no command can recopy a Done category. Keeping RavenDB is the remedy. +- **Verify uses the same reader as the copy**, so it cannot catch a reader that always misses the same rows, or a source pointed at the wrong database. +- **Expiry can hide rows that were never read**, because in verify a deleted row and a missed row cancel out. +- **A write that silently stores fewer rows still ends Done**, because already present is what is left after copied and skipped. +- **A crash during a batch with bodies leaves orphan bodies** in the body store, never rows pointing at missing bodies. + +### Gaps nothing enforces + +- **An ingestion worker started at the same moment as the first migration start can slip in before the first checkpoint exists**, and ingest into a part-copied database. Only the instructions prevent this. +- **Reusing an old copy after going back to RavenDB cannot be detected**, because it looks exactly like a normal resume. The copy skips every category already finished and misses anything RavenDB received since. + +## Glossary + +- **Category:** one kind of data copied as a unit, such as known endpoints or the event log. +- **Required category:** one ServiceControl needs the moment it opens, so it copies while ServiceControl is closed. +- **Optional category:** one copied in the background after ServiceControl opens. It can be shortened or turned off before it starts. +- **Window:** how far back an optional category copies. +- **Closed period:** the time ServiceControl is not serving traffic because the required copy is running. +- **Checkpoint:** the row in SQL recording one category's progress, saved in the same transaction as the rows it describes. +- **Cursor:** the position in RavenDB the copy has reached in a category. +- **State:** one of Copying, Done, Failed or Abandoned. +- **Copying:** not finished yet, including waiting to start, or interrupted until the next offline command or online copier invocation. +- **Done:** every row is in SQL, or was left out for a harmless reason, and the counts balance. +- **Failed:** something went wrong. Stays Failed until you run `--migration-retry` or `--migration-abandon`. +- **Abandoned:** you gave up on what a category had not copied, or on one it depends on. Final. +- **Already present:** a row the copier found in SQL already, so it left it alone. +- **Fault skip:** a row lost because something went wrong. +- **Harmless skip:** a row left behind because SQL would have removed it anyway, or because it is a result of a check the source's own storage runs about itself (the RavenDB source flags its own checks, and the target skips flagged rows). +- **Free abort:** going back to RavenDB at no cost, possible only while ServiceControl is still closed. +- **Dry run:** a read-only rehearsal that reports what would move, what would be skipped and how long the outage would be. + +## Further reading + +- [The instructions](ravendb-to-sql-migration-instructions.md): how an operator runs the migration. +- [How the migration is put together](ravendb-to-sql-migration-system-design.md): which class does what. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md new file mode 100644 index 0000000000..d066366a1b --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -0,0 +1,302 @@ +# Migration Engine: System Architecture Design + +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. The offline command, checkpoint-based startup gate, and online orchestration are present, but production copying remains blocked by `MigrationIsReleasedCheck`. + +[The overview](ravendb-to-sql-migration-overview.md) covers behaviour and guarantees. [The instructions](ravendb-to-sql-migration-instructions.md) cover the operator procedure. This page maps both onto types, call order and assembly boundaries. + +> [!IMPORTANT] +> Startup check order and category order are load-bearing. Read the required copy and category sections before reordering checks or adding a category. + +## Overview + +```mermaid +flowchart LR + subgraph host["ServiceControl host"] + req["MigrateCommand<br/>then MigrationStartup"] + opt["OptionalCategoryCopier"] + guard["MigrationBeforeTheHostOpens"] + cmd["Migration commands"] + end + subgraph neutral["ServiceControl.Persistence"] + eng["MigrationEngine"] + end + subgraph raven["ServiceControl.Persistence.RavenDB"] + src["RavenMigrationSource<br/>IMigrationCategoryReader per category"] + end + subgraph ef["ServiceControl.Persistence.EFCore"] + tgt["EFCoreMigrationTarget<br/>IMigrationCategoryWriter per category"] + cp["EFMigrationCheckpointStore"] + end + req --> eng + opt --> eng + guard --> cp + eng --> src + eng --> tgt + eng --> cp + cmd --> cp + cmd --> src + cmd --> tgt + src --> rdb[("RavenDB<br/>read only")] + tgt --> sql[("SQL Server or PostgreSQL")] + cp --> sql + tgt --> bod[("Body store")] +``` + +`MigrationEngine` is store-agnostic. It drives `IMigrationSource` and `IMigrationTarget` per category and treats the cursor as an opaque string that it hands from source to target. Progress is persisted per category as a checkpoint row in the target database, committed in the same transaction as the rows it describes. Every command except `--migration-source-report` reads the checkpoint table. + +## Assemblies + +Dependencies point inward: each persister knows only its own store, the neutral assembly knows neither, and the host is the only composition root that names both. + +| Assembly | Types | Boundary rationale | +| --- | --- | --- | +| `ServiceControl.Persistence` | `MigrationEngine`, `MigrationBodyLoader`. Contracts: `IMigrationSource`, `IMigrationTarget`, `IMigrationCheckpointStore`, `IMigrationStartupCheck`, `IMigrationTargetReadiness`, `IMigrationSourceFactory`. Data: `MigrationCategory`, `MigrationCategoryKind`, `MigrationCategoryIds`, `MigrationCategoryRegistry`, `MigrationBatch`, `MigrationRow`, `MigrationBody`, `MigrationCheckpoint`, `MigrationWriteResult`, `MigrationSkipReason`, `MigrationSourceDescription`, `MigrationSourceFact`. Rules: `MigrationCategoryStateExtensions`, `MigrationSkipReasonExtensions`, `MigrationCheckpointRules`, `HaltThreshold`. Options: `MigrationEngineOptions`, `MigrationSettings`. `MigrationCheckpointConflictException`. Read-only views: `MigrationStatusView`, `MigrationVerification`, `MigrationDryRunArithmetic` | Referenced by both persisters and the host, references neither persister, so the engine is unit-tested against in-memory fakes in `ServiceControl.UnitTests/Migration` | +| `ServiceControl.Persistence.RavenDB` | `RavenMigrationSource`, `RavenReadOnlySourceLifecycle`, `RavenDocumentStream`, `IMigrationCategoryReader`, `IWindowedMigrationCategoryReader`, one reader per category | Owns document id prefixes, sessions and the embedded server | +| `ServiceControl.Persistence.EFCore` | `EFCoreMigrationTarget`, `EFCoreMigrationTargetReadiness`, `IMigrationCategoryWriter`, `IStoreKeyedCategoryWriter`, one writer per category, `PreparedBatch`, `IMigrationSqlDialect`, `MigrationInsert`, `StoreKeyedInsert`, `EFMigrationCheckpointStore`, `MigrationCheckpointExtensions`, `EFCoreMigrationRowAssessor`, and the three target readiness checks | Owns tables, column widths and provider parameter limits | +| `ServiceControl` (host) | `MigrateCommand`, `MigrationBeforeTheHostOpens`, `RecordHostOpenedOnTarget`, `MigrationStartup` with nested `ClosedWindowProgress`, `MigrationStartupCheckRunner`, the four host checks, `OptionalCategoryCopier`, `FinishedCopyBeforeAnIngestionNodeOpens`, `MigrationProgressReporter`, the migration commands with `MigrationOperatorCommands` and `MigrationSourceDescriptionPrinter`, `PersistenceFactory.CreateMigrationSource` and `OpenMigrationSource` | Pair support is the one question neither persister can answer, so it lives here | + +Supporting assemblies: + +- `SqlServerMigrationSqlDialect` and `PostgreSqlMigrationSqlDialect` implement `IMigrationSqlDialect` and are the only provider-specific migration code. SQL Server reads key-column collations in `Open`, builds each key column's `KeyComparer` from the collation's comparison style (case, accent, kana type and width, plus SQL Server's trailing-space padding), and inserts with `MERGE ... WITH (HOLDLOCK)`, de-duplicating the batch with `ROW_NUMBER() OVER (PARTITION BY <key> COLLATE <column collation>)`. PostgreSQL uses `INSERT ... ON CONFLICT DO NOTHING RETURNING`. Both implement `AddLicensingEndpointThroughput`, which groups each statement's rows by key and sums their counts before adding them to the stored day, so the database decides which names are one day. SQL Server groups under the key column's collation; PostgreSQL's deterministic collations make a plain `GROUP BY` the same equality. Each provider assembly carries its own `AddMigrationCheckpoints` EF migration and model snapshot, so a checkpoint schema change needs a migration in both. +- `ServiceControl.DomainEvents` holds `MigrationStarted`, `MigrationCategoryHalted`, `MigrationFinished` and `MigrationSourceUnreachable`. + +The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadiness` or `IMigrationCheckpointStore`, so the guards below resolve `null` on RavenDB and return. `MigrationPairIsSupportedCheck` is what rejects RavenDB as a target. + +## What runs on every SQL instance + +- **Registrations.** `BasePersistence.RegisterDataStores`, reached through each provider's `AddPersistence`, registers `IMigrationCheckpointStore` as `EFMigrationCheckpointStore`, `IMigrationTargetReadiness` as `EFCoreMigrationTargetReadiness` and `IMigrationTarget` as `EFCoreMigrationTarget`, all singletons. +- **`MigrationBeforeTheHostOpens`.** `RunCommand` registers a lifecycle service that checks required checkpoints in `StartingAsync`, before any hosted service starts or Kestrel binds. It delegates to `FinishedCopyBeforeAnIngestionNodeOpens`, treating unknown ids as required. An unfinished required row refuses startup and names `--start-data-migration`. The service never copies data. With no checkpoint store registered (RavenDB), it returns immediately. +- **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race the offline command on the checkpoint version. +- **`RecordHostOpenedOnTarget` on every writer.** The main host, ingestion workers, and SQL imports register `RecordHostOpenedOnTarget` and stamp `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so rollback advice must never offer a free abort after a worker has written. + +The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row, see [open decisions](ravendb-to-sql-migration-overview.md#gaps-nothing-enforces). + +> [!IMPORTANT] +> Stop workers before running `--start-data-migration`. Restart them only after the command succeeds and the main host opens. A worker cannot detect a migration before its first checkpoint exists. + +## The required copy + +Unresolved and retry-issued failed messages copy while the host is closed because the copy cannot merge with live writes. `FailedMessagesWriterBase` inserts only messages the target lacks and never updates one it holds. From the moment the host opens, ingestion writes to the same rows: + +- **A retry confirmation finds no row.** `ResolveRetriedMessages` updates only rows already in SQL, and `FailedMessageBatchWriter.ResolveRetried` deletes the retry claim either way. The copy then inserts the message as still failing, though its retry succeeded. +- **A message that fails again loses its history.** Ingestion upserts a row holding only the new attempt. The copy then finds the id present and drops the RavenDB row, with its earlier attempts, first failure time and retry-issued status. +- **Group actions see part of the list.** Retrying or archiving a group in ServicePulse acts only on the messages copied so far. + +Archived and resolved messages are safe to copy after the host opens. If one of them fails again, the live unresolved row should win, and an insert-only copy gives exactly that. Copying unresolved messages in the background would need a writer that merges by attempt time, and retry confirmations held until their row arrives. Both change live ingestion. + +For developer smoke tests, Debug builds honor `SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED=true` in both `--start-data-migration` and normal startup. Release builds ignore this override. + +```mermaid +sequenceDiagram + participant Run as Operator + participant Copy as MigrateCommand + participant Start as MigrationStartup + participant Checks as MigrationStartupCheckRunner + participant Source as RavenMigrationSource + participant Target as EFCoreMigrationTarget + participant Engine as MigrationEngine + + Run->>Copy: --start-data-migration, with ServiceControl stopped + Copy->>Copy: build persistence-only host; never StartAsync + Copy->>Start: RunRequiredCopy(app.Services, settings) + Start->>Checks: MigrationPairIsSupportedCheck + Start->>Start: PersistenceFactory.CreateMigrationSource, not connected + Start->>Checks: RunChecksAndOpenTarget + Checks->>Target: Open + Start->>Start: IMigrationCheckpointStore.ReadAll + Start->>Source: Open + Start->>Checks: source.ContributedChecks(), empty for RavenDB + Start->>Engine: RunRequiredCategories under ClosedWindowProgress + Engine-->>Start: one checkpoint per category + Start->>Start: ReportWhatTheCopyLeftBehind + Start->>Start: RefuseIfAnyCategoryDidNotComplete + Start-->>Copy: returns, or throws and the command fails + Copy-->>Run: success: start ServiceControl to complete online migration + Run->>Run: start service or container + Note over Run: Startup checks required checkpoints; StartedAsync stamps the target +``` + +Each position in the sequence is deliberate: + +1. **Inside the offline command.** `MigrateCommand` builds a host containing persistence, logging, and the clock. It never calls `StartAsync` or `RunAsync`, so no hosted services run. It calls `MigrationStartup.RunRequiredCopy` directly and prints the instruction to start ServiceControl only after success. Ctrl+C cancels the command; committed batches remain resumable through another `--start-data-migration` invocation. The Windows service is not starting during this work, so its start timeout does not apply. +2. **`MigrationPairIsSupportedCheck` first.** It needs no source, target or options. It resolves `PersistenceType` through `PersistenceManifestLibrary` and checks it against `PersistenceFactory.SqlPersistenceNames`; the source type is fixed at `PersistenceFactory.MigrationSourcePersistenceType`. `--start-data-migration` against a RavenDB target is rejected here. +3. **`PersistenceFactory.CreateMigrationSource`** resolves the source persistence and hard-casts its configuration to `IMigrationSourceFactory`, implemented only by `RavenPersistenceConfiguration`. A non-implementing persister fails with `InvalidCastException`. No I/O happens yet. +4. **`MigrationStartup.RunChecksAndOpenTarget`**, the target half, in order: + - `OptionalCategoryWindowsAreValidCheck` runs `MigrationEngineOptions.FromSettings`, refusing a window that is not a non-negative `TimeSpan` and naming the key. A zero window removes the category from `SelectedOptionalCategoryIds`. + - `RetryHistoryDepthIsSafeCheck` is host-owned because it needs only `Settings.RetryHistoryDepth`, which keeps that value off `PersistenceSettings` and out of `/api/configuration`. + - `EFCoreMigrationTargetReadiness.ContributedChecks`: `SchemaIsCurrentCheck`, `TargetHoldsNoServiceControlDataCheck`, `BodyStorageIsWritableCheck`. The body probe is deleted after writing, so it does not skew body counts. + - `IMigrationTarget.Open`, where the SQL Server dialect loads key-column collations. + - `OptionalCategoryWindowsAreUnchangedCheck`, last because it reads checkpoint rows. It refuses a started, unfinished optional category whose configured window differs from `StartedWindowSeconds`. + + `MigrationStartupCheckRunner` short-circuits on the first refusal. +5. **`TargetHoldsNoServiceControlDataCheck`** passes immediately if a checkpoint row exists for any required category. Otherwise it refuses if any mapped table other than `MigrationCheckpoints` has a row, `Settings` included, naming tables by `GetTableName()` (`endpoint_settings`, `settings` on PostgreSQL). +6. **Source open, last.** It is the only step that can spawn a process. `RunRequiredCopy` reads the checkpoint rows and builds the engine first. `RavenReadOnlySourceLifecycle.Open` then: + - starts the embedded server, if configured; + - connects: on an external server it rejects an expired or not-yet-valid client certificate and an `https://` URL without one, then attaches `RefuseWrite` to the request pipeline before `Initialize`, so the version request is covered; + - checks an external server's version against the client; + - waits for both databases, with a 5-minute budget per database on an embedded server. The budget is only checked when RavenDB raises its own timeout, so the wait can exceed it. + + > [!IMPORTANT] + > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexes`. +7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the command skips the copy and succeeds. The operator can then start ServiceControl; its online copier handles any later source outage independently. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. +8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. +9. **Checkpoint initialization.** The command seeds `NotStarted` rows for required and selected optional categories before copying. `MigrationStartup.RunRequiredCategories` also ensures required rows exist, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. +10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. +11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. +12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row with the explanation `HarmlessSkipExplanations` holds for each harmless reason. A test fails unless that map covers exactly the `IsBenign` reasons. +13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`, and harmless reasons flagged `, harmless` through `IsBenign`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. +14. **`RecordHostOpenedOnTarget`** is registered on normal startup, never on the offline command. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. + +## The background copy + +`OptionalCategoryCopier` is a `BackgroundService` registered by `RunCommand` only on the main instance. It waits for `ApplicationStarted`, then reads checkpoints. With no unfinished optional rows, it exits without creating a source or opening a target. + +- It runs the selected, supported optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. The offline command has already seeded the rows that identify the online work. +- It creates and opens its own source only when a supported category is still copying. On failure it calls `RecordSourceOutage` and exits for this start. It does not depend on in-process state from `--start-data-migration`, which ran in another process. +- An exception leaves the category `InProgress` with `LastError`, untouched until the next start, so a transient failure never forces an abandon. + +Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. + +> [!IMPORTANT] +> Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. The online startup path must validate unchanged windows too, because `--start-data-migration` no longer runs during normal startup. + +`Migration/ThrottlePauseMilliseconds` inserts a delay between optional-category batches, skipping the first batch of each category. Required categories are never throttled. + +## Copying a batch + +`MigrationEngine.RunCategoryAsync` is the per-category loop. Every store-specific step is delegated. + +| Step | Owner | Rationale | +| --- | --- | --- | +| Load the checkpoint | `EFMigrationCheckpointStore.Read` | The table lives in the target database | +| Short-circuit terminal or Failed rows | `MigrationCategoryStateExtensions.IsFinished`, `IsFailed` | Single definition shared by every caller; the source is not read | +| Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | +| Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | +| Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor<TEntity>` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; `FailedMessagesReader` streams a query on `FailedMessageViewIndex` ordered by `id()`, after waiting for the index to be up to date, and falls back to `ByPrefix` when the index is errored, disabled or holds fewer failed messages with a copied status than the collection does; the two licensing endpoint readers use `ThroughputDocuments.Endpoints`; single-document readers load by id | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen. Both unresolved paths resume on the same document id cursor | +| Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyMissing` after one attempt, a permanent fault no retry fixes, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | +| Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold. A crash leaves orphan bodies, see [open decisions](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch) | +| Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `AddLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips, merges and changes are logged after commit. `MigrationWriteResult.Merged` and `Changed` carry the batch's counts of each, which the engine adds up per category for the current run in `MigrationEngine.MergedAndChangedThisRun`; the checkpoint stores neither. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | +| Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | +| Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | +| Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | + +Failed message body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageDocumentTranslator.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. `FailedErrorImportsWriter` places a failed error import's body by the SQL import store's own rule, inline up to `MaxBodySizeToStore` and otherwise external only, under `FailedErrorImportEntity.ExternalBodyId`, also before the transaction opens. + +Invariants enforced across class boundaries: + +- `MigrationCheckpoint.Extend` throws unless the per-`MigrationSkipReason` counts sum exactly to the skipped count. `EFCoreMigrationTarget.AccountForEveryRow` throws unless prepared rows plus skips equal the batch size, catching rows dropped during preparation. `AlreadyPresentIn` throws when a writer over-reports. Every row is copied, skipped or already present. +- `MigrationSkipReasonExtensions.IsBenign` is fixed: `PastRetention`, `BlankGroupComment`, `SourceStorageCheck`. Everything else is a fault, `Unknown` included. +- `PreparedBatch.Merges` reports rows whose key collides with another under the target's collation. Writers with a text key detect them with `KeyMerges.In` or `KeyMerges.Split`, which compare with `IMigrationSqlDialect.KeyComparer` against the batch and the existing table. `FailedErrorImportsWriter`, whose key is a GUID, reports a second import of the same message in the batch, and one the table holds with another failure time. `EFCoreMigrationTarget` logs one warning per merge naming both keys, the source row kept, or the stored row, and the source row dropped. The dropped row counts as already present, and also in `MigrationWriteResult.Merged`, so the category summary `MigrationStartup.ReportWhatTheCopyLeftBehind` logs shows merges apart from the rest of already present. Every writer whose key can fold two source rows into one reports them here, or the summary understates merges. +- `PreparedBatch.Changes` reports rows the insert adds whose stored value differs from the source's on purpose: a failed message's cut message type, dropped or blank failure groups, and failure groups one key folds together; a failed error import's missing exception or body stored empty; and a group comment on a group with no copied failed message, which the retention sweep deletes. A failed message keeping only its newest processing attempt is not a change, because every row does that by design. Writers report only rows the target does not already hold, so a re-read reports nothing twice. `EFCoreMigrationTarget` logs one warning per change after the commit, and `MigrationWriteResult.Changed` counts them. A process that stops between the commit and the log loses that batch's lines for good. `MigrationStartup.ReportWhatTheCopyLeftBehind` warns, naming both counts, when a category's merged count exceeds its already-present count, because the key comparer then disagrees with the database. `PreparedBatch.RenumberedRows` names rows the database gives new ids, and the target logs it once per copy of a category, on the batch whose checkpoint has copied nothing yet. +- `TargetCountAtSettle` is read with `IMigrationTarget.Count` just before the save that settles a category at the end of the source, or that abandons it, and is never updated afterwards. For a required category it is exactly what the copy left in SQL: the host is closed, `TargetHoldsNoServiceControlDataCheck` started the target empty, and a second copier would move the row's version so the settle save is refused. That is why the count needs no shared transaction with the save. For an optional category it also includes what ServiceControl wrote while the copy ran. + +## Category states + +`MigrationCategoryState` has seven stored values, surfaced as four operator states: + +| Operator state | Stored | Meaning | +| --- | --- | --- | +| Copying | `NotStarted`, `InProgress`, `Blocked` | Not terminal, including a follower waiting on its predecessor and an optional category stopped by an error the next start retries | +| Done | `Complete` | Every row copied or benignly skipped, counts reconciled | +| Failed | `Halted`, `CompleteWithErrors` | Stays Failed until `--migration-retry` or `--migration-abandon` | +| Abandoned | `Abandoned` | Terminal, set by the operator directly or by cascade | + +The required-copy refusal prints both forms for a Failed row (`EndpointSettings is Failed (Halted)`) and the stored state alone for a Copying row. Retry and abandon refusals print the operator state only. + +Early stops during a run: + +- Threshold exceeded or post-commit mismatch: `Halted`, either kind. +- Required category exception or 30-minute stall: `Halted`. +- Optional category exception: stays `InProgress` with `LastError`. + +Settlement at end of source, in order: + +1. Rows read this run differ from rows processed: `Halted`. +2. Any fault skip in the checkpoint: `CompleteWithErrors`. +3. Otherwise: `Complete`. + +`MigrationCategoryStateExtensions.IsFinished` (`Complete` or `Abandoned`) and `IsFailed` (`Halted` or `CompleteWithErrors`) are the single definitions, used by the engine, the required-copy refusal, the background copier, the end-of-migration guard, the ingestion gate, status, verify and both operator commands. The host opens when every required category `IsFinished`; the migration is finished when every category is. + +`MigrationCheckpointRules` holds the rules shared by the guard, the commands and status: + +- `IsOutstanding`: not finished, except an optional `NotStarted` row whose window is `0`. +- `CanBeRetried`: an `IsFailed` row, or an optional `NotStarted` or `InProgress` row with `LastError` (which covers `RecordSourceOutage` rows). Never `Blocked` or a missing row. `MigrationOperatorCommands.Retry` rejects `EventLog` before consulting it. +- `CanBeAbandoned`: a required row that `IsFailed` or is `InProgress` with copied plus skipped plus already-present above 0, treating an id unknown to the registry as required; an optional category in any state except `Complete` or `Abandoned`, including no row. + +Checkpoint schema: one row per category in `MigrationCheckpoints` (`migration_checkpoints` on PostgreSQL), primary key `CategoryId`. Columns: `State`, `Cursor`, `CopiedCount`, `SkippedCount`, `AlreadyPresentCount`, `SkipReasons` (JSON keyed by enum name; unknown names deserialize to `Unknown`, a fault), `StartedAt`, `LastProgressAt`, `SettledAt`, `LastError`, `StartedWindowSeconds`, `TargetCountAtSettle` (null until the category first settles) and `Version`, the optimistic concurrency token. No attempt counter and no decision record. `AddMigrationCheckpoints` creates the table in both providers and has not been released, so `TargetCountAtSettle` replaces the never-written `SourceTotal` column inside it rather than in a second migration. This build does not store merged or changed counts: `MigrationEngine.MergedAndChangedThisRun` holds them for the current run only, so the summary of a resumed category counts only that run's. Phase 13 adds them to the checkpoint so that verify can show them. Run-level facts are `Settings` rows under `Migration/`, such as `Migration/HostOpenedOnTarget`. + +## Commands + +Every migration command except the source report builds a host with `AddPersistence` and never calls `StartAsync`, so no hosted service runs, including the retention sweeper and the integration event dispatcher. The source report builds no host and calls `PersistenceFactory.OpenMigrationSource` directly. + +| Command | Class | Opens RavenDB | Reads and writes | +| --- | --- | --- | --- | +| `--start-data-migration` | `MigrateCommand` | Yes | Runs the required-copy checks and engine, seeds selected optional checkpoints, and exits. On success it instructs the operator to start ServiceControl; it never starts the service itself | +| `--migration-source-report` | `MigrationSourceReportCommand` | Yes | `IMigrationSource.Describe` and `Inventory` through `MigrationSourceDescriptionPrinter`; the copy never calls either. Each `MigrationSourceFact` carries its originating setting key. `Inventory` covers every collection, including uncopied ones | +| `--migration-dry-run` | `MigrationDryRunCommand`, `MigrationDryRunArithmetic` | Yes | `MigrationPairIsSupportedCheck`, then `MigrationStartup.RunChecksAndOpen` (both halves), so it refuses wherever a real start would. Reads `Count`, `Read`, `ReadBody`, `ReadBodyVolume`, `Inventory`, `Describe`, `IMigrationTarget.Assess` (via `EFCoreMigrationRowAssessor`) and `BatchSizeFor`. Writes no row; the body probe is the only write. Exit code 0 even when the report shows problems; a refused check throws | +| `--migration-status` | `MigrationStatusCommand`, `MigrationStatusView` | No | `IMigrationCheckpointStore.ReadAll` and `MigrationEngineOptions.FromSettings`. Read-only, safe while ServiceControl runs | +| `--migration-verify` | `MigrationVerifyCommand`, `MigrationVerification` | Yes | `ReadAll`, `FromSettings`, then `IMigrationSource.Count` per selected category, judged against the rows the copy read (below). A windowed category is counted from `CopiesFrom` set to `StartedAt` minus `StartedWindowSeconds`, the window the copy used, on the source side only. Never calls `IMigrationTarget.Count`. Streams every selected category, though not the bodies, so on an embedded source ServiceControl is stopped first. Exit code 1 only when a category `IsOutstanding`; rows never read on a Done category are reported, not failed | +| `--migration-retry <category>` | `MigrationRetryCommand`, `MigrationOperatorCommands` | No | `Retry` checks `CanBeRetried` and builds the reset row; `IMigrationTarget.ResetForRetry` persists it | +| `--migration-abandon <category>` | `MigrationAbandonCommand`, `MigrationOperatorCommands` | No | Checks `CanBeAbandoned`, then writes `Abandoned` rows through the checkpoint store only | + +`MigrationOperatorCommands.Retry` builds the reset row without I/O: state `NotStarted`; `Cursor`, the four counts, `SkipReasons`, `SettledAt`, `TargetCountAtSettle` and `LastError` cleared; `StartedAt` and `StartedWindowSeconds` kept. Change retry semantics there. `ResetForRetry` only persists the row it receives, in one transaction under the execution strategy. A category whose rows a re-read cannot recognise as already copied cannot be re-read idempotently: store-keyed rows come back under new keys, and throughput counts are added again. Its writer implements `IStoreKeyedCategoryWriter.DeleteCopiedRows` and the reset deletes its rows in the same transaction: + +- `RetryOperationsWriter`: `HistoricRetryOperations`, then `UnacknowledgedRetryOperations`. +- `PendingIntegrationEventsWriter`: `ExternalIntegrationDispatchRequests`. +- `LicensingThroughputWriter`: `LicensingEndpointThroughput`. + +All three are required categories, and `CanBeRetried` takes a required category only when it is Failed, which keeps the host closed, so the host has not opened and the target started empty. `ResetForRetry` also checks this itself: inside the reset transaction it reads the `Migration/HostOpenedOnTarget` setting, and when it is set it refuses a reset that would delete, naming the category, and deletes and saves nothing. A reset that deletes nothing, such as an optional category's, is not refused. `EventLog` is also store-keyed but copies after the host opens, when its rows are indistinguishable from live ones, so it is abandon-only. + +`--migration-abandon` first re-saves the named row unchanged, when it exists, to bump its version and fence off a concurrent copy. It then abandons dependants along `MigrationCategory.MustFollowIsDataLink` (`EndpointSettings` after `KnownEndpoints`, `LicensingThroughput` after `LicensingEndpoints`), deepest first, skipping `Complete` and `Abandoned` dependants and inserting `Abandoned` rows for dependants without one, and saves the named row last. `GroupComments` follows the unresolved failed messages for ordering only and is not cascaded. An optional category with no row gets a new `Abandoned` row. Every row it saves `Abandoned` carries `TargetCountAtSettle`, read just before the save, as a settle does. No domain event is raised, because ServiceControl is stopped. + +Both operator commands rely on `MigrationCheckpointConflictException` to detect a race with a running copy. + +`MigrationVerification` judges one thing per category: RavenDB now against the rows the copy read, `CopiedCount` plus `SkippedCount` plus `AlreadyPresentCount`. That sum is exactly the rows the last complete pass read, body skips included: `RunCategoryAsync` settles `Halted` when the rows a run read differ from the rows the target accounted for, a crash resume adds to the same totals, and a retry clears them. Once the source opens RavenDB only shrinks, because `RefuseWrite` refuses every client write and only RavenDB's expiry deletes. + +- **RavenDB higher than the sum** is rows the copy never read. Verify flags them, by category. +- **RavenDB lower than the sum** is rows RavenDB removed after the copy read them, normal for the event log, the archive and a few unresolved messages with a stale expiry. Printed, not judged. +- **Expiry can hide the same number of unread rows** in the categories it touches, so a flagged number is a floor. +- **The count streams the same reader the copy used** (`RavenMigrationSource.Count`), so verify finds rows one run missed, such as a resume that skipped a range or a stream that ended early. It cannot find a reader that always misses the same rows, or a source pointed at the wrong database. +- **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `TargetCountAtSettle` is printed beside the sum instead. Merged rows are printed there only once phase 13 stores their count on the checkpoint; until then they appear only in the log of the run that made them. +- **An abandoned category's sum covers its last pass only.** Rows an earlier pass copied before a retry are in `TargetCountAtSettle` but not in the sum, so RavenDB minus the sum overstates what was never copied. + +For rows never read on a Done category, verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, and neither operator command takes a Done row. Whether to fix them is an [open decision](ravendb-to-sql-migration-overview.md#what-the-checks-do-not-catch). + +## Readers and writers + +Readers live at `ServiceControl.Persistence.RavenDB/DataMigration/Readers/<Category>Reader.cs` and are keyed in `RavenMigrationSource`'s reader dictionary. Writers live at `ServiceControl.Persistence.EFCore/DataMigration/Writers/<Category>Writer.cs` and are keyed in the `FrozenDictionary` initialised in `EFCoreMigrationTarget`. `MigrationCategoryCoverageTests` asserts that every registry category has both. + +Every reader derives from `MigrationCategoryReader<TDocument>` and every writer from `MigrationCategoryWriter<TDocument>`, and the two for one category take the same `TDocument`, the type of each row's `Document`. The reader base streams whole documents from the primary database through `WholeDocuments`, and exposes `Lifecycle` to a reader that projects or reads the throughput database itself. The writer base casts every row once and hands `PrepareDocuments` the typed documents, so a row of another type fails the batch with an `InvalidCastException` naming the category, the row and both types. Four tests hold this: `MigrationCategoryCoverageTests.Every_reader_and_its_writer_agree_on_the_document_type`, `MigrationCategoryReadersTests.Every_reader_derives_from_the_typed_base`, `MigrationCategoryWritersTests.Every_writer_derives_from_the_typed_base` and `MigrationCategoryWritersTests.Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type`. + +| Order | Category | Kind | Reader, writer | Notes | +| --- | --- | --- | --- | --- | +| 1 | `KnownEndpoints` | Required | `KnownEndpointsReader`, `KnownEndpointsWriter` | | +| 2 | `EndpointSettings` | Required | `EndpointSettingsReader`, `EndpointSettingsWriter` | Data link to `KnownEndpoints`. Copies every setting, including one for an unknown endpoint, which `HeartbeatEndpointSettingsSyncHostedService` removes once the host opens. Reports case merges on a case-insensitive name column | +| 3 | `MessageRedirects` | Required | `MessageRedirectsReader`, `MessageRedirectsWriter` | | +| 4 | `Subscriptions` | Required | `SubscriptionsReader`, `SubscriptionsWriter` | Rows differing only in message-type version merge | +| 5 | `NotificationSettings` | Required | `NotificationSettingsReader`, `NotificationSettingsWriter` | One `Settings` row via `SettingRowMapper` | +| 6 | `TrialEndDate` | Required | `TrialEndDateReader`, `TrialEndDateWriter` | One `Settings` row via `SettingRowMapper` | +| 7 | `RetryOperations` | Required | `RetryOperationsReader`, `RetryOperationsWriter` | Two tables: historic rows store-keyed via `StoreKeyedInsert`, unacknowledged rows via `InsertMissing`, which sizes the batch. Deletes its rows on retry | +| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database. Reads only the `EndpointDocuments` collection, through `ThroughputDocuments.Endpoint`, because the throughput database gives its documents no id prefix and also holds the settings documents. With no prefix RavenDB streams in reverse etag order and ignores `startAfter`, so `ThroughputDocuments.Endpoints` reads every endpoint, sorts the ids in ordinal order and skips past the cursor itself. Reports through `KeyMerges.In` each endpoint folded into another's key. `LicensingThroughputWriter` reports no merge for the days, because the fold is reported here | +| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. The reader gives one batch per endpoint from `ThroughputDocuments.Endpoints` with a batch size of one, and ignores the requested batch size, because one endpoint can hold a year or more of days. Each batch's cursor is its endpoint's id. The writer keeps the last row of a day repeated for one source endpoint, compared ordinally. `AddLicensingEndpointThroughput` splits the batch into statements and adds each day's count to the stored one, summing endpoints whose names the key column treats as the same, in one batch or across batches. A re-read is not idempotent, so a resume relies on each batch committing with its checkpoint, and a retry deletes the copied rows first | +| 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | +| 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database. One `Settings` row via `SettingRowMapper` | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageDocumentTranslator`. `FailedMessageStream.ResolveRetryClaims` looks up each batch's RavenDB retry claims in one request and gives every retry-issued row the `RetryClaimHeld` metadata key. The writer writes a claim only for a retry-issued message it inserts whose flag is true, so the claims mirror RavenDB's, and fails the batch when the flag is missing or not a bool | +| 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | +| 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | The RavenDB reader flags a check whose id and category match one of the RavenDB persister's own six checks with the `SourceStorageCheck` metadata key, and the writer skips a flagged row as `SourceStorageCheck` without knowing any id. A RavenDB audit instance's dirty memory result reports under another category, so it is copied | +| 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | +| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment`. Reports group ids that differ only in trailing spaces as merges on SQL Server. Reports each copied comment on a group with no copied failed message as a change, because the retention sweep deletes it | +| 1 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | +| 2 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | + +## Progress reporting + +There is no migration custom check: custom checks are being removed from ServiceControl, so progress uses the activity feed, the log and `--migration-status` only. + +- `MigrationProgressReporter` raises `MigrationStarted`, `MigrationCategoryHalted` (for both Failed states), `MigrationFinished` (carrying `CategoriesAbandoned`) and `MigrationSourceUnreachable`, mapped into the ServicePulse activity feed by their `EventLogMappingDefinition` classes. It logs and swallows publish failures but propagates cancellation. All four are raised from `OptionalCategoryCopier`, so they cover the background copy only, and a Failed required category produces no activity-feed entry; its record is the startup refusal and the log. +- `MigrationSourceUnreachable` is raised once per start when `OptionalCategoryCopier` idles on a source outage, its own failed open or one the required start recorded, carrying the error and the optional categories left waiting. +- Completed checkpoints stop the online copier automatically. No enable setting or final configuration edit is needed. +- `ClosedWindowProgress` logs running required categories every 30 seconds while the host is closed. +- `--migration-status` reports state, progress, last error and available commands per category at any time, and whether migration can be turned off. diff --git a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj index 718b22bc78..a55fcba55b 100644 --- a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj +++ b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj @@ -12,6 +12,9 @@ <ItemGroup> <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" /> <ProjectReference Include="..\ServiceControl.Persistence.EFCore.PostgreSql\ServiceControl.Persistence.EFCore.PostgreSql.csproj" /> + <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. --> + <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" /> + <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" /> <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" /> <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" /> <ProjectReference Include="..\TestHelper\TestHelper.csproj" /> @@ -20,6 +23,7 @@ <ItemGroup> <PackageReference Include="GitHubActionsTestLogger" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" /> + <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" /> <PackageReference Include="Microsoft.NET.Test.Sdk" /> <PackageReference Include="NServiceBus.Heartbeat" /> <PackageReference Include="NServiceBus.SagaAudit" /> @@ -31,6 +35,9 @@ <ItemGroup> <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" /> + <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" /> + <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" /> + <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\PostgreSqlSharedContainer.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\TestSchema.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\StopSharedPostgreSql.cs" /> diff --git a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj index cbc66d5b4a..0d67353174 100644 --- a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj +++ b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj @@ -12,6 +12,9 @@ <ItemGroup> <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" /> <ProjectReference Include="..\ServiceControl.Persistence.EFCore.SqlServer\ServiceControl.Persistence.EFCore.SqlServer.csproj" /> + <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. --> + <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" /> + <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" /> <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" /> <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" /> <ProjectReference Include="..\TestHelper\TestHelper.csproj" /> @@ -20,6 +23,7 @@ <ItemGroup> <PackageReference Include="GitHubActionsTestLogger" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" /> + <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" /> <PackageReference Include="Microsoft.NET.Test.Sdk" /> <PackageReference Include="NServiceBus.Heartbeat" /> <PackageReference Include="NServiceBus.SagaAudit" /> @@ -31,6 +35,9 @@ <ItemGroup> <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" /> + <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" /> + <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" /> + <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\SqlServerSharedContainer.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\TestSchema.cs" /> <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\StopSharedSqlServer.cs" /> diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs index 58cef8074f..d7dde350e4 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs @@ -29,10 +29,9 @@ namespace ServiceControl.AcceptanceTests.Recoverability using ServiceControl.Infrastructure; using ServiceControl.MessageFailures; using ServiceControl.Operations; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; - using ServiceControl.Persistence.EFCore.Infrastructure; - using ServiceControl.Recoverability; using ServiceControl.Transports; class When_hosting_error_ingestion_only : AcceptanceTest @@ -98,9 +97,37 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured() "InternalCustomChecksHostedService", // reports this node's ingestion health to the database "MetricsReporterHostedService", "HealthCheckPublisherHostedService", // inert, no IHealthCheckPublisher is registered - "ExternalIntegrationRequestsDataStore" // its drain is inert here, nothing calls Subscribe + "ExternalIntegrationRequestsDataStore", // its drain is inert here, nothing calls Subscribe + "FinishedCopyBeforeAnIngestionNodeOpens", // keeps this node out of a database a copy has not finished filling + "RecordHostOpenedOnTarget" // idempotent upsert of a marker only read for presence, safe on every node ]; + [Test] + public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished() + { + var settings = await CreateSettings(); + + await new SetupCommand().Execute(new HostArguments([]), settings); + + var host = ErrorIngestionOnlyCommand.BuildHost(settings); + + try + { + await host.Services.GetRequiredService<IMigrationCheckpointStore>().Upsert( + new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, "The source stopped responding.")); + + var exception = Assert.ThrowsAsync<Exception>(() => host.StartAsync()); + + Assert.That(exception.Message, Does.Contain("has not finished") + .And.Contain(MigrationCategoryIds.KnownEndpoints) + .And.Contain("--migration-abandon KnownEndpoints")); + } + finally + { + await host.DisposeAsync(); + } + } + [Test] public void Should_refuse_to_start_against_unsupported_storage() { @@ -205,6 +232,7 @@ public async Task Should_serve_health_over_https_with_the_configured_certificate await new SetupCommand().Execute(new HostArguments([]), settings); host = ErrorIngestionOnlyCommand.BuildHost(settings); + host.Urls.Clear(); host.Urls.Add("https://127.0.0.1:0"); await host.StartAsync(); diff --git a/src/ServiceControl.Migration.AcceptanceTests/.editorconfig b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig new file mode 100644 index 0000000000..da44eb13fb --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig @@ -0,0 +1,6 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0013.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs new file mode 100644 index 0000000000..920c6e2cce --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/CustomCheckMigrationTests.cs @@ -0,0 +1,97 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Infrastructure; + +[TestFixture] +[NonParallelizable] +class CustomCheckMigrationTests : MigrationAcceptanceTest +{ + static readonly EndpointDetails Endpoint = new() { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + + static Guid IdOf(EndpointDetails endpoint, string customCheckId) => + new CustomCheckDetail { OriginatingEndpoint = endpoint, CustomCheckId = customCheckId, Category = "Ping" }.GetDeterministicId(); + + async Task<IList<CustomCheckView>> CopiedChecks() => (await CustomChecks.GetStats(new PagingInfo())).Results!; + + // One of each status, because Status.Fail is the enum's zero and a status that never deserializes would read as Fail. + [Test] + public async Task Copies_a_failing_and_a_passing_custom_check_with_their_deterministic_ids_and_reported_status() + { + await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await SeedSourceCustomCheck(Endpoint, "Sales.DiskCheck", "Disk", failed: false); + + var result = await CopyCategory("CustomChecks"); + + var copied = await CopiedChecks(); + var failing = copied.Single(check => check.CustomCheckId == "Sales.HealthCheck"); + var passing = copied.Single(check => check.CustomCheckId == "Sales.DiskCheck"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!), Is.EqualTo(2), "Count is what --migration-verify reads"); + Assert.That(Guid.Parse(failing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.HealthCheck")), "a live report of the same check upserts on this id, so it replaces the copied row rather than adding a second"); + Assert.That(failing.Status, Is.EqualTo(Status.Fail)); + Assert.That(failing.Category, Is.EqualTo("Ping")); + Assert.That(failing.FailureReason, Is.EqualTo("timed out")); + Assert.That(failing.ReportedAt, Is.EqualTo(SeededReportedAt)); + Assert.That((failing.OriginatingEndpoint!.Name, failing.OriginatingEndpoint.Host, failing.OriginatingEndpoint.HostId), Is.EqualTo((Endpoint.Name, Endpoint.Host, Endpoint.HostId))); + Assert.That(Guid.Parse(passing.Id!), Is.EqualTo(IdOf(Endpoint, "Sales.DiskCheck"))); + Assert.That(passing.Status, Is.EqualTo(Status.Pass), "a status read back as the enum's zero would show every copied check red"); + Assert.That(passing.Category, Is.EqualTo("Disk")); + Assert.That(passing.FailureReason, Is.Null); + } + } + + [Test] + public async Task A_check_with_no_originating_endpoint_is_skipped_as_a_required_value_missing() + { + await SeedSource(($"CustomChecks/{Guid.NewGuid()}", new { CustomCheckId = "Sales.HealthCheck", Category = "Ping", Status = (int)Status.Pass, ReportedAt = DateTime.UtcNow })); + + var result = await CopyCategory("CustomChecks"); + var checkpoint = await ReadCheckpoint("CustomChecks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + Assert.That(checkpoint.CopiedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons?.GetValueOrDefault(MigrationSkipReason.RequiredValueMissing), Is.EqualTo(1), "CustomChecks.OriginatingEndpointName is NOT NULL, and a throw would stop the batch rather than count the row"); + } + } + + [Test] + public async Task Only_a_check_whose_id_and_category_match_a_RavenDB_persister_check_is_skipped_as_harmless_and_the_category_ends_done() + { + var serviceControl = new EndpointDetails { Name = "Particular.ServiceControl", Host = "MACHINE", HostId = Guid.NewGuid() }; + var audit = new EndpointDetails { Name = "Particular.ServiceControl.Audit", Host = "MACHINE", HostId = Guid.NewGuid() }; + await SeedSourceCustomCheck(serviceControl, "Error Database Index Lag", "ServiceControl Health", failed: true, reason: "index lag"); + await SeedSourceCustomCheck(audit, "Audit Database Index Lag", "ServiceControl.Audit Health", failed: true, reason: "index lag"); + await SeedSourceCustomCheck(audit, "RavenDB dirty memory", "ServiceControl.Audit Health", failed: true, reason: "dirty memory"); + await SeedSourceCustomCheck(Endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await SeedSourceCustomCheck(Endpoint, "Message Ingestion Process", "Sales", failed: true, reason: "stuck"); + + var result = await CopyCategory("CustomChecks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a harmless skip leaves a required category Done, so ServiceControl can open"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.SourceStorageCheck] = 1 }), "only the primary's index lag check is the RavenDB persister's own"); + Assert.That((await CopiedChecks()).Select(check => (check.CustomCheckId, check.Category)), Is.EquivalentTo(new[] + { + ("Audit Database Index Lag", "ServiceControl.Audit Health"), + ("RavenDB dirty memory", "ServiceControl.Audit Health"), + ("Sales.HealthCheck", "Ping"), + ("Message Ingestion Process", "Sales") + }), "the skip matches the id and the category, so the audit instance's checks and a customer check with a colliding id keep their failing results"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs new file mode 100644 index 0000000000..cef7c85604 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FailedErrorImportMigrationTests.cs @@ -0,0 +1,162 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[NonParallelizable] +class FailedErrorImportMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_a_small_body_inline_with_the_time_RavenDB_stored_the_import_as_its_failure_time() + { + await SeedSourceFailedErrorImport("native-1", [1, 2, 3], "Deserialization failed"); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + var import = await GetImport("native-1"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await Target.Count(Category), Is.EqualTo(1), "Count is what --migration-verify reads"); + Assert.That(import.Body, Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That(import.BodyStoredExternally, Is.False); + Assert.That(import.ExceptionInfo, Is.EqualTo("Deserialization failed")); + Assert.That(import.FailedAt, Is.EqualTo(await SourceLastModified("native-1")).Within(TimeSpan.FromMilliseconds(1)), "replay runs oldest first by this time, so the time of the copy would reorder a customer's stuck imports"); + Assert.That(RecordedBodies.Written, Is.Empty); + } + } + + [Test] + public async Task Migrated_bodies_replay_through_the_products_own_import_path() + { + byte[] small = [1, 2, 3]; + var large = LargeBody(); + await SeedSourceFailedErrorImport("native-3", small, "Deserialization failed"); + await SeedSourceFailedErrorImport("native-4", large, "Deserialization failed"); + + await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + var external = await GetImport("native-4"); + Assert.That(external.BodyStoredExternally, Is.True, "otherwise this proves nothing about the external placement"); + Assert.That(external.Body, Is.Empty, "the product keeps no inline copy of a body it stores in body storage"); + + var replayed = new Dictionary<string, FailedTransportMessage>(); + await FailedImportStore.ProcessFailedErrorImports((message, _) => + { + replayed[message.Id] = message; + return Task.CompletedTask; + }, TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(replayed.GetValueOrDefault("native-3")?.Body, Is.EqualTo(small), "an inline body is read from the row"); + Assert.That(replayed.GetValueOrDefault("native-4")?.Body, Is.EqualTo(large), "an external body is read from body storage under the body id the SQL import store uses for that key"); + Assert.That(replayed.GetValueOrDefault("native-3")?.Headers, Is.EqualTo(HeadersFor("native-3")), "the replay ingests the message under the headers the copy stored"); + Assert.That(replayed.GetValueOrDefault("native-4")?.Headers, Is.EqualTo(HeadersFor("native-4"))); + } + } + + [Test] + public async Task A_retry_finds_every_import_already_present_and_writes_no_body_a_second_time() + { + await SeedSourceFailedErrorImport("native-2", [9], "Deserialization failed"); + await SeedSourceFailedErrorImport("native-7", LargeBody(), "Deserialization failed"); + await CopyCategory(MigrationCategoryIds.FailedErrorImports); + var bodiesWritten = RecordedBodies.Written.Count; + + await Target.ResetForRetry(RetryResetMigrationTests.ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports))); + var second = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(bodiesWritten, Is.EqualTo(1), "the large body is the only one the first copy writes to body storage"); + Assert.That(second.AlreadyPresent, Is.EqualTo(2), "--migration-retry re-reads from the start, and the rows the first copy wrote are left alone"); + Assert.That(await Target.Count(Category), Is.EqualTo(2)); + Assert.That(RecordedBodies.Written, Has.Count.EqualTo(bodiesWritten), "a body is written only for an import the target does not hold yet"); + } + } + + [Test] + public async Task Only_the_first_import_of_a_key_in_a_batch_has_its_body_written() + { + var first = LargeBody(); + var second = LargeBody(); + await SeedSource((DocumentId("native-8-a"), ImportOf("native-8", first)), (DocumentId("native-8-b"), ImportOf("native-8", second))); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "both documents derive one key, and the insert keeps the first row of a key"); + Assert.That(RecordedBodies.Written.Select(written => written.Body), Is.EqualTo(new[] { first }), "a second body under the same id would replace the body of the row the insert kept"); + } + } + + [Test] + public async Task An_import_with_no_message_is_skipped_as_a_required_value_missing() + { + await SeedSource((DocumentId("native-5"), new FailedErrorImport { Id = "native-5", Message = null, ExceptionInfo = "Deserialization failed" })); + await SeedSourceFailedErrorImport("native-6", [6], "Deserialization failed"); + + var result = await CopyCategory(MigrationCategoryIds.FailedErrorImports); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + + var checkpoint = await ReadCheckpoint(MigrationCategoryIds.FailedErrorImports); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 }), + "FailedErrorImports.MessageId is NOT NULL; the source attaches an empty body, so nothing is skipped as BodyMissing or BodyUnreadable"); + } + } + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + static string DocumentId(string nativeMessageId) => $"FailedErrorImports/{nativeMessageId}"; + + static Dictionary<string, string> HeadersFor(string nativeMessageId) => new() { ["NServiceBus.MessageId"] = nativeMessageId }; + + // One byte over the limit, because the store moves a body out only when it is larger than the limit. + byte[] LargeBody() + { + var body = new byte[EFSettings.BodyStorage.MaxBodySizeToStore + 1]; + Random.Shared.NextBytes(body); + return body; + } + + Task SeedSourceFailedErrorImport(string nativeMessageId, byte[] body, string exceptionInfo) => + SeedSource((DocumentId(nativeMessageId), ImportOf(nativeMessageId, body, exceptionInfo))); + + static FailedErrorImport ImportOf(string nativeMessageId, byte[] body, string exceptionInfo = "Deserialization failed") => new() + { + Id = nativeMessageId, + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = HeadersFor(nativeMessageId), Body = body }, + ExceptionInfo = exceptionInfo + }; + + async Task<DateTime> SourceLastModified(string nativeMessageId) + { + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + + return session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedErrorImport>(DocumentId(nativeMessageId)))!.Value.ToUniversalTime(); + } + + Task<FailedErrorImportEntity> GetImport(string nativeMessageId) + { + var key = FailedErrorImport.DeriveKey(HeadersFor(nativeMessageId), nativeMessageId); + + return QueryTarget(dbContext => dbContext.FailedErrorImports.AsNoTracking().SingleAsync(row => row.UniqueMessageId == key)); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs new file mode 100644 index 0000000000..b0ed4e2e08 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FailedMessageBodyMigrationTests.cs @@ -0,0 +1,185 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class FailedMessageBodyMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_small_text_body_stays_inline_and_is_never_written_externally() + { + var failure = new IngestedFailure { ContentType = "text/xml" }; + var content = Encoding.UTF8.GetBytes("<order>1</order>"); + await SeedSourceWithBody(failure.ToFailedMessage(), content, "text/xml"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.EqualTo("<order>1</order>")); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + Assert.That(row.BodyContentType, Is.EqualTo("text/xml")); + Assert.That(RecordedBodies.Written, Is.Empty); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("text/xml")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + [Test] + public async Task A_large_text_body_leaves_a_search_prefix_inline_and_writes_the_whole_body_externally() + { + var failure = new IngestedFailure { ContentType = "application/json" }; + var text = new string('a', EFSettings.BodyStorage.MaxBodySizeToStore * 2); + var content = Encoding.UTF8.GetBytes(text); + await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/json"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Not.Null, "an over-cap text body still keeps a prefix inline so that message search can find it"); + Assert.That(text, Does.StartWith(row.BodyText)); + Assert.That(row.BodyText.Length, Is.EqualTo(EFSettings.BodyStorage.MaxBodySizeToStore)); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + } + + var written = RecordedBodies.Written.Single(); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(written.BodyId, Is.EqualTo(failure.UniqueMessageId.ToString())); + Assert.That(written.Body, Is.EqualTo(content)); + Assert.That(written.ContentType, Is.EqualTo("application/json")); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("application/json")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + [Test] + public async Task A_binary_body_goes_external_with_nothing_left_inline() + { + var failure = new IngestedFailure { ContentType = "application/x-protobuf" }; + var content = new byte[] { 0x08, 0x00, 0x96, 0x01, 0xFF }; + await SeedSourceWithBody(failure.ToFailedMessage(), content, "application/x-protobuf"); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + var written = RecordedBodies.Written.Single(); + var fetched = await BodyStorage.TryFetch(failure.UniqueMessageId.ToString()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.BodyText, Is.Null, "there is nothing in a binary payload for message search to index"); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodySize, Is.EqualTo(content.Length)); + Assert.That(row.BodyContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(written.BodyId, Is.EqualTo(failure.UniqueMessageId.ToString())); + Assert.That(written.Body, Is.EqualTo(content)); + Assert.That(written.ContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(fetched.State, Is.EqualTo(MessageBodyState.Available)); + Assert.That(fetched.Content.ContentType, Is.EqualTo("application/x-protobuf")); + Assert.That(await ReadAll(fetched.Content.Stream), Is.EqualTo(content)); + } + } + + // Covers the missing-attachment path (BodyMissing, one read) only; the retried BodyUnreadable path is covered by the loader unit tests. + [Test] + public async Task A_message_with_no_body_attachment_is_left_out_entirely() + { + var readable = new IngestedFailure(); + var missing = new IngestedFailure(); + + await SeedSourceWithBody(readable.ToFailedMessage(), Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml"); + await SeedSource(($"FailedMessages/{missing.UniqueMessageId}", missing.ToFailedMessage())); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var checkpoint = await ReadCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await FindFailedMessage(readable.UniqueMessageId), Is.Not.Null); + Assert.That(await FindFailedMessage(missing.UniqueMessageId), Is.Null, "a row with no body would dispatch an empty message if anyone retried it"); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Contains.Key(MigrationSkipReason.BodyMissing), "the skip is counted under the reason that tells the operator no retry can fix it"); + Assert.That(checkpoint.SkipReasons, Does.Not.ContainKey(MigrationSkipReason.BodyUnreadable)); + } + } + + [Test] + public async Task A_zero_length_body_is_copied_rather_than_treated_as_missing() + { + var failure = new IngestedFailure(); + await SeedSourceWithBody(failure.ToFailedMessage(), [], "text/plain"); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Skipped, Is.Zero); + Assert.That(row.BodySize, Is.Zero); + Assert.That(row.BodyText, Is.Null, "an empty body is stored nowhere, which is not the same as being stored externally"); + Assert.That(row.BodyStoredExternally, Is.False); + Assert.That(RecordedBodies.Written, Is.Empty); + } + } + + [Test] + public async Task The_copied_body_reads_back_through_the_products_own_body_storage() + { + var inline = new IngestedFailure { ContentType = "text/xml" }; + var external = new IngestedFailure { ContentType = "application/x-protobuf" }; + var inlineContent = Encoding.UTF8.GetBytes("<order>1</order>"); + var externalContent = new byte[] { 0x08, 0x00, 0x96, 0x01, 0xFF }; + + await SeedSourceWithBodies( + (inline.ToFailedMessage(), inlineContent, "text/xml"), + (external.ToFailedMessage(), externalContent, "application/x-protobuf")); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(await Fetch(inline.UniqueMessageId), Is.EqualTo(inlineContent)); + Assert.That(await Fetch(external.UniqueMessageId), Is.EqualTo(externalContent), "the body was stored under an id TryFetch does not resolve"); + } + + async Task<byte[]> Fetch(Guid uniqueMessageId) + { + var result = await BodyStorage.TryFetch(uniqueMessageId.ToString()); + + Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); + + return await ReadAll(result.Content.Stream); + } + + static async Task<byte[]> ReadAll(Stream stream) + { + await using (stream) + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + return buffer.ToArray(); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs b/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs new file mode 100644 index 0000000000..5ceb938368 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/FakeBrokerThroughputQuery.cs @@ -0,0 +1,97 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Transports.BrokerThroughput; + +/// <summary> +/// A broker that reports the daily counts a test stages, only for days on or after the start date the collector asks from, +/// records the start date of every request, and counts how many whole collector passes have read it. +/// </summary> +class FakeBrokerThroughputQuery : IBrokerThroughputQuery +{ + readonly ConcurrentDictionary<string, ConcurrentDictionary<DateOnly, long>> staged = new(); + readonly ConcurrentQueue<(string Queue, DateOnly StartDate)> requests = new(); + int queuesStillReading; + int completedPasses; + + /// <summary> + /// How many times the collector has finished reading every staged queue's days. + /// </summary> + public int CompletedPasses => Volatile.Read(ref completedPasses); + + /// <summary> + /// Every queue the collector asked for days of, with the first day it asked for, in the order it asked. + /// </summary> + public IReadOnlyList<(string Queue, DateOnly StartDate)> Requests => [.. requests]; + + /// <summary> + /// Makes the broker report count messages on the queue for that day, replacing any count staged for the same day. + /// </summary> + public void Stage(string queue, DateOnly day, long count) => staged.GetOrAdd(queue, _ => new())[day] = count; + + public Dictionary<string, string> Data { get; } = []; + + public string MessageTransport => "FakeBroker"; + + public string ScopeType => null; + + public KeyDescriptionPair[] Settings => []; + + public void Initialize(ReadOnlyDictionary<string, string> settings) { } + + public bool HasInitialisationErrors(out string errorMessage) + { + errorMessage = string.Empty; + return false; + } + + public Task<(bool Success, List<string> Errors, string Diagnostics)> TestConnection(CancellationToken cancellationToken = default) => + Task.FromResult((true, new List<string>(), "Fake broker reachable")); + + public async IAsyncEnumerable<IBrokerQueue> GetQueueNames([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var queues = staged.Keys.ToArray(); + Volatile.Write(ref queuesStillReading, queues.Length); + + foreach (var queue in queues) + { + yield return new DefaultBrokerQueue(queue); + } + + await Task.CompletedTask; + } + + public async IAsyncEnumerable<QueueThroughput> GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + requests.Enqueue((brokerQueue.QueueName, startDate)); + + try + { + foreach (var (day, count) in staged[brokerQueue.QueueName].Where(entry => entry.Key >= startDate).OrderBy(entry => entry.Key)) + { + yield return new QueueThroughput { DateUTC = day, TotalThroughput = count }; + } + } + finally + { + // The collector records each day before asking for the next, so the last queue's enumeration ending is the pass ending. + if (Interlocked.Decrement(ref queuesStillReading) == 0) + { + Interlocked.Increment(ref completedPasses); + } + } + + await Task.CompletedTask; + } + + public string SanitizeEndpointName(string endpointName) => endpointName; + + public string SanitizedEndpointNameCleanser(string endpointName) => endpointName; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs new file mode 100644 index 0000000000..0d5d7751b2 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/GroupCommentsMigrationTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class GroupCommentsMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory GroupCommentsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)!; + + [Test] + public async Task Copies_each_comment_onto_its_group_and_an_orphaned_one_too_and_skips_a_null_or_blank_one_as_harmless() + { + var kept = new IngestedFailure(); + var cleared = new IngestedFailure(); + var neverWritten = new IngestedFailure(); + var spaces = new IngestedFailure(); + await SeedSourceWithBodies([.. new[] { kept, cleared, neverWritten, spaces }.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]); + await SeedSourceGroupComments( + (kept.Groups.Single().Id, "Waiting on the payments team"), + (cleared.Groups.Single().Id, ""), + (neverWritten.Groups.Single().Id, null), + (spaces.Groups.Single().Id, " "), + ("group-with-no-copied-message", "Raised with the shipping team")); + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var result = await CopyCategory(MigrationCategoryIds.GroupComments); + var comments = (await GroupsStore.GetUnresolvedGroupsByClassifier(kept.Groups.Single().Type, null, new PagingInfo())).ToDictionary(view => view.Id, view => view.Comment); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "SQL never stores a blank comment, so leaving one behind loses nothing and must not keep ServiceControl closed"); + Assert.That(result.Copied, Is.EqualTo(2), "a comment whose group has no copied message is copied too, and left to the retention sweeper"); + Assert.That(await Target.Count(GroupCommentsCategory), Is.EqualTo(2), "--migration-verify reads this count"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BlankGroupComment] = 3 }), "a harmless skip is still counted, so status and verify can show it"); + Assert.That(comments[kept.Groups.Single().Id], Is.EqualTo("Waiting on the payments team"), "ServicePulse's group list shows the comment on the group it was written for"); + Assert.That(new[] { cleared, neverWritten, spaces }.Select(failure => comments[failure.Groups.Single().Id]), Is.All.Null, "the product shows no comment for a group whose comment was blank"); + } + } + + // The prefix is retyped because this project cannot reference the RavenDB persister that declares it. + Task SeedSourceGroupComments(params (string GroupId, string Comment)[] comments) => + SeedSource([.. comments.Select(comment => ($"GroupComment/{comment.GroupId}", (object)new GroupComment { Id = $"GroupComment/{comment.GroupId}", Comment = comment.Comment }))]); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs new file mode 100644 index 0000000000..6089eb2ea6 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingEndpointsMigrationTests.cs @@ -0,0 +1,127 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingEndpointsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_an_endpoint_and_reads_it_back_through_the_products_own_API() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) + { + SanitizedName = "Sales_Orders", + UserIndicator = "Sales team", + Scope = "Sales", + EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()], + CreatedDate = new DateTime(2026, 3, 4, 5, 6, 7, DateTimeKind.Utc) + }); + + await CopyCategory("LicensingEndpoints"); + + var fromTarget = await TargetLicensingDataStore.GetEndpoint("Sales.Orders", ThroughputSource.Broker); + var reportableEndpoints = await TargetLicensingDataStore.GetAllEndpoints(includePlatformEndpoints: false); + var row = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(fromTarget, Is.Not.Null); + Assert.That(fromTarget!.UserIndicator, Is.EqualTo("Sales team")); + Assert.That(fromTarget.Scope, Is.EqualTo("Sales")); + Assert.That(fromTarget.CreatedDate, Is.EqualTo(new DateTime(2026, 3, 4, 5, 6, 7, DateTimeKind.Utc)), "the decommissioned endpoint sweep removes an endpoint with no throughput once it is a day old, so a lost date makes a new endpoint eligible at once"); + Assert.That(row.SanitizedName, Is.EqualTo("Sales_Orders")); + Assert.That(row.NormalizedSanitizedName, Is.EqualTo("sales_orders"), "GetEndpointThroughputByQueueName matches on this column, so an unnormalised value hides a migrated endpoint's throughput from the licensing report"); + Assert.That(fromTarget.EndpointIndicators, Is.EqualTo(new[] { EndpointIndicator.PlatformEndpoint.ToString() }), "the indicators column holds what EF itself writes: JSON on SQL Server, a text[] on PostgreSQL"); + Assert.That(reportableEndpoints, Is.Empty, "the product filters platform endpoints inside the database, so an indicator value EF cannot query puts a platform endpoint in the licensing report"); + } + } + + [Test] + public async Task The_same_name_under_two_sources_is_two_rows_and_not_a_collision() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputSource(new Endpoint("SALES.ORDERS", ThroughputSource.Audit) { SanitizedName = "sales.orders" }); + + var result = await CopyCategory("LicensingEndpoints"); + + Assert.That(result.Copied, Is.EqualTo(2), "the key is (NormalizedName, ThroughputSource), so the throughput source must survive into the key or the second endpoint is treated as a duplicate of the first"); + Assert.That(result.Skipped, Is.Zero); + } + + [Test] + public async Task An_endpoint_the_target_already_holds_is_already_present_rather_than_skipped() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + + await CopyCategory("LicensingEndpoints"); + await DeleteCheckpoint("LicensingEndpoints"); + var second = await CopyCategory("LicensingEndpoints"); + + Assert.That(second.AlreadyPresent, Is.EqualTo(1)); + Assert.That(second.Skipped, Is.Zero, "an endpoint the target already holds was not lost, and counting it as a skip makes --migration-verify subtract it twice"); + } + + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_never_fail_the_batch() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + var batch = new MigrationBatch( + [ + new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary<string, object>()), + new MigrationRow("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }, new Dictionary<string, object>()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + + var keyIgnoresTrailingSpace = (await QueryTarget(dbContext => Task.FromResult(dbContext.Database.ProviderName))).Contains("SqlServer"); + var storedNames = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().Select(row => row.Name).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space and keeps the first row; PostgreSQL's does not and keeps both"); + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the row SQL Server's key treats as the first's is counted as already present"); + Assert.That(result.Skipped, Is.Zero); + Assert.That(await Target.Count(category), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); + + if (keyIgnoresTrailingSpace) + { + Assert.That(storedNames, Is.EqualTo(new[] { "Sales.Orders " }), "the row whose id sorts first by ordinal order is the one a shared key keeps, whatever order the batch arrives in"); + } + } + } + + [Test] + public async Task An_endpoint_with_no_name_is_skipped_as_a_required_value_missing() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + var batch = new MigrationBatch( + [ + new MigrationRow("/Broker", new Endpoint(new EndpointIdentifier(null!, ThroughputSource.Broker)) { SanitizedName = "nameless" }, new Dictionary<string, object>()), + new MigrationRow("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = null! }, new Dictionary<string, object>()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + var stored = await QueryTarget(dbContext => dbContext.LicensingEndpoints.AsNoTracking().SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.SanitizedName, Is.Empty, "a null sanitized name takes the empty string the product itself stores"); + Assert.That(stored.NormalizedSanitizedName, Is.Empty); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "/Broker" })); + Assert.That(result.SkipReasons![MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "Name and NormalizedName are NOT NULL and the product has no value of its own for a nameless endpoint, so the row is skipped and counted rather than halting the category"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs new file mode 100644 index 0000000000..3655d52306 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingSettingsMigrationTests.cs @@ -0,0 +1,98 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingSettingsMigrationTests : MigrationAcceptanceTest +{ + static LicensedEndpointDetails Details => new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + [Test] + public async Task Copies_report_masks_as_one_row_in_settings() + { + await SeedThroughputReportMasks(["contoso", "fabrikam"]); + + var result = await CopyCategory("LicensingReportMasks"); + + Assert.That(result.Copied, Is.EqualTo(1)); + var masks = await TargetLicensingDataStore.GetReportMasks(); + Assert.That(masks, Is.EquivalentTo(new[] { "contoso", "fabrikam" })); + } + + [Test] + public async Task Copies_the_licensed_endpoint_details_file_as_one_row_in_settings() + { + var seeded = Details; + await SeedThroughputLicensedEndpointDetails(seeded); + + var result = await CopyCategory("LicensedEndpointDetails"); + + Assert.That(result.Copied, Is.EqualTo(1)); + var details = await TargetLicensingDataStore.GetLicensedEndpointDetails(); + Assert.That(JsonSerializer.Serialize(details), Is.EqualTo(JsonSerializer.Serialize(seeded)), "every field, including the endpoint, queue and product arrays, must reach the SQL row"); + } + + [Test] + public async Task Report_masks_an_earlier_pass_copied_are_kept_and_reported_as_already_present() + { + await TargetLicensingDataStore.SaveReportMasks(["copied earlier"]); + await SeedThroughputReportMasks(["contoso"]); + + var result = await CopyCategory("LicensingReportMasks"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(await TargetLicensingDataStore.GetReportMasks(), Is.EquivalentTo(new[] { "copied earlier" }), "a copy inserts a settings row only where the target has none, so a value an earlier pass copied is never overwritten"); + } + } + + [Test] + public async Task Licensed_endpoint_details_an_earlier_pass_copied_are_kept_and_reported_as_already_present() + { + await TargetLicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "copied earlier" }); + await SeedThroughputLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }); + + var result = await CopyCategory("LicensedEndpointDetails"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That((await TargetLicensingDataStore.GetLicensedEndpointDetails())!.LicenseId, Is.EqualTo("copied earlier"), "a copy inserts a settings row only where the target has none, so a value an earlier pass copied is never overwritten"); + } + } + + [Test] + public async Task An_absent_document_writes_no_row() + { + var masks = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingReportMasks"); + var details = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensedEndpointDetails"); + + var masksResult = await CopyCategory(masks.Id); + var detailsResult = await CopyCategory(details.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(masksResult.Copied + detailsResult.Copied, Is.Zero); + Assert.That(await Target.Count(masks), Is.Zero, "an absent RavenDB document hands over nothing, so no empty masking list is invented on the SQL side"); + Assert.That(await Target.Count(details), Is.Zero); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs new file mode 100644 index 0000000000..b6f2048f0c --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/LicensingThroughputMigrationTests.cs @@ -0,0 +1,199 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class LicensingThroughputMigrationTests : MigrationAcceptanceTest +{ + [Test] + public void Writing_counts_before_their_endpoint_is_refused_naming_the_endpoint() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + var day = new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100); + var batch = new MigrationBatch([new MigrationRow("Sales.Orders/Broker|2026-01-01", day, new Dictionary<string, object>())], "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(category, batch, checkpointToExtend)); + + Assert.That(exception!.Message, Does.Contain("Endpoint Sales.Orders from Broker does not exist"), "a missing endpoint is refused by name and source, as the product's own throughput write refuses it, rather than surfacing as a raw foreign key violation"); + } + + [Test] + public async Task An_endpoint_skipped_by_its_own_copy_blocks_the_counts_and_none_are_written() + { + await SeedThroughputSource(new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = new string('s', 451) }); + await SeedThroughputTimeSeries("Billing/Broker", new DateOnly(2026, 1, 1), 50); + + await CopyCategory("LicensingEndpoints"); + Assert.ThrowsAsync<InvalidOperationException>(() => CopyCategory("LicensingThroughput")); + + var endpoints = await ReadCheckpoint("LicensingEndpoints"); + var throughput = await ReadCheckpoint("LicensingThroughput"); + var written = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the endpoint whose sanitized name is too long is skipped as a fault"); + Assert.That(throughput.State, Is.EqualTo(MigrationCategoryState.Blocked), "a Failed predecessor blocks the counts, so the writer never meets a day whose endpoint was skipped"); + Assert.That(throughput.LastError, Does.Contain("must follow LicensingEndpoints, which is CompleteWithErrors")); + Assert.That(written, Is.Zero); + } + } + + [Test] + public async Task A_document_of_another_collection_is_not_read_so_the_endpoints_complete_and_the_counts_copy() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + + using (var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase)) + { + var stray = new { Setting = "left by an older version" }; + await session.StoreAsync(stray, "LegacySettings/1"); + session.Advanced.GetMetadataFor(stray)["@collection"] = "LegacySettings"; + await session.SaveChangesAsync(); + } + + var endpoints = await CopyCategory("LicensingEndpoints"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a document outside the EndpointDocuments collection is not an endpoint, so it is not skipped as a nameless one that blocks the counts"); + Assert.That(endpoints.Copied, Is.EqualTo(1)); + Assert.That(endpoints.Skipped, Is.Zero); + } + + var throughput = await CopyCategory("LicensingThroughput"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(throughput.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(throughput.Copied, Is.EqualTo(1)); + Assert.That(await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()), Is.EqualTo(1)); + } + } + + [Test] + public async Task The_registry_order_blocks_the_counts_until_their_endpoints_are_copied() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + + Assert.ThrowsAsync<InvalidOperationException>(() => CopyCategory("LicensingThroughput")); + var blocked = await ReadCheckpoint("LicensingThroughput"); + + await CopyCategory("LicensingEndpoints"); + var result = await CopyCategory("LicensingThroughput"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(blocked.LastError, Does.Contain("must follow LicensingEndpoints")); + Assert.That(result.Copied, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_retry_copies_the_category_again_with_the_same_count() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + await CopyCategory("LicensingEndpoints"); + + await CopyCategory("LicensingThroughput"); + await Target.ResetForRetry(RetryResetMigrationTests.ResetForRetry(await ReadCheckpoint("LicensingThroughput"))); + var afterReset = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput.CountAsync()); + var second = await CopyCategory("LicensingThroughput"); + + var total = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == new DateOnly(2026, 1, 1)) + .Select(row => row.MessageCount) + .SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterReset, Is.Zero, "the reset deletes the copied days before the category is read again"); + Assert.That(second.Copied, Is.EqualTo(1), "the second run writes the one day once"); + Assert.That(total, Is.EqualTo(100), "the reset deletes the copied days, so copying again does not add the count a second time"); + } + } + + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_add_their_counts_into_one_day_without_failing_the_copy() + { + var day = new DateOnly(2026, 1, 1); + await SeedThroughputSource(new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders /Broker", day, 100); + await SeedThroughputTimeSeries("Sales.Orders/Broker", day, 250); + await CopyCategory("LicensingEndpoints"); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var result = await CopyCategory("LicensingThroughput"); + + var keyIgnoresTrailingSpace = (await QueryTarget(dbContext => Task.FromResult(dbContext.Database.ProviderName))).Contains("SqlServer"); + var count = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == day) + .Select(row => row.MessageCount) + .SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "the reader sends each endpoint in its own batch, and a day added to the row an earlier batch stored counts as copied"); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(result.Skipped, Is.Zero); + Assert.That(await Target.Count(category), Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2), "SQL Server's key ignores the trailing space, so the two days are one row; PostgreSQL's does not, so they are two"); + Assert.That(count, Is.EqualTo(keyIgnoresTrailingSpace ? 350 : 250), "where the key makes the two endpoints one, their counts for the day are added, as the product's own throughput write adds them"); + } + } + + [Test] + public async Task The_copied_counts_are_what_the_collector_reads_as_already_collected() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 1), 100); + await SeedThroughputTimeSeries("Sales.Orders/Broker", new DateOnly(2026, 1, 2), 200); + + await CopyCategory("LicensingEndpoints"); + await CopyCategory("LicensingThroughput"); + + var endpoint = await TargetLicensingDataStore.GetEndpoint(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)); + + Assert.That(endpoint!.LastCollectedDate, Is.EqualTo(new DateOnly(2026, 1, 2)), + "LastCollectedDate is derived from MAX(DateUtc) over the copied rows, and it is the only thing stopping the collector re-collecting these two days and adding to them"); + } + + [Test] + public async Task A_day_whose_endpoint_has_no_name_is_skipped_as_a_required_value_missing() + { + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await CopyCategory("LicensingEndpoints"); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + var day = new DateOnly(2026, 1, 1); + var batch = new MigrationBatch( + [ + new MigrationRow("/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier(null!, ThroughputSource.Broker), day, 100), new Dictionary<string, object>()), + new MigrationRow("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250), new Dictionary<string, object>()) + ], + "Sales.Orders/Broker"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await Target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "/Broker|2026-01-01" })); + Assert.That(result.SkipReasons![MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "a nameless endpoint's day has no NormalizedName to key under, so it is skipped and counted rather than halting the category"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs new file mode 100644 index 0000000000..207bfd706f --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MessageRedirectsMigrationTests.cs @@ -0,0 +1,73 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +[TestFixture] +[NonParallelizable] +class MessageRedirectsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_three_redirects_from_one_document_into_three_rows() + { + var seeded = new[] + { + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = new DateTime(2026, 3, 1, 12, 0, 1, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = new DateTime(2026, 3, 1, 12, 0, 2, DateTimeKind.Utc) }, + new MessageRedirect { FromPhysicalAddress = "old3", ToPhysicalAddress = "new3", LastModified = new DateTime(2026, 3, 1, 12, 0, 3, DateTimeKind.Utc) } + }; + await SeedSourceMessageRedirects(seeded); + + var result = await CopyCategory("MessageRedirects"); + + Assert.That(result.Copied, Is.EqualTo(3)); + + var redirects = await MessageRedirectsDataStore.GetRedirects(); + Assert.That(redirects.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified)), + Is.EquivalentTo(seeded.Select(r => (r.FromPhysicalAddress, r.ToPhysicalAddress, r.LastModified))), + "each redirect keeps its own destination and the time it was last changed"); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!), Is.EqualTo(3), "--migration-verify reads this count"); + } + + [TestCase("old1", "old1")] + [TestCase("Sales@HOST", "sales@host")] + public async Task A_repeated_address_is_copied_or_already_present_as_the_database_compares_it(string firstAddress, string secondAddress) + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = firstAddress, ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = secondAddress, ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + var stored = await MessageRedirectsDataStore.GetRedirects(); + + Assert.That(result.Skipped, Is.Zero, "a repeat loses no row the target lacks, so it is never a skip"); + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(result.Copied, Is.EqualTo(stored.Count), + "Copied is the rows the database stored: one where the key treats the addresses as the same, two where it does not"); + Assert.That(stored.Select(redirect => (redirect.FromPhysicalAddress, redirect.ToPhysicalAddress)), Does.Contain((firstAddress, "new1")), + "the first redirect in document order is copied on every database"); + } + + [Test] + public async Task A_redirect_with_no_destination_is_skipped_as_a_required_value_missing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = null!, LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = DateTime.UtcNow }, + new MessageRedirect { FromPhysicalAddress = null!, ToPhysicalAddress = "new3", LastModified = DateTime.UtcNow }); + + var result = await CopyCategory("MessageRedirects"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one fault skip ends a required category Failed"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "old1", "messageredirects#2" }), "a redirect with no source address is named by its place in the document"); + + var checkpoint = await ReadCheckpoint("MessageRedirects"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1), "a skip copies the rest of the batch, where a throw would have stopped it"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 })); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs new file mode 100644 index 0000000000..35590023e8 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -0,0 +1,739 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NServiceBus.Extensibility; +using NServiceBus.Transport; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; +using NUnit.Framework; +using Particular.LicensingComponent.BrokerThroughput; +using Particular.LicensingComponent.Contracts; +using Particular.LicensingComponent.Persistence; +using ServiceControl.Persistence.EFCore; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.AcceptanceTesting.InfrastructureConfig; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure.WebApi; +using ServiceControl.MessageFailures; +using ServiceControl.Migration.Checks; +using ServiceControl.Monitoring; +using ServiceControl.Notifications; +using ServiceControl.Operations; +using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.Tests; +using ServiceControl.Persistence.UnitOfWork; +using ServiceControl.Recoverability; +using ServiceControl.Transports.BrokerThroughput; +using TestHelper; +// Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. +using IDocumentStore = Raven.Client.Documents.IDocumentStore; + +// No base class: the persistence test bases sit in projects that cannot see RavenDB. +abstract class MigrationAcceptanceTest +{ + protected const string EndpointSettingsUrl = "api/endpointssettings"; + + readonly AcceptanceTestStorageConfiguration StorageConfiguration = new(); + readonly List<string> setVariables = []; + CancellationTokenSource hostCancellation; + Task runningHost; + + ServiceProvider targetServices; + IMigrationCheckpointStore targetCheckpointStore; + readonly FakeBrokerThroughputQuery brokerQuery = new(); + + // RavenDB refuses an incremental time series whose name does not start with "INC:". + // It must equal LicensingDataStore.ThroughputTimeSeriesName, which is internal to the RavenDB persister. + const string ThroughputTimeSeriesName = "INC: throughput data"; + + protected Settings Settings { get; private set; } + protected HttpClient HttpClient { get; private set; } + protected (string ServerUrl, string PrimaryDatabase, string ThroughputDatabase) Source { get; private set; } + protected IDocumentStore SourceStore { get; private set; } + + protected IMigrationTarget Target { get; private set; } + + protected InMemoryBodyStoragePersistence RecordedBodies { get; } = new(); + + [SetUp] + public async Task SetUp() + { + Source = await MigrationSourceServer.CreateDatabases(); + SourceStore = await (await MigrationSourceServer.GetInstance()).Connect(); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", Source.ServerUrl); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", Source.PrimaryDatabase); + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", Source.ThroughputDatabase); + SetSourceVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); +#if DEBUG + // Cleared because a developer who set it to smoke test the migration would let every test past the release check, which only the test host's own AllowUnreleasedMigration registration may do. + SetSourceVariable(RunCommand.AllowUnreleasedMigrationVariable, null); +#endif + + // Settings.Port has no public setter, so the port has to come in as the environment variable a customer would set. + SetSourceVariable("SERVICECONTROL_PORT", PortUtility.GetAssignedOrAvailablePort(33500).ToString(CultureInfo.InvariantCulture)); + + var transport = new ConfigureEndpointLearningTransport(); + + Settings = new Settings( + transportType: transport.TypeName, + persisterType: StorageConfiguration.PersistenceType, + forwardErrorMessages: false, + errorRetentionPeriod: TimeSpan.FromDays(10)) + { + TransportConnectionString = transport.ConnectionString + }; + + await StorageConfiguration.CustomizeSettings(Settings); + await new SetupCommand().Execute(new HostArguments([]), Settings); + + HttpClient = new HttpClient { BaseAddress = new Uri(Settings.RootUrl) }; + + var targetServiceCollection = new ServiceCollection(); + targetServiceCollection.AddLogging(); + targetServiceCollection.AddPersistence(Settings); + targetServiceCollection.AddSingleton<IBodyStoragePersistence>(RecordedBodies); + targetServiceCollection.AddSingleton(new NServiceBus.CriticalError((_, _) => Task.CompletedTask)); + targetServices = targetServiceCollection.BuildServiceProvider(); + + Target = targetServices.GetRequiredService<IMigrationTarget>(); + targetCheckpointStore = targetServices.GetRequiredService<IMigrationCheckpointStore>(); + await Target.Open(); + } + + // The variables go first because they are process wide: a failure in the cleanup below must not leave them set for the next test. + [TearDown] + public async Task TearDown() + { + foreach (var name in setVariables) + { + Environment.SetEnvironmentVariable(name, null); + } + + try + { + await StopHost(); + } + finally + { + HttpClient?.Dispose(); + SourceStore?.Dispose(); + + if (targetServices is not null) + { + await targetServices.DisposeAsync(); + } + + await StorageConfiguration.Cleanup(); + } + } + + protected void SetSourceVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + setVariables.Add(name); + } + + protected async Task SeedSource(string database, params (string Id, object Document)[] documents) + { + using var session = SourceStore.OpenAsyncSession(database); + + foreach (var (id, document) in documents) + { + await session.StoreAsync(document, id); + } + + await session.SaveChangesAsync(); + } + + protected Task SeedSource(params (string Id, object Document)[] documents) => + SeedSource(Source.PrimaryDatabase, documents); + + protected ILicensingDataStore TargetLicensingDataStore => targetServices.GetRequiredService<ILicensingDataStore>(); + + protected async Task SeedThroughputSource(Endpoint endpoint) + { + var documentId = $"{endpoint.Id.Name}/{endpoint.Id.ThroughputSource}"; + + var document = new + { + Id = documentId, + EndpointId = new { endpoint.Id.Name, endpoint.Id.ThroughputSource }, + endpoint.SanitizedName, + EndpointIndicators = endpoint.EndpointIndicators ?? [], + UserIndicator = endpoint.UserIndicator ?? string.Empty, + Scope = endpoint.Scope ?? string.Empty, + endpoint.CreatedDate + }; + + using var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase); + await session.StoreAsync(document, documentId); + session.Advanced.GetMetadataFor(document)["@collection"] = "EndpointDocuments"; + await session.SaveChangesAsync(); + } + + protected async Task SeedThroughputTimeSeries(string documentId, DateOnly day, long count) + { + using var session = SourceStore.OpenAsyncSession(Source.ThroughputDatabase); + session.IncrementalTimeSeriesFor(documentId, ThroughputTimeSeriesName) + .Increment(day.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), count); + await session.SaveChangesAsync(); + } + + protected void BrokerReports(string queue, DateOnly day, long count) => brokerQuery.Stage(queue, day, count); + + protected IReadOnlyList<(string Queue, DateOnly StartDate)> BrokerRequests => brokerQuery.Requests; + + protected Task WaitForCollectorPass() => + WaitUntil(() => Task.FromResult(brokerQuery.CompletedPasses > 0), "the broker throughput collector completed a pass"); + + // The product adds the broker collector only when a query is registered before AddServiceControl, and customize runs after it. + protected void CollectFromTheBrokerImmediately(IServiceCollection services) + { + services.AddSingleton<IBrokerThroughputQuery>(brokerQuery); + services.AddHostedService(provider => new BrokerThroughputCollectorHostedService( + provider.GetRequiredService<ILogger<BrokerThroughputCollectorHostedService>>(), + brokerQuery, + provider.GetRequiredService<ThroughputSettings>(), + provider.GetRequiredService<ILicensingDataStore>(), + provider.GetRequiredService<TimeProvider>()) + { + DelayStart = TimeSpan.Zero + }); + } + + protected Task SeedThroughputReportMasks(IReadOnlyList<string> maskedStrings) => + SeedSource(Source.ThroughputDatabase, ("ReportMasks", new { MaskedStrings = maskedStrings.ToList() })); + + protected Task SeedThroughputLicensedEndpointDetails(LicensedEndpointDetails details) => + SeedSource(Source.ThroughputDatabase, ("LicensedEndpointDetails", details)); + + protected Task SeedSourceMessageRedirects(params MessageRedirect[] redirects) => + SeedSource(("messageredirects", new + { + Redirects = redirects.Select(redirect => new { redirect.FromPhysicalAddress, redirect.ToPhysicalAddress, LastModifiedTicks = redirect.LastModified.Ticks }).ToList() + })); + + // Stores the type as the string MessageTypeConverter writes, because that is what the persister reads. + protected Task SeedSourceSubscriptions(params SubscriptionRow[] subscriptions) => + SeedSource(("Subscriptions/All", new + { + All = subscriptions + .GroupBy(subscription => new MessageType(subscription.MessageType, subscription.Version).ToString()) + .ToDictionary(group => group.Key, group => new + { + Id = group.Key, + MessageType = group.Key, + Subscribers = group.Select(subscription => new { subscription.TransportAddress, subscription.Endpoint }).ToList() + }) + })); + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + protected async Task SeedSourceNotificationSettings(EmailNotifications emailNotifications, string collection = "NotificationsSettingsDocuments") + { + var document = new { Email = emailNotifications }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, "NotificationsSettings/All"); + session.Advanced.GetMetadataFor(document)["@collection"] = collection; + await session.SaveChangesAsync(); + } + + protected Task SeedSourceTrialEndDate(DateOnly endDate) => + SeedSource(("metadata/trialinformation", new TrialMetadata { TrialEndDate = endDate })); + + protected Task SeedSourceRetryHistory(RetryHistory history) => + SeedSource(("RetryOperations/History", history)); + + // The id is the one the RavenDB persister gives a claim, retyped because this project cannot reference that persister. + protected async Task SeedSourceRetryClaim(string uniqueMessageId) + { + var document = new { FailedMessageId = $"FailedMessages/{uniqueMessageId}", RetryBatchId = "RetryBatches/forwarded" }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, $"FailedMessageRetries/{uniqueMessageId}"); + session.Advanced.GetMetadataFor(document)["@collection"] = "FailedMessageRetries"; + await session.SaveChangesAsync(); + } + + protected Task SeedSourceWithBody(FailedMessage document, byte[] body, string contentType) => + SeedSourceWithBodies((document, body, contentType)); + + protected async Task SeedSourceWithBodies(params (FailedMessage Document, byte[] Body, string ContentType)[] messages) + { + using (var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase)) + { + foreach (var (document, _, _) in messages) + { + await session.StoreAsync(document, $"FailedMessages/{document.UniqueMessageId}"); + } + + await session.SaveChangesAsync(); + } + + using (var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase)) + { + foreach (var (document, body, contentType) in messages) + { + // "body" is the attachment name the RavenDB persister reads, retyped because this project cannot reference that persister. + session.Advanced.Attachments.Store($"FailedMessages/{document.UniqueMessageId}", "body", new MemoryStream(body), contentType); + } + + await session.SaveChangesAsync(); + } + } + + protected Task SeedSourceEndpointSettings(params (string Name, bool TrackInstances)[] settings) => + SeedSource([.. settings.Select(setting => ($"EndpointSettings/{DeterministicGuid.MakeId(setting.Name)}", (object)new EndpointSettings { Name = setting.Name, TrackInstances = setting.TrackInstances }))]); + + // The target copies an endpoint setting only when that endpoint is known, so seed the endpoint too. + protected Task SeedSourceKnownEndpoints(params string[] names) => + SeedSource([.. names + .Select(name => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" } }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint))]); + + // A whole second in UTC, because PostgreSQL refuses an unspecified kind and keeps only microseconds. + protected static readonly DateTime SeededReportedAt = new(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + // Written out by shape, because the RavenDB persister's CustomCheck type is in an assembly this project does not reference. + protected async Task SeedSourceCustomCheck( + EndpointDetails endpoint, string customCheckId, string category = "Ping", bool failed = false, string reason = null) + { + var detail = new CustomCheckDetail + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = category, + ReportedAt = SeededReportedAt, + HasFailed = failed, + FailureReason = reason + }; + + var documentId = $"CustomChecks/{detail.GetDeterministicId()}"; + var document = new + { + Id = documentId, + detail.CustomCheckId, + detail.Category, + // The integer, so the document reads back the same whether or not the seeding store saves enums as integers. + Status = (int)(failed ? Status.Fail : Status.Pass), + detail.ReportedAt, + FailureReason = reason, + OriginatingEndpoint = endpoint + }; + + using var session = SourceStore.OpenAsyncSession(Source.PrimaryDatabase); + await session.StoreAsync(document, documentId); + session.Advanced.GetMetadataFor(document)["@collection"] = "CustomChecks"; + await session.SaveChangesAsync(); + } + + protected Task<IMigrationSource> OpenSource(CancellationToken cancellationToken = default) => + PersistenceFactory.OpenMigrationSource(Settings, cancellationToken); + + static IReadOnlyCollection<string> sourceSupportedCategoryIds; + + // SupportedCategoryIds answers before Open and cannot change across it, so one source built once for the + // whole run answers it for every test instead of loading the persister assembly again per test. + protected async Task<IReadOnlyCollection<string>> SourceSupportedCategoryIds() + { + if (sourceSupportedCategoryIds is null) + { + await using var source = PersistenceFactory.CreateMigrationSource(Settings); + sourceSupportedCategoryIds = source.SupportedCategoryIds; + } + + return sourceSupportedCategoryIds; + } + + + // No build carries the whole migration until its last phase ships, so without this marker every copy is refused. + protected static Action<WebApplicationBuilder> AllowingAnUnreleasedMigration(Action<WebApplicationBuilder> customize = null) => + builder => + { + builder.Services.AddSingleton<AllowUnreleasedMigration>(); + customize?.Invoke(builder); + }; + + protected async Task MigrateThenRunHost(Action<WebApplicationBuilder> customize = null) + { + await StopHost(); + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(customize)); + await RunHostUntilTheApiAnswers(customize); + } + + protected async Task RunHostUntilTheApiAnswers(Action<WebApplicationBuilder> customize = null) + { + await StopHost(); + + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + hostCancellation = new CancellationTokenSource(); + runningHost = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, customize)), hostCancellation.Token); + + await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + + // The API answers before StartedAsync runs, and a stop before StartedAsync finishes cancels the start, which RunCommand rightly treats as a failure. + await await Task.WhenAny(started.Task, runningHost).WaitAsync(TimeSpan.FromMinutes(1)); + } + + // ApplicationStarted fires only after every hosted service's StartedAsync, which is where the marker is written. + protected static Action<WebApplicationBuilder> SignallingOnceStarted(TaskCompletionSource started, Action<WebApplicationBuilder> customize) => + builder => + { + builder.Services.AddHostedService(provider => new SignalWhenStarted(provider.GetRequiredService<IHostApplicationLifetime>(), started)); + customize?.Invoke(builder); + }; + + sealed class SignalWhenStarted(IHostApplicationLifetime lifetime, TaskCompletionSource started) : IHostedService + { + public Task StartAsync(CancellationToken cancellationToken = default) + { + lifetime.ApplicationStarted.Register(() => started.TrySetResult()); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + protected Settings MigrationSettings(bool copyEventLog = false, Action<Settings> customize = null) + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", copyEventLog ? null : "0"); + SetSourceVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "0"); + customize?.Invoke(Settings); + return Settings; + } + + protected async Task RunRequiredCopy( + Action<IServiceCollection> customize = null, + bool leaveOptionalIncomplete = false, + CancellationToken cancellationToken = default) + { + MigrationSettings(copyEventLog: leaveOptionalIncomplete); + + await MigrateThenRunHost(builder => customize?.Invoke(builder.Services)); + + var copyable = MigrationStartup.CopyableCategoryIds(await SourceSupportedCategoryIds(), Target.SupportedCategoryIds); + + var copied = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Required && copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + await WaitUntil( + async () => (await targetCheckpointStore.ReadAll(cancellationToken)) + .Count(checkpoint => copied.Contains(checkpoint.CategoryId) && checkpoint.State.IsFinished()) == copied.Length, + "the required copy finished"); + + if (leaveOptionalIncomplete) + { + var eventLog = await targetCheckpointStore.Read(MigrationCategoryIds.EventLog, cancellationToken); + await targetCheckpointStore.Upsert(eventLog with + { + State = MigrationCategoryState.Halted, + LastError = "Halted by the test fixture." + }, cancellationToken); + } + } + + protected async Task<IReadOnlyList<EndpointSettings>> WaitForEndpointSettingsResponse(TimeSpan timeout) + { + IReadOnlyList<EndpointSettings> settings = null; + + await WaitUntil(async () => + { + // A host that refused to start surfaces its own exception here instead of waiting out the timeout. + if (runningHost is { IsFaulted: true }) + { + await runningHost; + } + + try + { + settings = await GetEndpointSettings(); + return true; + } + catch (HttpRequestException) + { + return false; + } + }, $"GET {EndpointSettingsUrl} answered", timeout); + + return settings; + } + + protected async Task<IReadOnlyList<EndpointSettings>> GetEndpointSettings() => + await HttpClient.GetFromJsonAsync<List<EndpointSettings>>(EndpointSettingsUrl, SerializerOptions.Default); + + async Task StopHost() + { + if (runningHost is null) + { + return; + } + + await hostCancellation.CancelAsync(); + await runningHost; + hostCancellation.Dispose(); + runningHost = null; + } + + // Copied from PersistenceTestBase.WaitUntil, which this project cannot compile. + protected static async Task WaitUntil(Func<Task<bool>> conditionChecker, string condition, TimeSpan timeout = default) + { + timeout = timeout == default ? TimeSpan.FromSeconds(10) : timeout; + + var start = DateTime.UtcNow; + + while (DateTime.UtcNow - start < timeout) + { + if (await conditionChecker()) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(500)); + } + + throw new Exception($"{condition} has not been meet in defined timespan: {timeout})"); + } + + // Copied from RavenMigrationSourceTestBase.CollectBatches, which this project cannot compile. + protected static async Task<List<MigrationBatch>> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List<MigrationBatch>(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } + + SyncPassCountingLogger syncPasses; + + protected void CountSyncPasses(IServiceCollection services) + { + syncPasses = null; + services.AddSingleton<ILogger<HeartbeatEndpointSettingsSyncHostedService>>(provider => + syncPasses = new SyncPassCountingLogger(provider.GetRequiredService<ILoggerFactory>().CreateLogger<HeartbeatEndpointSettingsSyncHostedService>())); + } + + protected async Task WaitForSyncPass(FakeTimeProvider time) + { + await WaitUntil(() => Task.FromResult(syncPasses?.Started >= 1), "the heartbeat settings sync started a pass"); + + time.Advance(TimeSpan.FromHours(6) + TimeSpan.FromSeconds(1)); + + await WaitUntil(() => Task.FromResult(syncPasses.Started >= 2), "the heartbeat settings sync finished its pass and reached its next tick"); + } + + sealed class SyncPassCountingLogger(ILogger inner) : ILogger<HeartbeatEndpointSettingsSyncHostedService> + { + int started; + + public int Started => started; + + public IDisposable BeginScope<TState>(TState state) where TState : notnull => inner.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) + { + if (state is IReadOnlyList<KeyValuePair<string, object>> values + && values.Any(value => value.Key == "{OriginalFormat}" && Equals(value.Value, "Performing sync for {ServiceName}"))) + { + Interlocked.Increment(ref started); + } + + inner.Log(logLevel, eventId, state, exception, formatter); + } + } + + protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService<IEndpointSettingsStore>(); + protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService<IMonitoringDataStore>(); + protected IRetryBatchStore RetryBatchStore => targetServices.GetRequiredService<IRetryBatchStore>(); + protected IRetryStagingStore RetryStagingStore => targetServices.GetRequiredService<IRetryStagingStore>(); + protected IFailedMessageRetryDataStore FailedMessageRetryDataStore => targetServices.GetRequiredService<IFailedMessageRetryDataStore>(); + protected IMessageRedirectsDataStore MessageRedirectsDataStore => targetServices.GetRequiredService<IMessageRedirectsDataStore>(); + protected INotificationsDataStore NotificationsStore => targetServices.GetRequiredService<INotificationsDataStore>(); + protected ITrialLicenseDataProvider TrialLicenseDataProvider => targetServices.GetRequiredService<ITrialLicenseDataProvider>(); + protected IRetryHistoryDataStore RetryHistoryStore => targetServices.GetRequiredService<IRetryHistoryDataStore>(); + protected ExternalIntegrationRequestsDataStore ExternalIntegrationStore => targetServices.GetRequiredService<ExternalIntegrationRequestsDataStore>(); + protected ICustomChecksDataStore CustomChecks => targetServices.GetRequiredService<ICustomChecksDataStore>(); + protected IFailedErrorImportDataStore FailedImportStore => targetServices.GetRequiredService<IFailedErrorImportDataStore>(); + protected IGroupsDataStore GroupsStore => targetServices.GetRequiredService<IGroupsDataStore>(); + + // Built by hand, because its DI constructor needs NServiceBus settings the target services do not have. + // A zero cache duration makes every read go to the table. + protected ISubscriptionStorage SubscriptionStorage => + new SubscriptionStorage(targetServices.GetRequiredService<IServiceScopeFactory>(), "Particular.ServiceControl", "Particular.ServiceControl", [], TimeSpan.Zero, TimeProvider.System); + + protected async Task<MigrationWriteResult> CopyCategory(string categoryId, CancellationToken cancellationToken = default) + { + var category = MigrationCategoryRegistry.Find(categoryId) ?? throw new ArgumentException($"'{categoryId}' is not a migration category.", nameof(categoryId)); + var before = await targetCheckpointStore.Read(categoryId, cancellationToken); + var target = new RecordingMigrationTarget(Target); + var options = new MigrationEngineOptions(TimeSpan.Zero, Persistence.DataMigration.MigrationSettings.DefaultHaltThresholdPercent, Persistence.DataMigration.MigrationSettings.DefaultHaltThresholdMinimum, []) { BodyRetryBackoff = TimeSpan.Zero }; + + await using var source = await OpenSource(cancellationToken); + var engine = new MigrationEngine(source, target, targetCheckpointStore, TimeProvider.System, options, NullLogger<MigrationEngine>.Instance); + + var after = await engine.RunCategoryAsync(category, cancellationToken); + + // Both reached the end of the source, so a test can assert a Failed category's skips as well as a Done one's. + if (after.State is not (MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors)) + { + throw new InvalidOperationException($"Copying '{categoryId}' ended {after.State}: {after.LastError}"); + } + + return new MigrationWriteResult( + after, + (int)(after.CopiedCount - (before?.CopiedCount ?? 0)), + (int)(after.SkippedCount - (before?.SkippedCount ?? 0)), + target.SkippedIds, + (int)(after.AlreadyPresentCount - (before?.AlreadyPresentCount ?? 0))); + } + + + // Makes the target look like a database a migration has touched, which is the only state the host-opened marker is stamped in. + protected async Task SeedCheckpoint(string categoryId, MigrationCategoryState state = MigrationCategoryState.Complete) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + var existing = await targetCheckpointStore.Read(categoryId); + await dbContext.UpsertCheckpoint(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null) + { + Version = existing?.Version ?? 0 + }); + } + + protected async Task DeleteCheckpoint(string categoryId) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + await dbContext.MigrationCheckpoints.Where(checkpoint => checkpoint.CategoryId == categoryId).ExecuteDeleteAsync(); + } + + protected async Task<T> QueryTarget<T>(Func<ServiceControlDbContext, Task<T>> query) + { + using var scope = targetServices.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + return await query(dbContext); + } + + protected async Task<FailedMessageEntity> GetFailedMessage(Guid uniqueMessageId) + { + var row = await QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + Assert.That(row, Is.Not.Null, $"No failed message row for {uniqueMessageId}"); + + return row; + } + + protected Task<FailedMessageEntity> FindFailedMessage(Guid uniqueMessageId) => + QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + protected EFPersisterSettings EFSettings => targetServices.GetRequiredService<EFPersisterSettings>(); + + protected IBodyStorage BodyStorage => targetServices.GetRequiredService<IBodyStorage>(); + + protected Task Ingest(params IngestedFailure[] failures) => + InBatch(async unitOfWork => + { + foreach (var failure in failures) + { + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups); + } + }); + + // Ingestion takes the body and its content type from the message context, so only the context changes. + protected Task IngestWithBody(IngestedFailure failure, byte[] body, string contentType) => + InBatch(unitOfWork => + { + var headers = new Dictionary<string, string>(failure.Headers) { [NServiceBus.Headers.ContentType] = contentType }; + var context = new MessageContext(failure.MessageId, headers, body, new TransportTransaction(), "receiveAddress", new ContextBag()); + + return unitOfWork.Recoverability.RecordFailedProcessingAttempt(context, failure.ProcessingAttempt, failure.Groups); + }); + + async Task InBatch(Func<IIngestionUnitOfWork, Task> record) + { + await using var unitOfWork = await targetServices.GetRequiredService<IIngestionUnitOfWorkFactory>().StartNew(); + + await record(unitOfWork); + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + } + + protected Task<MigrationCheckpoint> ReadCheckpoint(string categoryId) => + QueryTarget(async dbContext => + { + var row = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(checkpoint => checkpoint.CategoryId == categoryId); + + return row is null ? null : new MigrationCheckpoint( + row.CategoryId, row.State, row.Cursor, + row.CopiedCount, row.SkippedCount, row.SourceTotal, row.SkipReasons, + row.StartedAt, row.LastProgressAt, row.SettledAt, row.LastError, + row.AlreadyPresentCount, row.Version); + }); + + protected Task<string> ReadSetting(string key) => + QueryTarget(dbContext => dbContext.Settings.AsNoTracking() + .Where(setting => setting.Key == key) + .Select(setting => setting.Value) + .SingleOrDefaultAsync()); + + sealed class RecordingMigrationTarget(IMigrationTarget inner) : IMigrationTarget + { + public List<string> SkippedIds { get; } = []; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var result = await inner.Write(category, batch, checkpointToExtend, cancellationToken); + SkippedIds.AddRange(result.SkippedIds); + return result; + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary<string, Type> DocumentTypes => inner.DocumentTypes; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs new file mode 100644 index 0000000000..88a33b0003 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.RavenDB; +using TestHelper; + +// Its own copy rather than SharedEmbeddedServer: that one uses the RavenDB persister's own settings type, which this project cannot reference because it loads the persister from its manifest instead. +static class MigrationSourceServer +{ + public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancellationToken = default) + { + await startLock.WaitAsync(cancellationToken); + try + { + if (server != null) + { + return server; + } + + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "MigrationSource"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "MigrationSource"); + var port = PortUtility.GetAssignedOrAvailablePort(33350); + + var configuration = new EmbeddedDatabaseConfiguration($"http://localhost:{port}", "primary", dbPath, logPath, "Operations") { RunInMemory = true }; + + server = EmbeddedDatabase.Start(configuration, lifetime); + return server; + } + finally + { + startLock.Release(); + } + } + + public static async Task Stop() + { + await startLock.WaitAsync(); + try + { + if (server is null) + { + return; + } + + await server.Stop(CancellationToken.None); + server.Dispose(); + server = null; + lifetime.StopApplication(); + } + finally + { + startLock.Release(); + } + } + + public static async Task<(string ServerUrl, string PrimaryDatabase, string ThroughputDatabase)> CreateDatabases(CancellationToken cancellationToken = default) + { + var instance = await GetInstance(cancellationToken); + var primary = $"sc_src_{Guid.NewGuid():n}"; + var throughput = $"{primary}-throughput"; + + using var store = await instance.Connect(cancellationToken); + foreach (var name in new[] { primary, throughput }) + { + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(name)), cancellationToken); + } + + return (instance.ServerUrl, primary, throughput); + } + + static EmbeddedDatabase server; + static readonly ApplicationLifetime lifetime = new(new NullLogger<ApplicationLifetime>()); + static readonly SemaphoreSlim startLock = new(1, 1); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs new file mode 100644 index 0000000000..14ba89749c --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/NotificationSettingsMigrationTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class NotificationSettingsMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory NotificationSettingsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.NotificationSettings)!; + + // Every field differs from its default, so a field the copy drops cannot match by accident. + static EmailNotifications Seeded => new() + { + Enabled = true, + SmtpServer = "smtp.contoso.com", + SmtpPort = 587, + AuthenticationAccount = "servicecontrol", + AuthenticationPassword = "not-a-real-password", + EnableTLS = true, + To = "ops@contoso.com", + From = "servicecontrol@contoso.com" + }; + + // Version 6.19 and earlier stored this document in the NotificationsSettings collection. + [TestCase("NotificationsSettingsDocuments", "NotificationsSettings")] + [TestCase("NotificationsSettings", "NotificationsSettingsDocuments")] + public async Task Copies_notification_settings_as_one_row_in_settings_from_either_collection(string collection, string otherCollection) + { + var seeded = Seeded; + await SeedSourceNotificationSettings(seeded, collection); + + var result = await CopyCategory("NotificationSettings"); + + var collections = (await SourceStore.Maintenance.ForDatabase(Source.PrimaryDatabase).SendAsync(new GetCollectionStatisticsOperation())).Collections; + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(collections, Does.ContainKey(collection).And.Not.ContainKey(otherCollection), "the test proves nothing unless the seed landed in the collection this case names"); + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(seeded)), "LoadSettings reads through GetSetting<EmailNotifications>, so every field must survive that read"); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } + + [Test] + public async Task A_document_with_no_email_settings_copies_nothing() + { + await SeedSourceNotificationSettings(null); + + var result = await CopyCategory("NotificationSettings"); + + var settings = await NotificationsStore.LoadSettings(); + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(await Target.Count(NotificationSettingsCategory), Is.Zero, + "a row may not carry a null document, and the product reads a null value and a missing row as the same default"); + Assert.That(JsonSerializer.Serialize(settings.Email), Is.EqualTo(JsonSerializer.Serialize(new EmailNotifications()))); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs new file mode 100644 index 0000000000..b390acfd56 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/PendingIntegrationEventsMigrationTests.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Recoverability.ExternalIntegration; + +[TestFixture] +[NonParallelizable] +class PendingIntegrationEventsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_queued_event_is_copied_and_the_product_dispatches_it_as_the_type_it_was_raised_as() + { + var id = $"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}"; + await SeedSource((id, new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/1" } })); + + var result = await CopyCategory("PendingIntegrationEvents"); + + object[] dispatched = []; + ExternalIntegrationStore.Subscribe((contexts, _) => + { + dispatched = contexts; + return Task.CompletedTask; + }); + await ExternalIntegrationStore.DispatchNow(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(dispatched.Single(), Is.InstanceOf<FailedMessageArchivedPublisher.DispatchContext>()); + Assert.That(((FailedMessageArchivedPublisher.DispatchContext)dispatched.Single()).FailedMessageId, Is.EqualTo("FailedMessages/1")); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs new file mode 100644 index 0000000000..da6e68e86e --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryOperationsMigrationTests.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +[TestFixture] +[NonParallelizable] +class RetryOperationsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_one_document_into_both_target_tables() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-10), + CompletionTime = CompletedAt.AddMinutes(-9), + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + }); + history.HistoricOperations.Add(HistoricOperation("group-3", CompletedAt.AddMinutes(-20))); + history.UnacknowledgedOperations.Add(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-5), + CompletionTime = CompletedAt.AddMinutes(-4), + Last = DateTime.MaxValue, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + }); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(3)); + Assert.That(await Target.Count(MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)), Is.EqualTo(3), "--migration-verify counts both tables, and a count of either table alone would report 2 or 1"); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(Columns), Is.EqualTo(history.HistoricOperations.Select(Columns)), "every column of the historic operation arrives as the source holds it"); + Assert.That(stored.UnacknowledgedOperations.Select(Columns), Is.EqualTo(history.UnacknowledgedOperations.Select(Columns)), "every column of the pending operation arrives as the source holds it, even a Last of DateTime.MaxValue, which InMemoryRetry records when it has no last time"); + } + + [Test] + public async Task Duplicate_pending_operations_with_one_key_copy_once_and_count_the_rest_as_already_present() + { + var history = new RetryHistory(); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt, originator: "First in the document")); + history.UnacknowledgedOperations.Add(PendingOperation("group-2", CompletedAt.AddMinutes(1), originator: "Second in the document")); + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the second lands on the key the first fills, so it loses no row the target lacks and is neither a skip nor a primary-key violation"); + Assert.That(result.Skipped, Is.Zero); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.UnacknowledgedOperations.Single().Originator, Is.EqualTo("Second in the document"), "RavenDB appends a later retry of the same group, and the SQL store keeps the latest one, so the copy keeps the last in document order"); + } + + [Test] + public async Task Copies_more_operations_than_one_statement_could_carry() + { + var history = new RetryHistory(); + + foreach (var index in Enumerable.Range(0, OperationsPerList)) + { + history.HistoricOperations.Add(HistoricOperation($"historic-{index}", CompletedAt.AddMinutes(index))); + history.UnacknowledgedOperations.Add(PendingOperation($"pending-{index}", CompletedAt)); + } + + await SeedSourceRetryHistory(history); + + var result = await CopyCategory("RetryOperations"); + + Assert.That(result.Copied, Is.EqualTo(2 * OperationsPerList)); + + var stored = (await RetryHistoryStore.GetRetryHistory()).Results; + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.HistoricOperations.Select(operation => operation.RequestId))); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(history.UnacknowledgedOperations.Select(operation => operation.RequestId))); + } + + // More rows than SQL Server's 2,000 parameters could carry at even one column each. + const int OperationsPerList = 2_100; + + // Whole seconds, because PostgreSQL stores microseconds and would not give back a time with more precision. + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + static (string, RetryType, DateTime, DateTime, string, bool, int) Columns(HistoricRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Originator, operation.Failed, operation.NumberOfMessagesProcessed); + + static (string, RetryType, DateTime, DateTime, DateTime, string, string, bool, int) Columns(UnacknowledgedRetryOperation operation) => + (operation.RequestId, operation.RetryType, operation.StartTime, operation.CompletionTime, operation.Last, operation.Originator, operation.Classifier, operation.Failed, operation.NumberOfMessagesProcessed); + + internal static HistoricRetryOperation HistoricOperation(string requestId, DateTime completedAt) => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + internal static UnacknowledgedRetryOperation PendingOperation(string requestId, DateTime completedAt, string originator = "Shipping") => new() + { + RequestId = requestId, + RetryType = RetryType.FailureGroup, + StartTime = completedAt.AddMinutes(-1), + CompletionTime = completedAt, + Last = completedAt, + Originator = originator, + Classifier = "Shipping-Failures", + NumberOfMessagesProcessed = 1 + }; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs new file mode 100644 index 0000000000..349a2266cb --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/RetryResetMigrationTests.cs @@ -0,0 +1,117 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Recoverability; +using ServiceControl.Recoverability.ExternalIntegration; + +[TestFixture] +[NonParallelizable] +class RetryResetMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task A_retry_of_retry_operations_leaves_both_tables_as_one_copy_left_them() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var copied = await Target.Count(RetryOperations); + + var saved = await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.RetryOperations))); + var stored = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + var second = await CopyCategory(MigrationCategoryIds.RetryOperations); + + using (Assert.EnterMultipleScope()) + { + Assert.That(saved.Version, Is.EqualTo(stored.Version), "the returned checkpoint carries the version the reset saved"); + Assert.That(saved.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(copied, Is.EqualTo(4), "the first copy fills both tables"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(copied), "a historic row gets a new key on every insert, so a retry that kept them would double the history"); + Assert.That(second.AlreadyPresent, Is.Zero, "the pending table was emptied too, so the re-read finds none of its rows already present"); + } + } + + [Test] + public async Task A_retry_of_a_category_whose_keys_the_copy_supplies_deletes_nothing() + { + await SeedSourceMessageRedirects( + new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = ChangedAt }, + new MessageRedirect { FromPhysicalAddress = "old2", ToPhysicalAddress = "new2", LastModified = ChangedAt }); + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.MessageRedirects); + await CopyCategory(MigrationCategoryIds.RetryOperations); + + await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.MessageRedirects))); + var second = await CopyCategory(MigrationCategoryIds.MessageRedirects); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.AlreadyPresent, Is.EqualTo(2), "rows already in SQL are left alone, so the re-read finds both"); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the reset deletes only for the category it names, so another category's copied rows stay"); + } + } + + [Test] + public async Task A_stale_reset_deletes_nothing() + { + await SeedSourceRetryHistory(TwoHistoricAndTwoPending()); + await CopyCategory(MigrationCategoryIds.RetryOperations); + var stale = await ReadCheckpoint(MigrationCategoryIds.RetryOperations); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(stale)); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.ResetForRetry(ResetForRetry(stale))); + Assert.That(await Target.Count(RetryOperations), Is.EqualTo(4), "the delete shares the reset's transaction, so a refused reset leaves every copied row"); + } + + [Test] + public async Task A_retry_of_pending_integration_events_dispatches_each_event_once() + { + await SeedSource( + ($"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}", new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/1" } }), + ($"ExternalIntegrationDispatchRequests/{Guid.NewGuid()}", new ExternalIntegrationDispatchRequest { DispatchContext = new FailedMessageArchivedPublisher.DispatchContext { FailedMessageId = "FailedMessages/2" } })); + await CopyCategory(MigrationCategoryIds.PendingIntegrationEvents); + var copied = await Target.Count(PendingIntegrationEvents); + + await Target.ResetForRetry(ResetForRetry(await ReadCheckpoint(MigrationCategoryIds.PendingIntegrationEvents))); + await CopyCategory(MigrationCategoryIds.PendingIntegrationEvents); + + using (Assert.EnterMultipleScope()) + { + Assert.That(copied, Is.EqualTo(2), "the first copy writes both requests"); + Assert.That(await Target.Count(PendingIntegrationEvents), Is.EqualTo(copied), "each request gets a new key on every insert, so a retry that kept them would dispatch every event twice"); + } + } + + // The same reset --migration-retry makes. It keeps StartedAt and leaves SourceTotal alone, because the engine does not write it. + internal static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0, + SkipReasons = null, + SettledAt = null, + LastError = null + }; + + static RetryHistory TwoHistoricAndTwoPending() + { + var history = new RetryHistory(); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-1", ChangedAt)); + history.HistoricOperations.Add(RetryOperationsMigrationTests.HistoricOperation("group-2", ChangedAt.AddMinutes(-10))); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-3", ChangedAt)); + history.UnacknowledgedOperations.Add(RetryOperationsMigrationTests.PendingOperation("group-4", ChangedAt.AddMinutes(-10))); + return history; + } + + static MigrationCategory PendingIntegrationEvents => MigrationCategoryRegistry.Find(MigrationCategoryIds.PendingIntegrationEvents)!; + + static MigrationCategory RetryOperations => MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime ChangedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs new file mode 100644 index 0000000000..b42d2671f2 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[SetUpFixture] +public class StopMigrationSourceServer +{ + [OneTimeTearDown] + public Task Teardown() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs new file mode 100644 index 0000000000..87455299ac --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/SubscriptionsMigrationTests.cs @@ -0,0 +1,62 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Extensibility; +using NServiceBus.Unicast.Subscriptions; +using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class SubscriptionsMigrationTests : MigrationAcceptanceTest +{ + [Test] + public async Task Two_message_type_versions_for_the_same_subscriber_merge_onto_one_row_and_the_merge_is_already_present() + { + await SeedSourceSubscriptions( + new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), + new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); + + var result = await CopyCategory("Subscriptions"); + + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a version merge loses no row the target lacks, and a skip would leave the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete), "a merge is not a skip, so the category is Done"); + + var subscribers = await SubscriptionStorage.GetSubscriberAddressesForMessage( + [new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0))], new ContextBag(), TestContext.CurrentContext.CancellationToken); + + Assert.That(subscribers.Count(), Is.EqualTo(1)); + Assert.That(subscribers.Select(subscriber => (subscriber.TransportAddress, subscriber.Endpoint)), Is.EqualTo(new[] { ("Billing@BILLING01", "Billing") }), + "the copied row holds the subscriber's address and endpoint as the source has them"); + } + + [Test] + public async Task A_legacy_subscriber_with_no_endpoint_gets_the_endpoint_the_product_stores_for_one() + { + var copiedType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var storedType = new MessageType("MyMessages.OrderShipped", new Version(1, 0, 0)); + var token = TestContext.CurrentContext.CancellationToken; + + await SeedSourceSubscriptions(new SubscriptionRow(copiedType.TypeName, copiedType.Version, "Billing@BILLING01", Endpoint: null)); + + var result = await CopyCategory("Subscriptions"); + + // Another type, so the product's upsert cannot overwrite the copied row it is compared with. + await SubscriptionStorage.Subscribe(new Subscriber("Billing@BILLING01", null), storedType, new ContextBag(), token); + + var copied = await SubscriptionStorage.GetSubscriberAddressesForMessage([copiedType], new ContextBag(), token); + var stored = await SubscriptionStorage.GetSubscriberAddressesForMessage([storedType], new ContextBag(), token); + + Assert.That(result.Copied, Is.EqualTo(1), "Endpoint is NOT NULL on the target, so a row built without a fallback fails the batch and leaves the required category Failed"); + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(copied.Single().TransportAddress, Is.EqualTo("Billing@BILLING01")); + Assert.That(copied.Single().Endpoint, Is.EqualTo(stored.Single().Endpoint), + "the copy must store the endpoint SubscriptionStorage itself stores for the same legacy subscriber"); + Assert.That(copied.Single().Endpoint, Is.EqualTo("Billing"), "the endpoint is the part of the address before the '@'"); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs new file mode 100644 index 0000000000..a30e2caa57 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +static class TestMigrationTargets +{ + public static void ParkFirstMigrationWrite(this WebApplicationBuilder builder, TaskCompletionSource parked, Task release) => + builder.DecorateMigrationTarget(inner => new ParkingMigrationTarget(inner, parked, release)); + + public static void FailTheSecondMigrationWrite(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 2)); + + public static void HaltTheEndpointSettingsCategory(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 1)); + + public static void StopAfterTheFirstCategorySettles(this WebApplicationBuilder builder, CancellationTokenSource stopping) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationCheckpointStore)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationCheckpointStore is registered by a factory rather than by type, so the test decorator cannot rebuild the inner store. Register it as AddSingleton<IMigrationCheckpointStore, TStore>() or give the decorator a different seam."); + } + + builder.Services.AddSingleton<IMigrationCheckpointStore>(provider => + new StoppingAfterTheFirstSettleCheckpointStore((IMigrationCheckpointStore)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType), stopping)); + } + + public static void DecorateMigrationTarget(this WebApplicationBuilder builder, Func<IMigrationTarget, IMigrationTarget> decorate) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationTarget)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationTarget is registered by a factory rather than by type, so the test decorator cannot rebuild the inner target. Register it as AddSingleton<IMigrationTarget, TTarget>() or give the decorator a different seam."); + } + + builder.Services.AddSingleton<IMigrationTarget>(provider => + decorate((IMigrationTarget)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType))); + } +} + +// Throws on one EndpointSettings write, which is the only way to see what a copy does after a write has failed. +sealed class FaultingMigrationTarget(IMigrationTarget inner, int failingWrite) : IMigrationTarget +{ + int endpointSettingsWrites; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + // Two rows a batch, so five settings take three writes and a failure on the second leaves exactly one batch committed. + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings ? Task.FromResult(2) : inner.BatchSizeFor(category, cancellationToken); + + public Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings && ++endpointSettingsWrites == failingWrite + ? throw new Exception("injected mid-category failure") + : inner.Write(category, batch, checkpointToExtend, cancellationToken); + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary<string, Type> DocumentTypes => inner.DocumentTypes; +} + +// Stops the copy the moment the first category settles Done or Abandoned, which is the only way to stop it between two categories. +sealed class StoppingAfterTheFirstSettleCheckpointStore(IMigrationCheckpointStore inner, CancellationTokenSource stopping) : IMigrationCheckpointStore +{ + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => inner.ReadAll(cancellationToken); + + public Task<MigrationCheckpoint> Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public async Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var stored = await inner.Upsert(checkpoint, cancellationToken); + + if (stored.State.IsFinished()) + { + await stopping.CancelAsync(); + throw new OperationCanceledException(stopping.Token); + } + + return stored; + } +} + +// Holds the copy open at a moment a test can observe, which is the only way to ask what the API answers mid-copy. +sealed class ParkingMigrationTarget(IMigrationTarget inner, TaskCompletionSource parked, Task release) : IMigrationTarget +{ + int writes; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref writes, 1) == 0) + { + parked.SetResult(); + await release.WaitAsync(cancellationToken); + } + + return await inner.Write(category, batch, checkpointToExtend, cancellationToken); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => inner.ResetForRetry(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary<string, Type> DocumentTypes => inner.DocumentTypes; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs new file mode 100644 index 0000000000..a280f2d58a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TrialEndDateMigrationTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class TrialEndDateMigrationTests : MigrationAcceptanceTest +{ + static readonly MigrationCategory TrialEndDateCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.TrialEndDate)!; + + [Test] + public async Task Copies_the_trial_end_date_as_one_row_in_settings() + { + var endDate = new DateOnly(2027, 3, 15); + await SeedSourceTrialEndDate(endDate); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.Null, "the target holds no trial end date before the copy, so the date read afterwards came from the copy"); + + var result = await CopyCategory("TrialEndDate"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await TrialLicenseDataProvider.GetTrialEndDate(), Is.EqualTo(endDate), "the product's own trial licence store must read back the seeded date"); + Assert.That(await Target.Count(TrialEndDateCategory), Is.EqualTo(1), "--migration-verify reads this count"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs new file mode 100644 index 0000000000..c3d218938d --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_batch_exceeds_one_statement.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +// Proves a copy larger than one batch, where one batch is as many rows as one insert statement takes. +[TestFixture] +[NonParallelizable] +class When_a_batch_exceeds_one_statement : MigrationAcceptanceTest +{ + [Test] + public async Task Copies_more_messages_than_fit_in_one_statement() + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var batchSize = await Target.BatchSizeFor(category); + var failures = Enumerable.Range(0, (batchSize * 2) + 1).Select(_ => new IngestedFailure()).ToArray(); + + await SeedSourceWithBodies([.. failures.Select(failure => (failure.ToFailedMessage(), failure.Body, failure.ContentType))]); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(result.Copied, Is.EqualTo(failures.Length)); + + foreach (var failure in failures) + { + Assert.That(await FindFailedMessage(failure.UniqueMessageId), Is.Not.Null); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs new file mode 100644 index 0000000000..c5fc9dfa59 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; + +[TestFixture] +[NonParallelizable] +class When_a_copy_is_killed_mid_category : MigrationAcceptanceTest +{ + [Test] + public void A_host_not_migrating_stopped_while_starting_still_fails() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync<OperationCanceledException>(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); + } + + // Only a stop is a clean exit; a cancellation nobody asked for is still a failed start, migrating or not. + [Test] + public void A_migrating_host_whose_start_is_cancelled_without_a_stop_still_fails() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync<OperationCanceledException>(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(CancelledWithoutAStop), cancellation.Token)); + } + + [Test] + public async Task It_resumes_rather_than_starting_over() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + // Bounded rather than None: a copy that did not fail would otherwise hold this call open forever. + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var killed = Assert.ThrowsAsync<Exception>(async () => + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + Assert.That(killed, Is.Not.Null); + Assert.That(await TargetEndpointSettingsCount(), Is.EqualTo(2), "the first batch and its cursor should have committed together"); + + var refusedAgain = Assert.ThrowsAsync<Exception>(async () => + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + Assert.That(refusedAgain.Message, Is.EqualTo(killed.Message), "a start re-ran a Failed category the operator had not put back"); + + // The row a kill part way through the category leaves: still copying, with the first batch's cursor and counts. + var halted = await ReadCheckpoint("EndpointSettings"); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null })); + + await MigrateThenRunHost(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await GetEndpointSettingsNames(), Is.EquivalentTo(new[] { "A", "B", "C", "D", "E" }), "no gaps"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(5), "no row copied twice"); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero, "a resumed run must not re-read rows the first run committed"); + } + } + + // Proves the already-present count in the test above can fail: both runs end with five correct rows, so that count is the only thing telling a resume from a fresh copy. + [Test] + public async Task Without_its_checkpoint_the_second_run_re_reads_what_the_first_already_wrote() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.ThrowsAsync<Exception>(async () => + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + await DeleteCheckpoint("EndpointSettings"); + await MigrateThenRunHost(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + } + + // A copy stopped between two categories must still list the second, or every gate reading the table finds nothing outstanding. + [Test] + public async Task A_worker_started_after_a_copy_stopped_between_categories_names_the_one_it_never_reached() + { + await SeedSourceKnownEndpoints("A", "B"); + await SeedSourceEndpointSettings(("A", true), ("B", true)); + Settings.MaximumConcurrencyLevel = 1; + + using var stopping = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + Assert.CatchAsync<OperationCanceledException>(async () => + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.StopAfterTheFirstCategorySettles(stopping)), stopping.Token)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var worker = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + try + { + using (Assert.EnterMultipleScope()) + { + Assert.That((await ReadCheckpoint(MigrationCategoryIds.KnownEndpoints))?.State, Is.EqualTo(MigrationCategoryState.Complete), "the stop has to land between the two categories for this test to mean anything"); + Assert.That(async () => await worker.StartAsync(cancellation.Token), + Throws.Exception.With.Message.Contain("EndpointSettings is Copying (NotStarted)").And.Message.Not.Contain("KnownEndpoints is"), + "a worker let into a database whose copy stopped after its first category"); + Assert.That((await ReadCheckpoint(MigrationCategoryIds.EndpointSettings))?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + } + } + finally + { + await worker.DisposeAsync(); + } + } + + Task<int> TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + Task<string[]> GetEndpointSettingsNames() => + QueryTarget(dbContext => dbContext.EndpointSettings.Select(settings => settings.Name).ToArrayAsync()); + + static void StoppedWhileStarting(WebApplicationBuilder builder) => + builder.Services.AddHostedService(provider => new StopsTheHostWhileStarting(provider.GetRequiredService<IHostApplicationLifetime>())); + + static void CancelledWithoutAStop(WebApplicationBuilder builder) => + builder.Services.AddHostedService(_ => new CancelsWhileStarting()); + + sealed class CancelsWhileStarting : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) => throw new OperationCanceledException(); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + // What a service stop or the critical-error handler does to a host that is still starting. + sealed class StopsTheHostWhileStarting(IHostApplicationLifetime lifetime) : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) + { + lifetime.StopApplication(); + throw new OperationCanceledException(lifetime.ApplicationStopping); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs new file mode 100644 index 0000000000..ee4948759f --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs @@ -0,0 +1,266 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[NonParallelizable] +class When_a_startup_check_fails : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + + [Test] + public async Task An_optional_category_window_that_is_not_a_time_span_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the optional category windows are valid").And.Contain("a week")); + } + + [Test] + public async Task A_retry_history_depth_that_empties_the_migrated_table_is_refused() + { + // Assigned rather than set as an environment variable: Settings reads RetryHistoryDepth once, in the constructor the fixture has already run. + Settings.RetryHistoryDepth = 0; + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the retry history depth will not empty a migrated table") + .And.Contain("RetryHistoryDepth") + .And.Contain("HistoricRetryOperations")); + } + + [Test] + public async Task A_target_whose_schema_migrations_were_never_applied_is_refused() + { + await QueryTarget(async dbContext => + { + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService<IHistoryRepository>(); + var applied = await history.GetAppliedMigrationsAsync(); + + foreach (var row in applied) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + return applied.Count; + }); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the target schema is current").And.Contain("--setup")); + } + + [Test] + public async Task A_target_that_already_holds_data_is_refused() + { + await QueryTarget(dbContext => + { + dbContext.Settings.Add(new SettingEntity { Key = "NotificationEmails", Value = "{}" }); + return dbContext.SaveChangesAsync(); + }); + var settingsTable = await QueryTarget(dbContext => Task.FromResult(dbContext.Model.FindEntityType(typeof(SettingEntity))!.GetTableName()!)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration target holds no ServiceControl data").And.Contain(settingsTable).And.Contain("--setup")); + } + + [Test] + public async Task Message_body_storage_that_cannot_be_written_to_is_refused() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + ((FileSystemBodyStorageSettings)EFSettings.BodyStorage).StoragePath = Path.Combine(parentThatIsAFile, "bodies"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("message body storage is writable")); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + [Test] + public async Task A_client_certificate_that_has_expired_is_refused() + { + var notBefore = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero); + var notAfter = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEBASE64", ExpiredClientCertificate(notBefore, notAfter)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain($"{notBefore.UtcDateTime:u} to {notAfter.UtcDateTime:u}")); + } + + [Test] + public async Task A_throughput_database_that_does_not_exist_is_refused() + { + var missing = $"{Source.ThroughputDatabase}-does-not-exist"; + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", missing); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain(missing) + .And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + } + + // Without this check a build that copies the required set commits a customer to SQL before the rest of the migration exists. + [Test] + public async Task A_build_without_the_whole_migration_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(allowUnreleasedMigration: false); + + Assert.That(refusal.Message, Does.Contain("this build carries the whole migration") + .And.Not.Contain("the optional category windows are valid"), + "this refusal comes first, so no other check is consulted on an unreleased build"); + } + + [Test] + public async Task A_halted_required_category_stops_the_host_rather_than_opening_it() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var exception = Assert.ThrowsAsync<Exception>(async () => + await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.HaltTheEndpointSettingsCategory()), cancellation.Token)); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain("EndpointSettings").And.Contain("Halted")); + Assert.That(exception.Message, Does.Contain("nothing has been lost")); + Assert.That(exception.Message, Does.Contain("--migration-retry EndpointSettings").And.Contain("--migration-abandon EndpointSettings"), "a Failed category waits for the operator, so the refusal has to name what moves it on"); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf<HttpRequestException>(), "Kestrel bound its socket behind a halted required category"); + }); + } + + [Test] + public async Task A_source_that_will_not_open_once_the_required_copy_settled_still_lets_the_host_open() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await MigrateThenRunHost(); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + await MigrateThenRunHost(); + + var endpointSettings = await GetEndpointSettings(); + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + var archived = await ReadCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpointSettings.Select(setting => setting.Name), Does.Contain("Sales")); + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(eventLog?.LastError, Does.Contain(missing).And.Not.Contain("will not start"), "status reads the outage from the row, on a start that did open"); + Assert.That(archived?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(archived?.LastError, Does.Contain(missing)); + } + } + + [Test] + public async Task A_source_that_will_not_open_before_the_required_copy_settled_still_refuses() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + await SeedCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + var refusal = await RefusedStartup(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("the migration source opens").And.Contain(missing)); + Assert.That(refusal.Message, Does.Contain("--migration-abandon"), "a required category that never started has no exit once the source is gone, and the refusal has to say so"); + Assert.That(await ReadCheckpoint(MigrationCategoryIds.EventLog), Is.Null, "a refusal writes nothing"); + } + } + + [Test] + public async Task A_failed_optional_row_does_not_stop_the_host_opening() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await MigrateThenRunHost(); + + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.CompleteWithErrors); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", $"{Source.PrimaryDatabase}-does-not-exist"); + + await RunHostUntilTheApiAnswers(); + + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(eventLog?.LastError, Is.Null, "the outage leaves a Failed row alone"); + } + } + + // Every refusal is asserted against a source that has rows to copy: an empty target proves nothing otherwise. + // Only the release gate's own test withholds the marker; every other refusal has to get past it. + async Task<Exception> RefusedStartup(bool allowUnreleasedMigration = true) + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var refusal = Assert.ThrowsAsync<Exception>(async () => + await MigrateCommand.Run(Settings, allowUnreleasedMigration ? AllowingAnUnreleasedMigration() : null, cancellation.Token)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("this run has copied nothing")); + Assert.That(await TargetEndpointSettingsCount(), Is.Zero, "a check that fires after rows have moved is worse than no check"); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing, so the abort is still free"); + } + + return refusal; + } + + Task<int> TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + static string ExpiredClientCertificate(DateTimeOffset notBefore, DateTimeOffset notAfter) + { + using var key = RSA.Create(2048); + var request = new CertificateRequest("CN=sc-migration-source-expired", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = request.CreateSelfSigned(notBefore, notAfter); + + return Convert.ToBase64String(certificate.Export(X509ContentType.Pkcs12)); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs new file mode 100644 index 0000000000..539c9b12f7 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_categories_change_between_restarts : MigrationAcceptanceTest +{ + [Test] + public async Task Changing_the_windows_between_restarts_deletes_nothing() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await MigrateThenRunHost(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "a disabled optional category has no checkpoint"); + var firstRun = await ReadCheckpoint("EndpointSettings"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "normal startup does not select new migration categories; it uses the rows the offline command seeded"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await RunHostUntilTheApiAnswers(); + + var endpointSettings = await ReadCheckpoint("EndpointSettings"); + + Assert.Multiple(() => + { + Assert.That(endpointSettings.CopiedCount, Is.EqualTo(1), "a finished category must not be copied again when the selection changes around it"); + Assert.That(endpointSettings.Cursor, Is.Not.Null, "changing the selection must not delete or reset what an earlier run recorded"); + Assert.That(endpointSettings.Version, Is.EqualTo(firstRun.Version), "a row saved again, even unchanged, would have moved its version"); + Assert.That(endpointSettings.StartedAt, Is.EqualTo(firstRun.StartedAt), "a copy run again from scratch would have a new start"); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs new file mode 100644 index 0000000000..916b3422c5 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_endpoint_settings_are_migrated : MigrationAcceptanceTest +{ + [Test] + public async Task The_source_server_starts_and_creates_both_databases() + { + var source = await MigrationSourceServer.CreateDatabases(); + + Assert.That(source.ServerUrl, Does.StartWith("http://localhost:")); + Assert.That(source.PrimaryDatabase, Is.Not.Empty); + Assert.That(source.ThroughputDatabase, Is.EqualTo($"{source.PrimaryDatabase}-throughput")); + } + + [Test] + public async Task They_are_readable_through_the_product_read_api() + { + await SeedSourceKnownEndpoints("Sales", "Billing"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", false), ("", true)); + + // A clock that never moves keeps HeartbeatEndpointSettingsSyncHostedService's twenty second + // delay from elapsing and rewriting these rows while the assertions read them. + await MigrateThenRunHost(builder => builder.Services.AddSingleton<TimeProvider>(new FakeTimeProvider())); + + var settings = await GetEndpointSettings(); + + Assert.Multiple(() => + { + Assert.That(settings.Single(row => row.Name == "Sales").TrackInstances, Is.True); + Assert.That(settings.Single(row => row.Name == "Billing").TrackInstances, Is.False); + Assert.That(settings.Single(row => row.Name == string.Empty).TrackInstances, Is.True); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs new file mode 100644 index 0000000000..b850bb4523 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; + +[TestFixture] +[NonParallelizable] +class When_the_api_is_called_during_the_required_copy : MigrationAcceptanceTest +{ + [Test] + public async Task The_api_opens_only_after_the_offline_command_finishes_and_the_operator_starts_ServiceControl() + { + await SeedSourceKnownEndpoints("Sales", "Billing", "Shipping"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", true), ("Shipping", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var copy = MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task)), cancellation.Token); + + try + { + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf<HttpRequestException>()); + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Throws.Exception.With.Message.Contains("ServiceControl will not start")); + } + finally + { + releaseTheCopy.TrySetResult(); + await copy; + } + + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf<HttpRequestException>(), + "the offline command must not start ServiceControl itself"); + + await RunHostUntilTheApiAnswers(); + var settings = await GetEndpointSettings(); + Assert.That(settings.Select(row => row.Name), Is.SupersetOf(new[] { "Sales", "Billing", "Shipping" })); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs new file mode 100644 index 0000000000..a553e8cc93 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_copy_merges_or_changes_rows.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Concurrent; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class When_the_copy_merges_or_changes_rows : MigrationAcceptanceTest +{ + [Test] + public async Task The_category_summary_counts_changed_and_merged_rows_apart_from_the_rest() + { + var retried = new IngestedFailure { MessageType = new string('m', 600) }; + await SeedSourceWithBody(retried.ToFailedMessage(numberOfAttempts: 2), retried.Body, retried.ContentType); + await SeedSourceSubscriptions( + new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"), + new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")); + var summaries = new SummaryLines(); + + await MigrateCommand.Run(MigrationSettings(), AllowingAnUnreleasedMigration(builder => builder.Services.AddSingleton<ILoggerProvider>(summaries))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(summaries.Lines, Has.Exactly(1).EqualTo((LogLevel.Warning, $"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages}: 1 copied (1 changed), 0 already present, 0 merged, nothing skipped")), + "the message type was cut, which the summary has to count even though nothing was skipped, and the dropped older attempt is not a change"); + Assert.That(summaries.Lines, Has.Exactly(1).EqualTo((LogLevel.Warning, $"{MigrationCategoryIds.Subscriptions}: 1 copied (0 changed), 0 already present, 1 merged, nothing skipped")), + "the second version is counted as already present by the checkpoint, and the summary has to show it as a merge instead"); + } + } + + // Keeps the lines MigrationStartup logs, which is where the summary is written. + sealed class SummaryLines : ILoggerProvider + { + readonly ConcurrentQueue<(LogLevel Level, string Message)> lines = new(); + + public (LogLevel Level, string Message)[] Lines => [.. lines]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(MigrationStartup).FullName ? new SummaryLogger(lines) : NullLogger.Instance; + + public void Dispose() + { + } + + sealed class SummaryLogger(ConcurrentQueue<(LogLevel Level, string Message)> lines) : ILogger + { + public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) => + lines.Enqueue((logLevel, formatter(state, exception))); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs new file mode 100644 index 0000000000..6802772bce --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_heartbeat_sync_runs_after_a_migration.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class When_the_heartbeat_sync_runs_after_a_migration : MigrationAcceptanceTest +{ + [Test] + public async Task Migrated_endpoint_settings_survive_the_sync_under_the_name_their_known_endpoint_uses() + { + var time = new FakeTimeProvider(); + + var billing = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = "BILLING01" }, Monitored = true }; + + await SeedSource(($"KnownEndpoints/{billing.EndpointDetails.GetDeterministicId()}", billing)); + await SeedSourceKnownEndpoints("SmokeSales"); + await SeedSourceEndpointSettings(("Billing", true), ("smokesales", true), ("SmokeSales", false)); + + await RunRequiredCopy(customize: services => + { + services.AddSingleton<TimeProvider>(time); + CountSyncPasses(services); + }); + + time.Advance(TimeSpan.FromSeconds(21)); + await WaitForSyncPass(time); + + var settings = await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(settings.Select(setting => setting.Name), Does.Contain(string.Empty), + "The heartbeat sync did not create the default row, which it does after its delete loop. The pass did not finish, so the other checks prove nothing."); + Assert.That(settings.Select(setting => setting.Name), Does.Contain("Billing"), + "The heartbeat sync deleted the migrated Billing setting. It only does that when no Billing known endpoint was copied."); + Assert.That(settings.Where(setting => setting.Name == "SmokeSales").Select(setting => setting.TrackInstances), Is.EqualTo(new[] { false }), + "SmokeSales must keep the operator's Do Not Track. On a target whose name column ignores case, the two settings become one row. If the copy keeps 'smokesales', the sync deletes it and SmokeSales gets the default."); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs new file mode 100644 index 0000000000..8a8523bc72 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs @@ -0,0 +1,250 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Hosting.Commands; + +[TestFixture] +// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set +// process-global environment variables. One fixture added without it makes the whole suite intermittent. +[NonParallelizable] +class When_the_host_opens_on_the_target : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + const string CategoryFromANewerBuild = "CategoryFromANewerBuild"; + + [Test] + public async Task The_row_does_not_exist_while_the_required_copy_is_still_running() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var copy = MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task)), cancellation.Token); + + try + { + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the abort is still free while the offline copy runs"); + } + finally + { + releaseTheCopy.TrySetResult(); + await copy; + } + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the offline command never opens the service"); + + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); + } + + [Test] + public async Task A_start_detects_and_stamps_a_migration_without_an_enable_setting() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); + } + + [Test] + public async Task A_start_requires_the_checkpoint_schema_to_check_for_an_unfinished_migration() + { + await DropTheCheckpointTable(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), Throws.Exception); + } + + [Test] + public async Task An_ingestion_only_host_is_refused_while_a_required_category_is_failed() + { + Settings.MaximumConcurrencyLevel = 1; + await SeedRequiredCategoriesComplete(except: MigrationCategoryIds.MessageRedirects); + await SeedCheckpoint(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync<Exception>(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("MessageRedirects is Failed (CompleteWithErrors)") + .And.Contain("--migration-retry MessageRedirects") + .And.Contain("--migration-abandon MessageRedirects")); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_starts_while_an_optional_category_is_still_copying_or_failed() + { + Settings.MaximumConcurrencyLevel = 1; + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + Assert.That(async () => await app.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds a worker back"); + await app.StopAsync(cancellation.Token); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_is_refused_over_a_category_this_build_does_not_know() + { + Settings.MaximumConcurrencyLevel = 1; + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(CategoryFromANewerBuild, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync<Exception>(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain($"{CategoryFromANewerBuild} is Copying (InProgress)"), "nobody here can say whether an id this build does not hold is optional, so it holds the worker back"); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task Importing_failed_errors_is_refused_while_a_copy_is_unfinished() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + var refusal = Assert.ThrowsAsync<Exception>(async () => await host.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("--import-failed-errors will not start").And.Contain("KnownEndpoints is Copying (InProgress)")); + } + + [Test] + public async Task Importing_failed_errors_starts_while_an_optional_category_is_still_copying_or_failed() + { + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + Assert.That(async () => await host.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds the import back"); + await host.StopAsync(cancellation.Token); + } + + // A customer whose migration never started must not be told they have passed the point of no return. + [Test] + public async Task The_row_does_not_exist_after_a_startup_check_refused() + { + // Seeded so the marker would be written if anything stamped it, which makes the refusal the only reason it is not. + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "NoSuchWindow"); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.That(async () => await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Throws.Exception.With.Message.Contains("NoSuchWindow")); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing"); + } + + // Ingestion-only hosts write failed messages into the same database a migration targets, so a rollback + // gate that had not seen them would discard everything they ingested. + [Test] + public async Task An_ingestion_only_host_records_the_row_too() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + // The ingestion-only host runs a real transport, which the migration fixture does not otherwise configure. + Settings.MaximumConcurrencyLevel = 1; + + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null, "only RunCommand used to record this, so a scaled-out ingestion host wrote to the target and left no trace of having opened on it"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + + // A customer running on SQL who has never migrated must not be told the way back is gone. + [Test] + public async Task A_database_no_migration_has_touched_is_not_stamped() + { + Settings.MaximumConcurrencyLevel = 1; + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "no checkpoint row exists, so no migration has ever run against this database"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + async Task SeedRequiredCategoriesComplete(string except = null) + { + foreach (var category in MigrationCategoryRegistry.All.Where(category => category.Kind == MigrationCategoryKind.Required && category.Id != except)) + { + await SeedCheckpoint(category.Id); + } + } + + Task<int> DropTheCheckpointTable() => + QueryTarget(dbContext => + { + var table = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + var name = dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(table.GetTableName()!, table.GetSchema()); + + // The name comes from the EF model and the provider delimits it, so no outside input reaches this SQL. +#pragma warning disable EF1003 + return dbContext.Database.ExecuteSqlRawAsync("DROP TABLE " + name); +#pragma warning restore EF1003 + }); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs new file mode 100644 index 0000000000..0e805ce94a --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_offline_migration_logs.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure; + +[TestFixture] +[NonParallelizable] +class When_the_offline_migration_logs : MigrationAcceptanceTest +{ + [Test] + public void It_logs_only_through_the_providers_the_instance_is_configured_with() + { + string[] registered = null; + + Assert.ThrowsAsync<ProvidersCaptured>(async () => await MigrateCommand.Run(Settings, builder => + { + registered = ProviderTypes(builder.Services); + throw new ProvidersCaptured(); + })); + + var configured = ProviderTypes(new ServiceCollection().AddLogging(logging => logging.ConfigureLogging(Settings.LoggingSettings.LogLevel))); + + Assert.That(registered, Is.EquivalentTo(configured), + "an extra default provider prints every line twice and prints EF Core's SQL, which the configured providers filter out"); + } + + static string[] ProviderTypes(IServiceCollection services) => + [.. services + .Where(service => service.ServiceType == typeof(ILoggerProvider)) + .Select(service => (service.ImplementationType ?? service.ImplementationInstance?.GetType() ?? service.ImplementationFactory?.Method.ReturnType)?.FullName ?? "unknown")]; + + sealed class ProvidersCaptured : Exception; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs new file mode 100644 index 0000000000..0bf218a543 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_throughput_collector_runs_after_a_migration.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; + +[TestFixture] +[NonParallelizable] +class When_the_throughput_collector_runs_after_a_migration : MigrationAcceptanceTest +{ + [Test] + public async Task Copied_days_are_not_collected_a_second_time_and_added_to() + { + var time = new FakeTimeProvider(); + var yesterday = DateOnly.FromDateTime(time.GetUtcNow().UtcDateTime).AddDays(-1); + + await SeedThroughputSource(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await SeedThroughputTimeSeries("Sales.Orders/Broker", yesterday, 100); + BrokerReports(queue: "sales.orders", day: yesterday, count: 70); + + await RunRequiredCopy(customize: services => + { + services.AddSingleton<TimeProvider>(time); + CollectFromTheBrokerImmediately(services); + }); + + await WaitForCollectorPass(); + + var total = await QueryTarget(dbContext => dbContext.LicensingEndpointThroughput + .Where(row => row.NormalizedName == "sales.orders" && row.ThroughputSource == ThroughputSource.Broker && row.DateUtc == yesterday) + .Select(row => row.MessageCount) + .SingleAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(BrokerRequests, Is.EqualTo(new[] { ("sales.orders", yesterday.AddDays(1)) }), + "the collector asks only for days after the last copied one, so it never tried to collect the copied day again and a failed write cannot hide that it did"); + Assert.That(total, Is.EqualTo(100), + "100 is the copied count; 70 means the copy wrote nothing and the collector collected the day itself; 170 means the collector re-collected a copied day and the additive write path added the two, so the licensing report overstates usage"); + } + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs new file mode 100644 index 0000000000..25c51dc849 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_unresolved_failed_messages_are_migrated.cs @@ -0,0 +1,391 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using Raven.Client.Documents.Operations; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure.WebApi; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.Tests; + +[TestFixture] +[NonParallelizable] +class When_unresolved_failed_messages_are_migrated : MigrationAcceptanceTest +{ + [Test] + public async Task Fills_every_column_the_newest_attempt_supplies() + { + var failure = new IngestedFailure { FailingEndpointAddress = "Sales@MACHINE" }; + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.MessageId, Is.EqualTo(failure.MessageId)); + Assert.That(row.MessageType, Is.EqualTo(failure.MessageType)); + Assert.That(row.TimeSent, Is.EqualTo(failure.TimeSent)); + Assert.That(row.ConversationId, Is.EqualTo(failure.ConversationId)); + Assert.That(row.IsSystemMessage, Is.False); + Assert.That(row.SendingEndpointName, Is.EqualTo(failure.SendingEndpoint.Name)); + Assert.That(row.SendingEndpointHostId, Is.EqualTo(failure.SendingEndpoint.HostId)); + Assert.That(row.SendingEndpointHost, Is.EqualTo(failure.SendingEndpoint.Host)); + Assert.That(row.ReceivingEndpointName, Is.EqualTo(failure.ReceivingEndpoint.Name)); + Assert.That(row.ReceivingEndpointHostId, Is.EqualTo(failure.ReceivingEndpoint.HostId)); + Assert.That(row.ReceivingEndpointHost, Is.EqualTo(failure.ReceivingEndpoint.Host)); + Assert.That(row.ExceptionType, Is.EqualTo(failure.ExceptionType)); + Assert.That(row.ExceptionMessage, Is.EqualTo(failure.ExceptionMessage)); + Assert.That(row.FailingEndpointAddress, Is.EqualTo("Sales@MACHINE")); + Assert.That(row.HeadersJson, Does.Contain(failure.MessageId)); + } + } + + [Test] + public async Task Collapses_attempt_history_to_the_newest_attempt() + { + var failure = new IngestedFailure(); + await SeedFailure(failure, numberOfAttempts: 5); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(5), "the count survives even though the attempts do not"); + Assert.That(row.LastAttemptedAt, Is.EqualTo(failure.AttemptedAt)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(failure.AttemptedAt.AddMinutes(-4))); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(failure.TimeOfFailure)); + } + } + + [Test] + public async Task A_resumed_copy_leaves_the_rows_the_first_run_wrote() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + var first = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + await QueryTarget(async db => + { + var existing = await db.FailedMessages.SingleAsync(row => row.UniqueMessageId == failure.UniqueMessageId); + existing.ExceptionMessage = "Edited after the first copy, the way a live instance would have"; + await db.SaveChangesAsync(); + return existing; + }); + + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(1)); + Assert.That(row.ExceptionMessage, Is.EqualTo("Edited after the first copy, the way a live instance would have"), "the copier overwrote a row that was already in the target"); + Assert.That(second.Copied, Is.Zero); + Assert.That(second.AlreadyPresent, Is.EqualTo(1), "a row the target already holds is a third outcome"); + Assert.That(second.Skipped, Is.Zero, "counting it as a skip would push a healthy category toward the halt threshold"); + Assert.That(second.SkippedIds, Is.Empty); + } + } + + [Test] + public async Task Counts_only_the_statuses_this_category_owns() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + await SeedFailure(unresolved); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + // Written straight into the target, because this build has no category that copies archived messages. + await QueryTarget(async db => + { + db.FailedMessages.Add(new FailedMessageEntity + { + UniqueMessageId = Guid.NewGuid(), + Status = FailedMessageStatus.Archived, + HeadersJson = "{}", + FailingEndpointAddress = "error" + }); + + return await db.SaveChangesAsync(); + }); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await Target.Count(category), Is.EqualTo(2), "the category's count takes in the archive's rows unless it honours the category's statuses"); + Assert.That(await QueryTarget(db => db.FailedMessages.CountAsync()), Is.EqualTo(3), "the filter is doing the work, not an empty table"); + } + } + + [Test] + public async Task A_copied_message_keeps_its_failure_groups() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var groups = await QueryTarget(db => db.FailedMessageGroups + .AsNoTracking() + .Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId) + .ToListAsync()); + var expected = failure.Groups.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(groups, Has.Count.EqualTo(1), "a message copied without its group rows is missing from every group view"); + Assert.That(groups.Single().GroupId, Is.EqualTo(expected.Id)); + Assert.That(groups.Single().Title, Is.EqualTo(expected.Title)); + Assert.That(groups.Single().Type, Is.EqualTo(expected.Type)); + } + } + + [Test] + public async Task A_retry_by_id_after_the_copy_stages_the_unresolved_message_and_not_the_retry_issued_one() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + await SeedFailure(unresolved); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); + + var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + Assert.That(copy.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + + string[] ids = [unresolved.UniqueMessageIdString, retryIssued.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-by-id", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EquivalentTo(new[] { unresolved.UniqueMessageIdString }), + "RavenDB ignores a retry of a message whose first retry is still outstanding, so after the move it must not be sent again"); + } + + [Test] + public async Task A_retry_issued_message_resent_from_pending_retries_after_the_copy_is_staged() + { + var retryIssued = new IngestedFailure(); + await SeedFailure(retryIssued, FailedMessageStatus.RetryIssued); + await SeedSourceRetryClaim(retryIssued.UniqueMessageIdString); + + var copy = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(copy.Copied, Is.EqualTo(1), "the test proves nothing unless the message reached the target"); + Assert.That(await QueryTarget(db => db.FailedMessageRetries.CountAsync(claim => claim.UniqueMessageId == retryIssued.UniqueMessageId)), Is.EqualTo(1), + "the test proves nothing about releasing a migrated claim unless the copy wrote one"); + } + + // Pending Retries releases the claim, then sends a retry by id. + await FailedMessageRetryDataStore.RemoveFailedMessageRetry(retryIssued.UniqueMessageIdString); + string[] ids = [retryIssued.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-pending", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EqualTo(ids), + "a migrated claim that the release by message id cannot remove leaves the message unretryable for good"); + } + + [Test] + public async Task A_message_already_in_the_target_gets_no_group_rows_from_the_copy() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + await QueryTarget(db => db.FailedMessageGroups.Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId).ExecuteDeleteAsync()); + await DeleteCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var second = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var groups = await QueryTarget(db => db.FailedMessageGroups.AsNoTracking().CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.AlreadyPresent, Is.EqualTo(1)); + Assert.That(groups, Is.Zero, "the copy wrote RavenDB's groups onto a message it did not insert"); + } + } + + [Test] + public async Task Answers_for_a_copied_message_on_the_URL_it_had_on_RavenDB() + { + var failure = new IngestedFailure(); + await SeedFailure(failure); + + await MigrateThenRunHost(); + + var response = await HttpClient.GetAsync($"api/errors/last/{failure.UniqueMessageIdString}"); + + Assert.That(response.IsSuccessStatusCode, Is.True, "the bookmark a customer had on RavenDB stopped resolving"); + + var fromApi = await response.Content.ReadFromJsonAsync<FailedMessageView>(SerializerOptions.Default); + + using (Assert.EnterMultipleScope()) + { + Assert.That(fromApi.Id, Is.EqualTo(failure.UniqueMessageIdString)); + Assert.That(fromApi.Status, Is.EqualTo(FailedMessageStatus.Unresolved)); + Assert.That(fromApi.MessageType, Is.EqualTo(failure.MessageType)); + Assert.That(fromApi.ReceivingEndpoint.Name, Is.EqualTo(failure.ReceivingEndpoint.Name)); + Assert.That(fromApi.QueueAddress, Is.EqualTo(failure.QueueAddress)); + Assert.That(fromApi.NumberOfProcessingAttempts, Is.EqualTo(1)); + } + } + + [Test] + public async Task The_required_copy_completes_the_category_before_the_host_opens() + { + var failures = Enumerable.Range(0, 3).Select(_ => new IngestedFailure()).ToArray(); + + foreach (var failure in failures) + { + await SeedFailure(failure); + } + + await MigrateThenRunHost(); + + var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SourceTotal, Is.Null, "the engine no longer counts the source before copying"); + } + } + + [Test] + public async Task Records_each_skip_under_the_reason_that_caused_it() + { + var good = new IngestedFailure(); + await SeedFailure(good); + + var unparseable = new IngestedFailure(); + var unparseableDocument = unparseable.ToFailedMessage(); + unparseableDocument.UniqueMessageId = "not-a-guid"; + await SeedSourceWithBody(unparseableDocument, unparseable.Body, unparseable.ContentType); + + var attemptless = new IngestedFailure(); + var attemptlessDocument = attemptless.ToFailedMessage(); + attemptlessDocument.ProcessingAttempts.Clear(); + await SeedSourceWithBody(attemptlessDocument, attemptless.Body, attemptless.ContentType); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var refused = Assert.ThrowsAsync<Exception>(async () => await MigrateCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + + var checkpoint = await ReadCheckpoint("UnresolvedAndRetryIssuedFailedMessages"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refused.Message, Does.Contain("UnresolvedAndRetryIssuedFailedMessages").And.Contain("--migration-retry").And.Contain("--migration-abandon"), "a fault skip leaves a required category Failed, so the host stays closed and names both commands"); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.UnparseableUniqueMessageId] = 1, + [MigrationSkipReason.NoProcessingAttempts] = 1 + })); + } + } + + [Test] + public async Task A_document_whose_TimeSent_cannot_be_read_is_skipped_as_malformed_and_one_whose_host_id_is_not_a_GUID_copies_with_the_empty_host_id() + { + var good = new IngestedFailure(); + await SeedFailure(good); + + var badTimeSent = new IngestedFailure(); + var badTimeSentDocument = badTimeSent.ToFailedMessage(); + badTimeSentDocument.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = "garbage"; + + var badHostId = new IngestedFailure(); + var badHostIdDocument = badHostId.ToFailedMessage(); + var attempt = badHostIdDocument.ProcessingAttempts[^1]; + attempt.Headers[NServiceBus.Headers.HostId] = "not-a-guid"; + attempt.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + await SeedSourceWithBodies( + (badTimeSentDocument, badTimeSent.Body, badTimeSent.ContentType), + (badHostIdDocument, badHostId.Body, badHostId.ContentType)); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Saved.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the copy reaches the end of the source instead of halting at the first bad document"); + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{badTimeSent.UniqueMessageIdString}" })); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(await QueryTarget(db => db.FailedMessages.Select(row => row.UniqueMessageId).ToListAsync()), Is.EquivalentTo(new[] { good.UniqueMessageId, badHostId.UniqueMessageId })); + var copied = await QueryTarget(db => db.FailedMessages.Where(row => row.UniqueMessageId == badHostId.UniqueMessageId).SingleAsync()); + Assert.That(copied.ReceivingEndpointHostId, Is.EqualTo(Guid.Empty), "ingestion stores an unparseable host id as the empty GUID, so the message copies with it"); + Assert.That(copied.ReceivingEndpointName, Is.EqualTo("Sales"), "the endpoint is rebuilt from the headers, not taken from the stored dictionary"); + Assert.That(copied.ReceivingEndpointHost, Is.EqualTo("MACHINE")); + } + } + + [Test] + public async Task A_failure_stored_without_its_time_is_skipped_when_newest_and_left_out_of_the_failure_times_when_older() + { + var newestHasNoTime = new IngestedFailure(); + var olderHasNoTime = new IngestedFailure(); + await SeedFailure(newestHasNoTime); + await SeedFailure(olderHasNoTime, numberOfAttempts: 2); + await DeleteStoredTimeOfFailure(newestHasNoTime, attempt: 0); + await DeleteStoredTimeOfFailure(olderHasNoTime, attempt: 0); + + var result = await CopyCategory(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var copied = await GetFailedMessage(olderHasNoTime.UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{newestHasNoTime.UniqueMessageIdString}" }), "a missing time read back as the copy's own clock, so the message copied with a made-up failure time"); + Assert.That(result.Saved.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 })); + Assert.That(copied.FirstTimeOfFailure, Is.EqualTo(olderHasNoTime.TimeOfFailure), "the older attempt has no time, so only the newest one counts"); + Assert.That(copied.LastTimeOfFailure, Is.EqualTo(olderHasNoTime.TimeOfFailure), "the copy's own clock would become the last failure time"); + } + } + + // Removes the key itself, because a stored default date is a different case that the source already reads correctly. + Task DeleteStoredTimeOfFailure(IngestedFailure failure, int attempt) => + SourceStore.Operations.ForDatabase(Source.PrimaryDatabase).SendAsync(new PatchOperation( + $"FailedMessages/{failure.UniqueMessageIdString}", + null, + new PatchRequest { Script = $"delete this.ProcessingAttempts[{attempt}].FailureDetails.TimeOfFailure;" })); + + // Seeded with its body, because the copy leaves out a message whose body the source does not hold. + Task SeedFailure(IngestedFailure failure, FailedMessageStatus status = FailedMessageStatus.Unresolved, int numberOfAttempts = 1) => + SeedSourceWithBody(failure.ToFailedMessage(status, numberOfAttempts), failure.Body, failure.ContentType); +} diff --git a/src/ServiceControl.Migration.Tests/.editorconfig b/src/ServiceControl.Migration.Tests/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Migration.Tests/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs new file mode 100644 index 0000000000..1bdc4ed89f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs @@ -0,0 +1,100 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationCategoryCoverageTests +{ + Settings settings; + + [SetUp] + public void SetUp() + { + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", + Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n"))); + + // SQL Server stands in for PostgreSQL too, because both register the same EFCoreMigrationTarget and its writer list does not depend on the provider. + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + // A reader with no writer, or the reverse, is a category the intersection would otherwise drop in silence. + [Test] + public async Task Every_reader_has_a_writer_and_the_reverse() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + var sourceCategoryIds = source.SupportedCategoryIds; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService<IMigrationTarget>(); + + var readerOnly = sourceCategoryIds.Except(target.SupportedCategoryIds).ToArray(); + var writerOnly = target.SupportedCategoryIds.Except(sourceCategoryIds).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(readerOnly, Is.Empty, $"the reader claims a category the writer does not: {string.Join(", ", readerOnly)}"); + Assert.That(writerOnly, Is.Empty, $"the writer claims a category the reader does not: {string.Join(", ", writerOnly)}"); + } + } + + [Test] + public async Task Every_reader_and_its_writer_agree_on_the_document_type() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService<IMigrationTarget>(); + + var shared = source.DocumentTypes.Keys.Intersect(target.DocumentTypes.Keys).ToArray(); + Assert.That(shared, Is.Not.Empty, "the test proves nothing if no category has both a reader and a writer"); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in shared) + { + Assert.That(target.DocumentTypes[categoryId], Is.EqualTo(source.DocumentTypes[categoryId]), $"the {categoryId} reader yields a document type its writer does not take"); + } + } + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List<string> variables = []; +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs new file mode 100644 index 0000000000..737a26bd4f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Hosting.Commands; + +[TestFixture] +[NonParallelizable] +class MigrationSourceReportCommandTests +{ + [SetUp] + public void SetUp() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", null); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", null); + } + + [Test] + public async Task The_report_names_the_databases_the_settings_and_the_collections() + { + var report = await RunReport(); + + Assert.Multiple(() => + { + Assert.That(report, Does.Match(@"Mode\s+: external")); + Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); + Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); + Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); + }); + } + + static async Task<string> RunReport() + { + var settings = new Settings(persisterType: "RavenDB", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + + var writer = new StringWriter(); + var original = Console.Out; + Console.SetOut(writer); + + try + { + await new MigrationSourceReportCommand().Execute(new HostArguments([]), settings); + } + finally + { + Console.SetOut(original); + } + + return writer.ToString(); + } +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs new file mode 100644 index 0000000000..9b6e933c03 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence; +using ServiceControl.RavenDB; +using TestHelper; + +/// <summary> +/// One embedded RavenDB for the whole assembly, holding a seeded database that stands in for a +/// customer's old instance. EmbeddedServer.Instance is process wide, so there can only be one. +/// </summary> +static class MigrationSourceServer +{ + public static string ServerUrl { get; private set; } + + public static string PrimaryDatabase { get; private set; } + + public static string ThroughputDatabase { get; private set; } + + public static async Task Start(CancellationToken cancellationToken = default) + { + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Data"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Logs"); + var port = PortUtility.GetAssignedOrAvailablePort(33377); + + ServerUrl = $"http://localhost:{port}"; + PrimaryDatabase = "primary"; + ThroughputDatabase = "throughput"; + + var configuration = new EmbeddedDatabaseConfiguration(ServerUrl, PrimaryDatabase, dbPath, logPath, "Operations") { RunInMemory = true }; + database = EmbeddedDatabase.Start(configuration, lifetime); + + using var store = await database.Connect(cancellationToken); + + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(PrimaryDatabase)), cancellationToken); + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(ThroughputDatabase)), cancellationToken); + + using var seeder = new DocumentStore + { + Urls = [ServerUrl], + Database = PrimaryDatabase, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + using var session = seeder.OpenAsyncSession(); + await session.StoreAsync(new EndpointSettings { Name = "Sales", TrackInstances = true }, "EndpointSettings/1", cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + + public static async Task Stop() + { + if (database is null) + { + return; + } + + await database.Stop(CancellationToken.None); + database.Dispose(); + database = null; + lifetime.StopApplication(); + } + + static EmbeddedDatabase database; + static readonly TestLifetime lifetime = new(); + + sealed class TestLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => stopping.Cancel(); + + readonly CancellationTokenSource stopping = new(); + } +} + +[SetUpFixture] +public class MigrationSourceServerFixture +{ + [OneTimeSetUp] + public Task StartSource() => MigrationSourceServer.Start(); + + [OneTimeTearDown] + public Task StopSource() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj new file mode 100644 index 0000000000..da4cda3938 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj @@ -0,0 +1,25 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <TestCategory>Migration</TestCategory> + </PropertyGroup> + + <!-- References no persister, so both resolve by manifest name into their own PluginAssemblyLoadContext. + ServiceControl.RavenDB is here only for the embedded server MigrationSourceServer starts. --> + <ItemGroup> + <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" /> + <ProjectReference Include="..\ServiceControl.Persistence\ServiceControl.Persistence.csproj" /> + <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" /> + <ProjectReference Include="..\TestHelper\TestHelper.csproj" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="GitHubActionsTestLogger" /> + <PackageReference Include="Microsoft.NET.Test.Sdk" /> + <PackageReference Include="NUnit" /> + <PackageReference Include="NUnit.Analyzers" /> + <PackageReference Include="NUnit3TestAdapter" /> + </ItemGroup> + +</Project> diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs new file mode 100644 index 0000000000..b246735641 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -0,0 +1,103 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Loader; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class TwoPersistersInOneProcessTests +{ + Settings settings; + string bodyStoragePath; + + [SetUp] + public void SetUp() + { + bodyStoragePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n")); + + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", + Environment.GetEnvironmentVariable("ServiceControl_Persistence_SqlServer_ConnectionString") + ?? "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", bodyStoragePath); + SetVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + SetVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + SetVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + [Test] + public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_identity() + { + Assert.That(settings.AssemblyLoadContextResolver(typeof(Settings).Assembly.Location), Is.InstanceOf<PluginAssemblyLoadContext>(), + "This test is meaningless if the resolver is pinned to AssemblyLoadContext.Default the way every acceptance test pins it."); + + var services = new ServiceCollection(); + services.AddPersistence(settings); + var targetSettings = settings.PersisterSpecificSettings; + + await using var source = await PersistenceFactory.OpenMigrationSource(settings); + + var sourceContext = AssemblyLoadContext.GetLoadContext(source.GetType().Assembly); + var targetContext = AssemblyLoadContext.GetLoadContext(settings.PersisterSpecificSettings.GetType().Assembly); + + Assert.Multiple(() => + { + Assert.That(sourceContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The RavenDB persister must load into its own plugin context, or this proves nothing."); + Assert.That(targetContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The target persister must be isolated too, otherwise only one half of the pairing is being tested."); + Assert.That(sourceContext, Is.Not.SameAs(targetContext), + "Source and target must land in separate contexts. That separation is the whole feature, and nothing else here asserts it."); + Assert.That(settings.PersisterSpecificSettings, Is.SameAs(targetSettings), + "Opening a source must leave the target's settings object exactly where it was."); + }); + + var description = await source.Describe(); + + Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + "The source read its own RavenDB settings rather than the target's."); + } + + [Test] + public void Asking_a_SQL_persister_for_maintenance_mode_is_still_refused() + { + var refusal = Assert.Throws<Exception>(() => PersistenceFactory.Create(settings, maintenanceMode: true)); + + Assert.That(refusal.Message, Does.Contain("Maintenance mode is not supported").And.Contain("SQLServer"), + "PersistenceFactory's SupportsMaintenanceMode guard is what turns a persister mismatch into a sentence instead of a cast exception. A restructure that drops it compiles and passes every other test."); + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List<string> variables = []; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..f9b6c71345 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,991 @@ +// <auto-generated /> +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20261007000244_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// <inheritdoc /> + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property<string>("RequestId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("request_id"); + + b.Property<int>("ArchiveType") + .HasColumnType("integer") + .HasColumnName("archive_type"); + + b.Property<int>("OperationType") + .HasColumnType("integer") + .HasColumnName("operation_type"); + + b.Property<int>("CurrentBatch") + .HasColumnType("integer") + .HasColumnName("current_batch"); + + b.Property<string>("GroupName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("group_name"); + + b.Property<string>("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property<string>("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_name"); + + b.Property<int>("NumberOfBatches") + .HasColumnType("integer") + .HasColumnName("number_of_batches"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property<string>("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property<DateTime>("Started") + .HasColumnType("timestamp with time zone") + .HasColumnName("started"); + + b.Property<int>("TotalNumberOfMessages") + .HasColumnType("integer") + .HasColumnName("total_number_of_messages"); + + b.HasKey("RequestId", "ArchiveType", "OperationType") + .HasName("pk_archive_operations"); + + b.HasIndex("Started") + .HasDatabaseName("ix_archive_operations_started"); + + b.ToTable("archive_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("Category") + .IsRequired() + .HasColumnType("text") + .HasColumnName("category"); + + b.Property<string>("CustomCheckId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("custom_check_id"); + + b.Property<string>("FailureReason") + .HasColumnType("text") + .HasColumnName("failure_reason"); + + b.Property<string>("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_host"); + + b.Property<Guid>("OriginatingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("originating_endpoint_host_id"); + + b.Property<string>("OriginatingEndpointName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_name"); + + b.Property<DateTime>("ReportedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("reported_at"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_custom_checks"); + + b.HasIndex("ReportedAt") + .HasDatabaseName("ix_custom_checks_reported_at"); + + b.HasIndex("Status", "ReportedAt") + .HasDatabaseName("ix_custom_checks_status_reported_at"); + + b.ToTable("custom_checks", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property<string>("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property<bool>("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id")); + + b.Property<string>("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category"); + + b.Property<string>("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property<string>("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("event_type"); + + b.Property<DateTime>("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection<List<string>>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property<int>("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("event_log_items", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id")); + + b.Property<string>("DispatchContextJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("dispatch_context_json"); + + b.Property<string>("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("dispatch_context_type_name"); + + b.HasKey("Id") + .HasName("pk_external_integration_dispatch_requests"); + + b.ToTable("external_integration_dispatch_requests", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property<byte[]>("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property<bool>("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property<string>("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property<DateTime>("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property<string>("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property<string>("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property<string>("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("edit_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_edits"); + + b.HasIndex("EditId") + .HasDatabaseName("ix_failed_message_edits_edit_id"); + + b.ToTable("failed_message_edits", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property<string>("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property<int>("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property<bool>("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property<string>("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property<string>("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property<string>("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property<string>("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property<string>("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + + b.Property<DateTime>("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property<string>("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property<bool>("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property<DateTime>("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property<DateTime>("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property<DateTime>("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property<string>("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property<string>("MessageType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_type"); + + b.Property<int>("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property<string>("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property<Guid?>("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property<string>("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property<string>("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property<Guid?>("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property<string>("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property<DateTime>("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property<DateTime?>("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastTimeOfFailure") + .HasDatabaseName("ix_failed_messages_status_last_time_of_failure"); + + b.HasIndex("Status", "LastModified", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_last_modified_unique_message_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified", "UniqueMessageId"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "MessageType", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_message_type_unique_message_id"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property<Guid>("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property<string>("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property<string>("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property<string>("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.HasIndex("Type", "GroupId") + .HasDatabaseName("ix_failed_message_groups_type_group_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "FailedMessageUniqueId", "Title" }); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property<Guid>("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.Property<int>("StageAttempts") + .HasColumnType("integer") + .HasColumnName("stage_attempts"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.HasIndex("RetryBatchId") + .HasDatabaseName("ix_failed_message_retries_retry_batch_id"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property<string>("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property<string>("Comment") + .IsRequired() + .HasColumnType("text") + .HasColumnName("comment"); + + b.HasKey("GroupId") + .HasName("pk_group_comments"); + + b.ToTable("group_comments", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<long>("Id")); + + b.Property<DateTime>("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property<bool>("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property<string>("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property<string>("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property<int>("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property<DateTime>("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("Id") + .HasName("pk_historic_retry_operations"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending() + .HasDatabaseName("ix_historic_retry_operations_completion_time_id"); + + b.ToTable("historic_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property<Guid>("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property<bool>("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property<string>("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property<int>("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.PrimitiveCollection<List<string>>("EndpointIndicators") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("endpoint_indicators"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("name"); + + b.Property<string>("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("normalized_sanitized_name"); + + b.Property<string>("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sanitized_name"); + + b.Property<string>("Scope") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("scope"); + + b.Property<string>("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("user_indicator"); + + b.HasKey("NormalizedName", "ThroughputSource") + .HasName("pk_licensing_endpoints"); + + b.HasIndex("NormalizedSanitizedName") + .HasDatabaseName("ix_licensing_endpoints_normalized_sanitized_name"); + + b.ToTable("licensing_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property<string>("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property<int>("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.Property<DateOnly>("DateUtc") + .HasColumnType("date") + .HasColumnName("date_utc"); + + b.Property<long>("MessageCount") + .HasColumnType("bigint") + .HasColumnName("message_count"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc") + .HasName("pk_licensing_endpoint_throughput"); + + b.HasIndex("DateUtc") + .HasDatabaseName("ix_licensing_endpoint_throughput_date_utc"); + + b.ToTable("licensing_endpoint_throughput", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property<string>("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("from_physical_address"); + + b.Property<DateTime>("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property<string>("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("to_physical_address"); + + b.HasKey("FromPhysicalAddress") + .HasName("pk_message_redirects"); + + b.ToTable("message_redirects", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property<string>("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property<long>("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property<long>("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property<string>("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property<string>("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property<DateTime?>("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property<DateTime?>("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property<string>("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property<long>("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property<long?>("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property<DateTime?>("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property<long?>("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property<int>("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property<long>("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("Classifier") + .HasColumnType("text") + .HasColumnName("classifier"); + + b.Property<string>("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property<int>("InitialBatchSize") + .HasColumnType("integer") + .HasColumnName("initial_batch_size"); + + b.Property<string>("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property<string>("InitiatedByName") + .HasColumnType("text") + .HasColumnName("initiated_by_name"); + + b.Property<DateTime?>("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property<string>("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property<string>("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property<string>("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("request_id"); + + b.Property<string>("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_session_id"); + + b.Property<int>("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property<string>("StagingId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("staging_id"); + + b.Property<DateTime>("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_retry_batches"); + + b.HasIndex("Status", "RetrySessionId") + .HasDatabaseName("ix_retry_batches_status_retry_session_id"); + + b.ToTable("retry_batches", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property<int>("Id") + .HasColumnType("integer") + .HasColumnName("id"); + + b.Property<Guid>("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.HasKey("Id") + .HasName("pk_retry_batch_now_forwarding"); + + b.ToTable("retry_batch_now_forwarding", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property<string>("Key") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("key"); + + b.Property<string>("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Key") + .HasName("pk_settings"); + + b.ToTable("settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property<string>("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property<string>("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property<string>("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property<string>("RequestId") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property<int>("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property<string>("Classifier") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("classifier"); + + b.Property<DateTime>("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property<bool>("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property<DateTime>("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property<string>("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property<DateTime>("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("RequestId", "RetryType") + .HasName("pk_unacknowledged_retry_operations"); + + b.ToTable("unacknowledged_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licensing_endpoint_throughput_licensing_endpoints_normalize"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..6eaf796b17 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// <inheritdoc /> + public partial class AddMigrationCheckpoints : Migration + { + /// <inheritdoc /> + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "migration_checkpoints", + columns: table => new + { + category_id = table.Column<string>(type: "character varying(450)", maxLength: 450, nullable: false), + state = table.Column<int>(type: "integer", nullable: false), + cursor = table.Column<string>(type: "text", nullable: true), + copied_count = table.Column<long>(type: "bigint", nullable: false), + skipped_count = table.Column<long>(type: "bigint", nullable: false), + source_total = table.Column<long>(type: "bigint", nullable: true), + skip_reasons = table.Column<string>(type: "text", nullable: true), + started_at = table.Column<DateTime>(type: "timestamp with time zone", nullable: true), + last_progress_at = table.Column<DateTime>(type: "timestamp with time zone", nullable: true), + settled_at = table.Column<DateTime>(type: "timestamp with time zone", nullable: true), + last_error = table.Column<string>(type: "text", nullable: true), + already_present_count = table.Column<long>(type: "bigint", nullable: false), + version = table.Column<long>(type: "bigint", nullable: false), + started_window_seconds = table.Column<long>(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_migration_checkpoints", x => x.category_id); + }); + } + + /// <inheritdoc /> + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "migration_checkpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index cb89bee289..145f62fe0e 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -717,6 +717,72 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("message_redirects", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property<string>("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property<long>("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property<long>("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property<string>("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property<string>("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property<DateTime?>("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property<DateTime?>("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property<string>("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property<long>("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property<long?>("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property<DateTime?>("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property<long?>("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property<int>("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property<long>("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property<Guid>("Id") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs new file mode 100644 index 0000000000..86962c5a44 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -0,0 +1,83 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// <summary> +/// The migration's PostgreSQL statements. An insert that does nothing on a conflicting key is all it takes to +/// add only what is absent, and the keys it inserted come back from the same statement. +/// </summary> +class PostgreSqlMigrationSqlDialect : PostgreSqlDialect, IMigrationSqlDialect +{ + // Deterministic collations compare keys byte for byte, so there is no schema fact to read and none to name in the statement. + public Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public IEqualityComparer<string> KeyComparer(Type entityType, string propertyName) => StringComparer.Ordinal; + + // A fixed chunk, because PostgreSQL's parameter limit is far away and the row width does not bring it closer. + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement; + + // A varchar(n) column counts characters, which are code points, so a pair of UTF-16 surrogates is one. + public int TextLength(string value) => value.EnumerateRunes().Count(); + + // PostgreSQL text types cannot hold U+0000 in any encoding, and refuse the statement rather than drop it. + public char? RejectedCharacterIn(string value) => value.Contains('\0') ? '\0' : null; + + public async Task<IReadOnlyList<TEntity>> InsertMissing<TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TEntity> rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert<TEntity>.For(dbContext); + var keyColumns = string.Join(", ", insert.KeyColumns); + var inserted = new List<TEntity>(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + inserted.AddRange(await insert.Execute( + dbContext, + $""" + INSERT INTO {insert.Table} ({string.Join(", ", insert.Columns)}) + VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ON CONFLICT ({keyColumns}) DO NOTHING + RETURNING {keyColumns} + """, + chunk, + configureParameter: null, + cancellationToken)); + } + + return inserted; + } + + public async Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); + var columns = string.Join(", ", insert.Columns); + var keyColumns = string.Join(", ", insert.KeyColumns); + var counts = insert.Columns.Except(insert.KeyColumns).ToList(); + var written = 0; + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // Grouped first, because ON CONFLICT refuses a statement that reaches one row twice. + written += (await insert.Execute( + dbContext, + $""" + INSERT INTO {insert.Table} ({columns}) + SELECT {string.Join(", ", insert.Columns.Select(column => counts.Contains(column) ? $"SUM({column})" : column))} + FROM (VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}) + GROUP BY {keyColumns} + ON CONFLICT ({keyColumns}) + DO UPDATE SET {string.Join(", ", counts.Select(column => $"{column} = {insert.Table}.{column} + excluded.{column}"))} + RETURNING {keyColumns} + """, + chunk, + configureParameter: null, + cancellationToken)).Count; + } + + return written; + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index a32f5fa6b4..4b23202f13 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -23,6 +23,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton<IFailedMessageIngestionSqlDialect, PostgreSqlFailedMessageIngestionSqlDialect>(); + services.AddSingleton<IMigrationSqlDialect, PostgreSqlMigrationSqlDialect>(); services.AddSingleton<IRetryBatchSqlDialect, PostgreSqlRetryBatchSqlDialect>(); services.AddSingleton<IFullTextSearchDialect, PostgreSqlFullTextSearchDialect>(); services.AddSingleton<IEndpointThroughputDialect, PostgreSqlEndpointThroughputDialect>(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..2f862bcb80 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,794 @@ +// <auto-generated /> +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20261007000240_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// <inheritdoc /> + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property<string>("RequestId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property<int>("ArchiveType") + .HasColumnType("int"); + + b.Property<int>("OperationType") + .HasColumnType("int"); + + b.Property<int>("CurrentBatch") + .HasColumnType("int"); + + b.Property<string>("GroupName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<int>("NumberOfBatches") + .HasColumnType("int"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property<string>("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("Started") + .HasColumnType("datetime2"); + + b.Property<int>("TotalNumberOfMessages") + .HasColumnType("int"); + + b.HasKey("RequestId", "ArchiveType", "OperationType"); + + b.HasIndex("Started"); + + b.ToTable("ArchiveOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("Category") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("CustomCheckId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("FailureReason") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<Guid>("OriginatingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("OriginatingEndpointName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime>("ReportedAt") + .HasColumnType("datetime2"); + + b.Property<int>("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ReportedAt"); + + b.HasIndex("Status", "ReportedAt"); + + b.ToTable("CustomChecks"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property<string>("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<bool>("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id")); + + b.Property<string>("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection<string>("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<int>("Severity") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id")); + + b.Property<string>("DispatchContextJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("ExternalIntegrationDispatchRequests"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property<byte[]>("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property<bool>("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property<string>("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime>("FailedAt") + .HasColumnType("datetime2"); + + b.Property<string>("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("EditId"); + + b.ToTable("FailedMessageEdits"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<int>("BodySize") + .HasColumnType("int"); + + b.Property<bool>("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property<string>("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property<string>("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<bool>("IsSystemMessage") + .HasColumnType("bit"); + + b.Property<DateTime>("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property<DateTime>("LastModified") + .HasColumnType("datetime2"); + + b.Property<DateTime>("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property<string>("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("MessageType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<int>("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property<string>("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<Guid?>("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<Guid?>("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<int>("Status") + .HasColumnType("int"); + + b.Property<DateTime>("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property<DateTime?>("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("FailingEndpointAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "LastTimeOfFailure"); + + b.HasIndex("Status", "MessageType", "UniqueMessageId"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property<Guid>("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property<string>("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.HasIndex("Type", "GroupId"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "Title" }); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property<Guid>("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property<Guid>("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.Property<int>("StageAttempts") + .HasColumnType("int"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("RetryBatchId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property<string>("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property<string>("Comment") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("GroupId"); + + b.ToTable("GroupComments"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property<long>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property<long>("Id")); + + b.Property<DateTime>("CompletionTime") + .HasColumnType("datetime2"); + + b.Property<bool>("Failed") + .HasColumnType("bit"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property<string>("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property<int>("RetryType") + .HasColumnType("int"); + + b.Property<DateTime>("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending(); + + b.ToTable("HistoricRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<Guid>("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property<bool>("Monitored") + .HasColumnType("bit"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property<string>("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property<int>("ThroughputSource") + .HasColumnType("int"); + + b.PrimitiveCollection<string>("EndpointIndicators") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property<string>("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("Scope") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("NormalizedName", "ThroughputSource"); + + b.HasIndex("NormalizedSanitizedName"); + + b.ToTable("LicensingEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property<string>("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property<int>("ThroughputSource") + .HasColumnType("int"); + + b.Property<DateOnly>("DateUtc") + .HasColumnType("date"); + + b.Property<long>("MessageCount") + .HasColumnType("bigint"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc"); + + b.HasIndex("DateUtc"); + + b.ToTable("LicensingEndpointThroughput"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property<string>("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("LastModified") + .HasColumnType("datetime2"); + + b.Property<string>("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("FromPhysicalAddress"); + + b.ToTable("MessageRedirects"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property<string>("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<long>("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property<long>("CopiedCount") + .HasColumnType("bigint"); + + b.Property<string>("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime?>("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property<DateTime?>("SettledAt") + .HasColumnType("datetime2"); + + b.Property<string>("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property<long>("SkippedCount") + .HasColumnType("bigint"); + + b.Property<long?>("SourceTotal") + .HasColumnType("bigint"); + + b.Property<DateTime?>("StartedAt") + .HasColumnType("datetime2"); + + b.Property<long?>("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property<int>("State") + .HasColumnType("int"); + + b.Property<long>("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property<Guid>("Id") + .HasColumnType("uniqueidentifier"); + + b.Property<string>("Classifier") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("Context") + .HasColumnType("nvarchar(max)"); + + b.Property<int>("InitialBatchSize") + .HasColumnType("int"); + + b.Property<string>("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("InitiatedByName") + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime?>("Last") + .HasColumnType("datetime2"); + + b.Property<string>("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<int>("RetryType") + .HasColumnType("int"); + + b.Property<string>("StagingId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("StartTime") + .HasColumnType("datetime2"); + + b.Property<int>("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Status", "RetrySessionId"); + + b.ToTable("RetryBatches"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property<int>("Id") + .HasColumnType("int"); + + b.Property<Guid>("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.ToTable("RetryBatchNowForwarding"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property<string>("Key") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<string>("Value") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property<string>("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property<string>("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property<string>("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property<string>("RequestId") + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property<int>("RetryType") + .HasColumnType("int"); + + b.Property<string>("Classifier") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<DateTime>("CompletionTime") + .HasColumnType("datetime2"); + + b.Property<bool>("Failed") + .HasColumnType("bit"); + + b.Property<DateTime>("Last") + .HasColumnType("datetime2"); + + b.Property<int>("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property<string>("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime>("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("RequestId", "RetryType"); + + b.ToTable("UnacknowledgedRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..c3dab88c81 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// <inheritdoc /> + public partial class AddMigrationCheckpoints : Migration + { + /// <inheritdoc /> + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "MigrationCheckpoints", + columns: table => new + { + CategoryId = table.Column<string>(type: "nvarchar(450)", maxLength: 450, nullable: false), + State = table.Column<int>(type: "int", nullable: false), + Cursor = table.Column<string>(type: "nvarchar(max)", nullable: true), + CopiedCount = table.Column<long>(type: "bigint", nullable: false), + SkippedCount = table.Column<long>(type: "bigint", nullable: false), + SourceTotal = table.Column<long>(type: "bigint", nullable: true), + SkipReasons = table.Column<string>(type: "nvarchar(max)", nullable: true), + StartedAt = table.Column<DateTime>(type: "datetime2", nullable: true), + LastProgressAt = table.Column<DateTime>(type: "datetime2", nullable: true), + SettledAt = table.Column<DateTime>(type: "datetime2", nullable: true), + LastError = table.Column<string>(type: "nvarchar(max)", nullable: true), + AlreadyPresentCount = table.Column<long>(type: "bigint", nullable: false), + Version = table.Column<long>(type: "bigint", nullable: false), + StartedWindowSeconds = table.Column<long>(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_MigrationCheckpoints", x => x.CategoryId); + }); + } + + /// <inheritdoc /> + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "MigrationCheckpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index acca0378a6..f6e5ed81b7 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -573,6 +573,57 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("MessageRedirects"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property<string>("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property<long>("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property<long>("CopiedCount") + .HasColumnType("bigint"); + + b.Property<string>("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property<string>("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property<DateTime?>("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property<DateTime?>("SettledAt") + .HasColumnType("datetime2"); + + b.Property<string>("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property<long>("SkippedCount") + .HasColumnType("bigint"); + + b.Property<long?>("SourceTotal") + .HasColumnType("bigint"); + + b.Property<DateTime?>("StartedAt") + .HasColumnType("datetime2"); + + b.Property<long?>("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property<int>("State") + .HasColumnType("int"); + + b.Property<long>("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property<Guid>("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs index 0e26ef0ece..132096e217 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs @@ -111,6 +111,11 @@ protected static string ParameterRows(int rowCount, int columnCount) return sql.ToString(); } + // The same rows as ParameterRows, each ending with its position in the batch, so a statement can keep the first of two duplicate keys. + protected static string ParameterRowsWithOrdinal(int rowCount, int columnCount) => + string.Join(",\n", Enumerable.Range(0, rowCount).Select(row => + $"({string.Join(", ", Enumerable.Range(0, columnCount).Select(column => $"@p{(row * columnCount) + column}"))}, {row})")); + protected static int MaxRowsPerStatement(int columns) => MaxParametersPerStatement / columns; // SQL Server's ceiling is 2100 (https://learn.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server). diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs new file mode 100644 index 0000000000..2ce3a44741 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -0,0 +1,201 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using System.Data.Common; +using System.Globalization; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// <summary> +/// The migration's SQL Server statements. Two keys that differ only in case, accents, kana type, width or trailing spaces can be one row here, because a +/// column's collation decides what counts as equal, so the statements compare keys under the collation the +/// schema gives them and the batch is de-duplicated the same way before it is offered to the table. +/// </summary> +class SqlServerMigrationSqlDialect : SqlServerDialect, IMigrationSqlDialect +{ + OpenedKeys? OpenedState { get; set; } + + public async Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) + { + var sql = dbContext.GetService<ISqlGenerationHelper>(); + var collations = new Dictionary<(string Table, string Column), string>(); + var comparers = new Dictionary<(Type EntityType, string Property), IEqualityComparer<string>>(); + + foreach (var entityType in dbContext.Model.GetEntityTypes()) + { + if (entityType.GetTableName() is not { } tableName || entityType.FindPrimaryKey() is not { } key) + { + continue; + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var table = sql.DelimitIdentifier(tableName, entityType.GetSchema()); + + foreach (var property in key.Properties.Where(property => property.ClrType == typeof(string))) + { + // A key property with no column of its own is refused by name when MigrationInsert builds the statement. + if (property.GetColumnName(storeObject) is not { } column) + { + continue; + } + + var collation = await dbContext.Database + .SqlQuery<KeyColumnCollation>($""" + SELECT c.collation_name AS [Name], + CONVERT(int, COLLATIONPROPERTY(c.collation_name, 'ComparisonStyle')) AS [ComparisonStyle] + FROM sys.columns AS c + WHERE c.object_id = OBJECT_ID({table}) AND c.name = {column} AND c.collation_name IS NOT NULL + """) + .SingleOrDefaultAsync(cancellationToken) + ?? throw new InvalidOperationException($"The migration target found no collation for {tableName}.{column}, so the SQL Server schema is missing or not current. Run ServiceControl with --setup against this database first."); + + collations[(table, sql.DelimitIdentifier(column))] = collation.Name; + comparers[(entityType.ClrType, property.Name)] = new PaddedKeyComparer(ComparerFor(collation.ComparisonStyle)); + } + } + + OpenedState = new OpenedKeys(collations, comparers); + } + + public IEqualityComparer<string> KeyComparer(Type entityType, string propertyName) => + Keys.Comparers.TryGetValue((entityType, propertyName), out var comparer) + ? comparer + : throw new ArgumentException($"{entityType.Name}.{propertyName} is not a string primary-key column, so it has no collation to compare under."); + + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement(parametersPerRow); + + // An nvarchar(n) column counts byte-pairs, which are UTF-16 units, so a surrogate pair takes two. + public int TextLength(string value) => value.Length; + + // An nvarchar column stores any UTF-16 unit, U+0000 included. + public char? RejectedCharacterIn(string value) => null; + + public async Task<IReadOnlyList<TEntity>> InsertMissing<TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TEntity> rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert<TEntity>.For(dbContext); + var columns = string.Join(", ", insert.Columns); + // The collation is an identifier sys.columns returned, and COLLATE takes no parameter. + var partitionBy = string.Join(", ", insert.KeyColumns.Select(column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column)); + var inserted = new List<TEntity>(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // HOLDLOCK, or two writers can both find a key missing and both insert it. + // FORCESEEK, because on a small table SQL Server scans a whole index instead of seeking the primary key, and HOLDLOCK then locks every row the scan reads. + inserted.AddRange(await insert.Execute( + dbContext, + $""" + MERGE {insert.Table} WITH (HOLDLOCK, FORCESEEK) AS t + USING ( + SELECT {columns} + FROM ( + SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {partitionBy} ORDER BY [MigrationOrdinal]) AS [MigrationDuplicate] + FROM (VALUES + {ParameterRowsWithOrdinal(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}, [MigrationOrdinal]) + ) AS deduplicated + WHERE [MigrationDuplicate] = 1 + ) AS s ({columns}) + ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} + WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) + OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"inserted.{column}"))}; + """, + chunk, + DeclareUnboundedTextAsMax, + cancellationToken)); + } + + return inserted; + } + + public async Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); + var columns = string.Join(", ", insert.Columns); + var counts = insert.Columns.Except(insert.KeyColumns).ToList(); + var collatedKeys = insert.KeyColumns.ToDictionary(column => column, column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column); + var written = 0; + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // HOLDLOCK, or two writers can both find a day missing and both insert it. + // The output names the source's key values, because a stored name can differ from the chunk's by a trailing space and would match no row of it. + written += (await insert.Execute( + dbContext, + $""" + MERGE {insert.Table} WITH (HOLDLOCK) AS t + USING ( + SELECT {string.Join(", ", insert.Columns.Select(column => collatedKeys.TryGetValue(column, out var key) ? key : $"SUM({column})"))} + FROM (VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}) + GROUP BY {string.Join(", ", collatedKeys.Values)} + ) AS s ({columns}) + ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} + WHEN MATCHED THEN UPDATE SET {string.Join(", ", counts.Select(column => $"{column} = t.{column} + s.{column}"))} + WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) + OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"s.{column}"))}; + """, + chunk, + configureParameter: null, + cancellationToken)).Count; + } + + return written; + } + + // EF declares unbounded text as nvarchar(4000) or nvarchar(max) by each value's length, and each new mix compiles its own plan. + // Size -1 declares every such value as the nvarchar(max) its column already is. + static void DeclareUnboundedTextAsMax(IProperty property, DbParameter parameter) + { + if (property.ClrType == typeof(string) && property.GetMaxLength() is null) + { + parameter.Size = -1; + } + } + + // COLLATIONPROPERTY's ComparisonStyle is 0 for a binary collation and for one that ignores nothing, and otherwise sets one bit for each thing the collation ignores. + static StringComparer ComparerFor(int comparisonStyle) + { + if (comparisonStyle == 0) + { + return StringComparer.Ordinal; + } + + var options = ComparisonStyleOptions + .Where(style => (comparisonStyle & style.Bit) != 0) + .Aggregate(CompareOptions.None, (combined, style) => combined | style.Option); + + return CultureInfo.InvariantCulture.CompareInfo.GetStringComparer(options); + } + + static readonly (int Bit, CompareOptions Option)[] ComparisonStyleOptions = + [ + (1, CompareOptions.IgnoreCase), + (2, CompareOptions.IgnoreNonSpace), + (0x10000, CompareOptions.IgnoreKanaType), + (0x20000, CompareOptions.IgnoreWidth) + ]; + + OpenedKeys Keys => OpenedState ?? throw new InvalidOperationException($"The SQL Server migration dialect is not open. Call {nameof(Open)} first."); + + sealed record OpenedKeys( + IReadOnlyDictionary<(string Table, string Column), string> Collations, + IReadOnlyDictionary<(Type EntityType, string Property), IEqualityComparer<string>> Comparers); + + sealed record KeyColumnCollation(string Name, int ComparisonStyle); + + // SQL Server pads the shorter value with spaces before it compares, so keys that differ only in trailing spaces are equal. + sealed class PaddedKeyComparer(StringComparer inner) : IEqualityComparer<string> + { + public bool Equals(string? x, string? y) => inner.Equals(x?.TrimEnd(' '), y?.TrimEnd(' ')); + + public int GetHashCode(string obj) => inner.GetHashCode(obj.TrimEnd(' ')); + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index cff42ab67d..90ee720b51 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -18,6 +18,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton<IFailedMessageIngestionSqlDialect, SqlServerFailedMessageIngestionSqlDialect>(); + services.AddSingleton<IMigrationSqlDialect, SqlServerMigrationSqlDialect>(); services.AddSingleton<IRetryBatchSqlDialect, SqlServerRetryBatchSqlDialect>(); services.AddSingleton<IFullTextSearchDialect, SqlServerFullTextSearchDialect>(); services.AddSingleton<IEndpointThroughputDialect, SqlServerEndpointThroughputDialect>(); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs index 1e0a590cdb..f0fc102b07 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs @@ -8,12 +8,14 @@ namespace ServiceControl.Persistence.EFCore.Abstractions; using ServiceControl.CustomChecks; using ServiceControl.Infrastructure.StartupChecks; using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.EFCore.DataMigration; using ServiceControl.Persistence.EFCore.Implementation; using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; using ServiceControl.Persistence.EFCore.Implementation.Recoverability; using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.Infrastructure.Metrics; +using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Recoverability; using ServiceControl.Persistence.UnitOfWork; @@ -41,6 +43,10 @@ protected static void RegisterDataStores(IServiceCollection services, EFPersiste services.AddSingleton<IExternalIntegrationRequestsDataStore>(p => p.GetRequiredService<ExternalIntegrationRequestsDataStore>()); services.AddHostedService(p => p.GetRequiredService<ExternalIntegrationRequestsDataStore>()); + services.AddSingleton<IMigrationCheckpointStore, EFMigrationCheckpointStore>(); + services.AddSingleton<IMigrationTargetReadiness, EFCoreMigrationTargetReadiness>(); + services.AddSingleton<IMigrationTarget, EFCoreMigrationTarget>(); + if (settings.RunRetentionSweep) { services.AddSingleton<RetentionMetrics>(); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs new file mode 100644 index 0000000000..025cb1fe22 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes, reads and deletes one probe body, so a misconfigured body store stops the copy before it starts. +/// Bodies can live on a file share, in Azure Blob Storage or in S3, and without this check the first category +/// that carries bodies would only find out part way through, when a body it cannot write fails the batch and stops the category. +/// </summary> +sealed class BodyStorageIsWritableCheck(IBodyStoragePersistence bodyStorage) : IMigrationStartupCheck +{ + const string ProbeBodyId = "migration-writable-probe"; + + public string Name => "message body storage is writable"; + + public async Task Run(CancellationToken cancellationToken = default) + { + await bodyStorage.WriteBody(ProbeBodyId, Encoding.UTF8.GetBytes(ProbeBodyId), "text/plain", cancellationToken); + + try + { + var probe = await bodyStorage.ReadBody(ProbeBodyId, cancellationToken) + ?? throw new Exception($"Message body storage accepted the probe body '{ProbeBodyId}' and then did not return it."); + + await probe.Stream.DisposeAsync(); + } + finally + { + // The probe is not migrated data, so it goes even when the read fails: a probe left behind is a body the source never had. + await bodyStorage.DeleteBodyIfExists(ProbeBodyId, cancellationToken); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs new file mode 100644 index 0000000000..ae192efe04 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/CustomCheckRowMapper.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.EFCore.Entities; + +static class CustomCheckRowMapper +{ + public static CustomCheckEntity Map(CustomCheckDetail detail) => new() + { + // The key a live report of the same check upserts on, so the first report after the host opens replaces this row rather than adding a second. + Id = detail.GetDeterministicId(), + CustomCheckId = detail.CustomCheckId, + Category = detail.Category, + Status = detail.HasFailed ? Status.Fail : Status.Pass, + ReportedAt = detail.ReportedAt, + FailureReason = detail.FailureReason, + OriginatingEndpointName = detail.OriginatingEndpoint.Name, + OriginatingEndpointHostId = detail.OriginatingEndpoint.HostId, + OriginatingEndpointHost = detail.OriginatingEndpoint.Host + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs new file mode 100644 index 0000000000..6d8297c073 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -0,0 +1,182 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Collections.Frozen; +using Abstractions; +using Implementation; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; +using Writers; + +/// <summary> +/// Writes a migration into SQL Server or PostgreSQL. It holds one writer per category and knows nothing about +/// any of them beyond that, so a category this build cannot write is simply absent from +/// <see cref="SupportedCategoryIds" /> and never reaches the engine. +/// </summary> +sealed class EFCoreMigrationTarget( + IServiceScopeFactory scopeFactory, + IMigrationSqlDialect migrationDialect, + IBodyStoragePersistence storagePersistence, + EFPersisterSettings settings, + ILogger<EFCoreMigrationTarget> logger) : DataStoreBase(scopeFactory), IMigrationTarget +{ + readonly FrozenDictionary<string, IMigrationCategoryWriter> writers = new IMigrationCategoryWriter[] + { + new KnownEndpointsWriter(migrationDialect), + new EndpointSettingsWriter(migrationDialect), + new FailedMessagesWriter(migrationDialect, storagePersistence, settings), + new LicensingEndpointsWriter(migrationDialect), + new LicensingThroughputWriter(migrationDialect), + new LicensingReportMasksWriter(migrationDialect), + new LicensedEndpointDetailsWriter(migrationDialect), + new MessageRedirectsWriter(migrationDialect), + new SubscriptionsWriter(migrationDialect), + new NotificationSettingsWriter(migrationDialect), + new TrialEndDateWriter(migrationDialect), + new RetryOperationsWriter(migrationDialect), + new PendingIntegrationEventsWriter(migrationDialect), + new CustomChecksWriter(migrationDialect), + new FailedErrorImportsWriter(migrationDialect, storagePersistence, settings), + new GroupCommentsWriter(migrationDialect) + }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); + + public Task Open(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => migrationDialect.Open(dbContext, token), cancellationToken); + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, _) => Task.FromResult(WriterFor(category).BatchSize(dbContext)), cancellationToken); + + public Task<MigrationWriteResult> Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, writeToken) => + { + var prepared = await WriterFor(category).Prepare(dbContext, batch, writeToken); + + AccountForEveryRow(category, batch, prepared); + + var skipReasons = prepared.Skips.Count == 0 ? null : prepared.Skips.GroupBy(skip => skip.Reason).ToDictionary(group => group.Key, group => group.LongCount()); + + var (copied, alreadyPresent, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + // This block is retried, and an attempt that failed leaves the checkpoint it added still tracked, so the next attempt would insert it a second time. + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + var inserted = await prepared.Insert(dbContext, token); + var present = AlreadyPresentIn(category, batch, inserted, prepared.Skips.Count); + var stored = await dbContext.UpsertCheckpoint( + checkpointToExtend.Extend(inserted, prepared.Skips.Count, present, skipReasons), + token); + await transaction.CommitAsync(token); + + return (inserted, present, stored); + }, writeToken); + + foreach (var (sourceId, reason, detail) in prepared.Skips) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId} as {SkipReason}: {Detail}", sourceId, category.Id, reason, detail); + } + + foreach (var (sourceId, detail) in prepared.Merges) + { + logger.LogWarning("Merged {SourceId} in category {CategoryId} into a row with the same key: {Detail}", sourceId, category.Id, detail); + } + + foreach (var (sourceId, detail) in prepared.Changes) + { + logger.LogWarning("Changed {SourceId} in category {CategoryId} on the way into SQL: {Detail}", sourceId, category.Id, detail); + } + + // Once per copy of the category, because the line describes every batch alike and a retry's reset clears the count. + if (prepared.RenumberedRows is { } renumbered && checkpointToExtend.CopiedCount == 0) + { + logger.LogInformation("Category {CategoryId}: the database gives the copied {RenumberedRows} new ids. Nothing in SQL refers to the ids they had in the source.", category.Id, renumbered); + } + + return new MigrationWriteResult( + saved, + copied, + prepared.Skips.Count, + [.. prepared.Skips.Select(skip => skip.SourceId)], + AlreadyPresent: alreadyPresent, + SkipReasons: skipReasons, + Merged: prepared.Merges.Count, + Changed: prepared.Changes.Count); + }, cancellationToken); + + // Checked before the insert, because the subtraction below can only catch a writer that over-reports. + internal static void AccountForEveryRow(MigrationCategory category, MigrationBatch batch, PreparedBatch prepared) + { + if (prepared.PreparedRowCount + prepared.Skips.Count != batch.Rows.Count) + { + throw new InvalidOperationException($"The {category.Id} writer prepared {prepared.PreparedRowCount} rows and skipped {prepared.Skips.Count} of the {batch.Rows.Count} rows in the batch. Every row must be one or the other, or the rows it dropped would be counted as rows the target already held."); + } + } + + // Without the throw, a miscount would quietly shrink the halt threshold's denominator instead of failing. + internal static int AlreadyPresentIn(MigrationCategory category, MigrationBatch batch, int copied, int skipped) + { + var alreadyPresent = batch.Rows.Count - copied - skipped; + + if (alreadyPresent < 0) + { + throw new InvalidOperationException($"The target copied {copied} and skipped {skipped} of the {batch.Rows.Count} rows in category {category.Id}, leaving {alreadyPresent} already present. Every row is copied, skipped or already present."); + } + + return alreadyPresent; + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => WriterFor(category).Count(dbContext, token), cancellationToken); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, resetToken) => + { + var storeKeyed = writers.GetValueOrDefault(reset.CategoryId) as IStoreKeyedCategoryWriter; + + var (deleted, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + // Only a reset that deletes is refused, because an optional category is retried after ServiceControl opens and keeps its rows. + if (storeKeyed is not null && await dbContext.GetSetting<DateTime?>(SettingKeys.MigrationHostOpenedOnTarget, token) is not null) + { + throw new InvalidOperationException($"Category {reset.CategoryId} cannot be retried, because ServiceControl has opened on this database since the copy and its tables may now hold live rows. A retry deletes every row in them, so it is only possible before ServiceControl first opens on this database."); + } + + var deletedRows = storeKeyed is null ? 0 : await storeKeyed.DeleteCopiedRows(dbContext, token); + var stored = await dbContext.UpsertCheckpoint(reset, token); + await transaction.CommitAsync(token); + + return (deletedRows, stored); + }, resetToken); + + if (storeKeyed is null) + { + logger.LogInformation("Reset category {CategoryId} for a retry and kept the rows it copied", reset.CategoryId); + } + else + { + logger.LogInformation("Reset category {CategoryId} for a retry and deleted the {DeletedRowCount} rows it copied, because a re-read would add them a second time", reset.CategoryId, deleted); + } + + return saved; + }, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => writers.Keys; + + public IReadOnlyDictionary<string, Type> DocumentTypes => writers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + + IMigrationCategoryWriter WriterFor(MigrationCategory category) => + writers.TryGetValue(category.Id, out var writer) + ? writer + : throw new NotSupportedException($"The migration target cannot yet write the '{category.Id}' category."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs new file mode 100644 index 0000000000..9988718ea8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// What a SQL Server or PostgreSQL database has to be before a copy writes to it, and the stamp saying a host +/// has opened on it since. The stamp is a row in the settings table, so it survives a restart and belongs to +/// the database rather than to the instance. +/// </summary> +public class EFCoreMigrationTargetReadiness( + IServiceScopeFactory scopeFactory, + IBodyStoragePersistence bodyStorage, + TimeProvider timeProvider) : DataStoreBase(scopeFactory), IMigrationTargetReadiness +{ + // Cheapest first: the schema costs one query, and the body store costs a round trip to a file share or a + // cloud service. + public IReadOnlyList<IMigrationStartupCheck> ContributedChecks() => + [ + new SchemaIsCurrentCheck(scopeFactory), + new TargetHoldsNoServiceControlDataCheck(scopeFactory), + new BodyStorageIsWritableCheck(bodyStorage) + ]; + + public Task RecordHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + if (await dbContext.GetSetting<DateTime?>(SettingKeys.MigrationHostOpenedOnTarget, token) is null) + { + await dbContext.StoreSetting(SettingKeys.MigrationHostOpenedOnTarget, timeProvider.GetUtcNow().UtcDateTime, token); + } + }, cancellationToken); + + public Task<bool> HasHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + await dbContext.GetSetting<DateTime?>(SettingKeys.MigrationHostOpenedOnTarget, token) is not null, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs new file mode 100644 index 0000000000..942dc4ea71 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedErrorImportRowMapper.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +static class FailedErrorImportRowMapper +{ + public static FailedErrorImportEntity Map(FailedErrorImport import, DateTime failedAt, bool bodyStoredExternally) + { + var message = import.Message!; + + return new FailedErrorImportEntity + { + // The key a live import of the same message derives, so a later failure of that message on SQL updates this row rather than adding a second. + UniqueMessageId = FailedErrorImport.DeriveKey(message.Headers, message.Id), + FailedAt = failedAt, + MessageId = message.Id, + HeadersJson = MessageHeaders.Write(message.Headers), + Body = bodyStoredExternally ? [] : message.Body ?? [], + BodyStoredExternally = bodyStoredExternally, + ExceptionInfo = import.ExceptionInfo ?? string.Empty + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs new file mode 100644 index 0000000000..f23837b53e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslation.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// What one failed message document became: either a row with its failure group rows, or a skip with the reason. +/// </summary> +abstract record FailedMessageDocumentTranslation +{ + FailedMessageDocumentTranslation() + { + } + + /// <summary> + /// The document translates to a failed message row and the group rows keyed by it. + /// Changes has one sentence for each way the row differs from the document on purpose, and is empty when it differs in none. + /// </summary> + public sealed record Translated(FailedMessageEntity Row, IReadOnlyList<FailedMessageGroupEntity> Groups, IReadOnlyList<string> Changes) : FailedMessageDocumentTranslation; + + /// <summary> + /// The document cannot become a row, for the reason given, and <paramref name="Detail" /> says which document and why. + /// </summary> + public sealed record Skipped(MigrationSkipReason Reason, string Detail) : FailedMessageDocumentTranslation; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs new file mode 100644 index 0000000000..24d2b6435c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/FailedMessageDocumentTranslator.cs @@ -0,0 +1,228 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text.Json; +using ServiceControl.MessageFailures; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.EntityConfigurations; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// <summary> +/// Translates one RavenDB failed message document into a failed message row and its group rows, or a skip with the reason +/// when the document can't satisfy the target columns. +/// </summary> +static class FailedMessageDocumentTranslator +{ + public static FailedMessageDocumentTranslation Translate(MigrationRow source) + { + var document = (FailedMessage)source.Document; + + if (!Guid.TryParse(document.UniqueMessageId, out var uniqueMessageId)) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.UnparseableUniqueMessageId, + $"UniqueMessageId '{document.UniqueMessageId}' on document '{source.SourceId}' is not a GUID and the target column is a uniqueidentifier"); + } + + if (document.ProcessingAttempts is not { Count: > 0 }) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.NoProcessingAttempts, + $"Document '{source.SourceId}' has no processing attempts, so no attempt supplies the required columns"); + } + + if (document.ProcessingAttempts[^1] is not { } newest) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' is null, so no attempt supplies the required columns"); + } + + if (MissingFrom(newest) is { } missing) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' has no {missing}, which the row's columns are built from"); + } + + if (MalformedIn(newest.MessageMetadata) is { } malformed) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.MalformedValue, + $"The newest attempt on document '{source.SourceId}' holds a {malformed.Found.FullName} as its {malformed.Key} metadata, where the row needs a {malformed.Expected.FullName}: '{Shown(malformed.Value)}'"); + } + + // An attempt stored without FailureDetails reads back as the default object, whose address is empty. + if (string.IsNullOrEmpty(newest.FailureDetails.AddressOfFailingEndpoint)) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"FailedMessages.{nameof(FailedMessageEntity.FailingEndpointAddress)} is NOT NULL and the newest attempt on document '{source.SourceId}' has no AddressOfFailingEndpoint"); + } + + if (newest.FailureDetails.TimeOfFailure == default || newest.AttemptedAt == default) + { + return new FailedMessageDocumentTranslation.Skipped( + MigrationSkipReason.RequiredValueMissing, + $"The newest attempt on document '{source.SourceId}' has no {(newest.FailureDetails.TimeOfFailure == default ? nameof(newest.FailureDetails.TimeOfFailure) : nameof(newest.AttemptedAt))}, which the row's failure times are built from"); + } + + var metadata = newest.MessageMetadata; + // The newest attempt passed the checks above, so the list always holds at least one time for Min and Max. + var timesOfFailure = document.ProcessingAttempts + .Where(HasRecordedFailure) + .Select(attempt => attempt.FailureDetails.TimeOfFailure) + .ToList(); + var sendingEndpoint = (EndpointDetails?)metadata.GetValueOrDefault("SendingEndpoint"); + var receivingEndpoint = (EndpointDetails?)metadata.GetValueOrDefault("ReceivingEndpoint"); + + var row = new FailedMessageEntity + { + UniqueMessageId = uniqueMessageId, + Status = document.Status, + StatusChangedAt = ReadResolvedDate(source, MigrationMetadataKeys.StatusChangedAt), + LastModified = ReadResolvedDate(source, MigrationMetadataKeys.LastModified), + NumberOfProcessingAttempts = document.ProcessingAttempts.Count, + FirstTimeOfFailure = AsUtc(timesOfFailure.Min()), + LastTimeOfFailure = AsUtc(timesOfFailure.Max()), + LastAttemptedAt = AsUtc(newest.AttemptedAt), + MessageId = newest.MessageId, + MessageType = LastCharactersThatFit(metadata.GetValueOrDefault("MessageType") as string), + TimeSent = metadata.GetValueOrDefault("TimeSent") is DateTime timeSent ? AsUtc(timeSent) : null, + ConversationId = metadata.GetValueOrDefault("ConversationId") as string, + SendingEndpointName = sendingEndpoint?.Name, + SendingEndpointHostId = sendingEndpoint?.HostId, + SendingEndpointHost = sendingEndpoint?.Host, + ReceivingEndpointName = receivingEndpoint?.Name, + ReceivingEndpointHostId = receivingEndpoint?.HostId, + ReceivingEndpointHost = receivingEndpoint?.Host, + ExceptionType = newest.FailureDetails.Exception?.ExceptionType, + ExceptionMessage = newest.FailureDetails.Exception?.Message, + IsSystemMessage = metadata.GetValueOrDefault("IsSystemMessage") is true, + HeadersJson = MessageHeaders.Write(newest.Headers), + FailingEndpointAddress = newest.FailureDetails.AddressOfFailingEndpoint + }; + + // Null groups and groups with no id are dropped, because GroupId is part of a NOT NULL key. + IReadOnlyList<FailedMessageGroupEntity> groups = + [ + .. (document.FailureGroups ?? []) + .Where(failureGroup => failureGroup is { Id: not null }) + .Select(failureGroup => new FailedMessageGroupEntity + { + FailedMessageUniqueId = uniqueMessageId, + GroupId = failureGroup.Id, + Title = failureGroup.Title ?? string.Empty, + Type = failureGroup.Type ?? string.Empty + }) + ]; + + return new FailedMessageDocumentTranslation.Translated(row, groups, ChangesMadeTo(document, metadata.GetValueOrDefault("MessageType") as string)); + } + + // One sentence per way the row differs from the document, for the warning the target logs after the commit. + static IReadOnlyList<string> ChangesMadeTo(FailedMessage document, string? messageType) + { + List<string> changes = []; + var groups = document.FailureGroups ?? []; + FailedMessage.FailureGroup?[] withoutId = [.. groups.Where(failureGroup => failureGroup is not { Id: not null })]; + FailedMessage.FailureGroup[] withoutTitleOrType = [.. groups.Where(failureGroup => failureGroup is { Id: not null } && (failureGroup.Title is null || failureGroup.Type is null))]; + + if (messageType is { Length: > ColumnLengths.ShortTextLength }) + { + changes.Add($"Cut the message type from {messageType.Length} characters to its last {ColumnLengths.ShortTextLength}, as ingestion does."); + } + + if (withoutId.Length > 0) + { + changes.Add($"Dropped the failure groups with no id, because SQL keys a group by its id: {string.Join(", ", withoutId.Select(failureGroup => failureGroup is null ? "an empty entry" : $"the group titled '{failureGroup.Title}'"))}."); + } + + if (withoutTitleOrType.Length > 0) + { + changes.Add($"Stored an empty title or type for the failure groups that had none: {string.Join(", ", withoutTitleOrType.Select(failureGroup => $"'{failureGroup.Id}'"))}."); + } + + return changes; + } + + /// <summary> + /// Fills the four body columns of a mapped row the way ingestion does, letting <see cref="MessageBodyClassifier" /> + /// decide whether the body stays inline or goes to external storage. Returns true when the body goes to external + /// storage. It changes nothing else on the row and writes no body itself. + /// </summary> + public static bool SetBody(FailedMessageEntity row, IReadOnlyDictionary<string, string> headers, MigrationBody body, int maxBodySizeToStore) + { + var (bodyText, storeExternally) = MessageBodyClassifier.Classify(headers, body.Content, maxBodySizeToStore); + + row.BodyText = bodyText; + row.BodyStoredExternally = storeExternally; + row.BodySize = body.Content.Length; + row.BodyContentType = body.ContentType; + + return storeExternally; + } + + static string? MissingFrom(FailedMessage.ProcessingAttempt attempt) => + attempt.FailureDetails is null ? nameof(attempt.FailureDetails) + : attempt.MessageMetadata is null ? nameof(attempt.MessageMetadata) + : attempt.Headers is null ? nameof(attempt.Headers) + : null; + + // An attempt stored without FailureDetails reads back as a default object with an empty address and a time of failure set to when it was read. + static bool HasRecordedFailure(FailedMessage.ProcessingAttempt? attempt) => + attempt is { FailureDetails: { AddressOfFailingEndpoint: { Length: > 0 } } details } && details.TimeOfFailure != default; + + // A missing value becomes null, just as RavenDB's own read path shows it, so only a value of the wrong type needs a skip. + static (string Key, Type Found, Type Expected, object Value)? MalformedIn(Dictionary<string, object> metadata) + { + foreach (var (key, expected) in TypedMetadata) + { + if (metadata.GetValueOrDefault(key) is { } value && !expected.IsInstanceOfType(value)) + { + return (key, value.GetType(), expected, value); + } + } + + return null; + } + + // Every metadata value the row reads by type, so a value of another type is skipped rather than copied as null or false. + static readonly (string Key, Type Expected)[] TypedMetadata = + [ + ("TimeSent", typeof(DateTime)), + ("MessageType", typeof(string)), + ("ConversationId", typeof(string)), + ("IsSystemMessage", typeof(bool)) + ]; + + const int MaxShownValueLength = 100; + + // Cut short because a stored value can be any length and the skip detail is one log line. + static string Shown(object value) + { + var text = value as string ?? JsonSerializer.Serialize(value); + + return text.Length > MaxShownValueLength ? $"{text[..MaxShownValueLength]}..." : text; + } + + // Throws rather than make up a date, because a made-up StatusChangedAt or LastModified would give the row a retention or modification time it never had. + static DateTime ReadResolvedDate(MigrationRow source, string key) => + source.Metadata.GetValueOrDefault(key) switch + { + DateTime value => AsUtc(value), + null => throw new InvalidOperationException($"Row '{source.SourceId}' arrived without a {key} value, which the source resolves before a failed message crosses the seam"), + var other => throw new InvalidOperationException($"Row '{source.SourceId}' carries a {other.GetType().FullName} as its {key} value, where the source resolves a {typeof(DateTime).FullName} before a failed message crosses the seam") + }; + + // Npgsql only writes UTC to a timestamp with time zone column, and a value whose kind is Unspecified is already UTC, so it must not be shifted. + // Kept apart from the RavenDB source's copy because this project does not reference it. + internal static DateTime AsUtc(DateTime value) => + value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); + + // Uses the same length limit that ingestion applies to the MessageType column. + static string? LastCharactersThatFit(string? value) => + value is { Length: > ColumnLengths.ShortTextLength } ? value[^ColumnLengths.ShortTextLength..] : value; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs new file mode 100644 index 0000000000..86fb1ebcdd --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/GroupCommentRowMapper.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// Turns a group comment from the source into the row the product's own edit would write. The writer skips a null or blank comment before it calls this. +/// </summary> +static class GroupCommentRowMapper +{ + public static GroupCommentEntity Map(GroupComment comment) => new() + { + GroupId = comment.Id, + Comment = comment.Comment! + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs new file mode 100644 index 0000000000..82f6d827c1 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Everything the target needs to know about one category: how big a batch is, how its documents +/// become rows, and how many rows the target holds for it. +/// </summary> +interface IMigrationCategoryWriter +{ + /// <summary> + /// The category this writer handles, named as in <see cref="MigrationCategoryIds" />. + /// </summary> + string CategoryId { get; } + + /// <summary> + /// The type of <see cref="MigrationRow.Document" /> <see cref="Prepare" /> takes. A row holding any other type + /// fails the batch with an <see cref="InvalidCastException" />. + /// </summary> + Type DocumentType { get; } + + /// <summary> + /// The most rows the target will take from the source in one batch. It comes from how many values each row + /// carries and how many parameters one statement can hold, so a wider table takes fewer rows. The context is read for its model only. + /// </summary> + int BatchSize(ServiceControlDbContext dbContext); + + /// <summary> + /// Turns one batch of source documents into the rows to insert and the rows to skip, writing nothing to the database: the insert it returns runs later, inside the target's transaction. + /// A writer may store data its rows point at outside the database, such as message bodies, because that data has to exist before the insert commits. + /// Every batch row produces exactly one row or exactly one skip, never both and never neither, because the target works out how many were already there by subtracting both from the batch size. + /// </summary> + Task<PreparedBatch> Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default); + + /// <summary> + /// How many rows the target holds for this category. It counts only this category, even where two of them + /// share a table. + /// </summary> + Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs new file mode 100644 index 0000000000..f66dd7d99f --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IStoreKeyedCategoryWriter.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; + +/// <summary> +/// A category writer whose rows a re-read cannot recognise as already copied, so copying the category again would add every row a second time. +/// Rows whose keys the database makes are one kind, and throughput days, whose counts the copy adds to, are another. +/// The target deletes those rows when it resets the category for a retry. +/// The delete empties whole tables, so it is only safe for a category copied while ServiceControl is stopped. +/// </summary> +interface IStoreKeyedCategoryWriter +{ + /// <summary> + /// Deletes every row in this category's tables and returns how many it deleted. + /// Runs in the same transaction as the checkpoint reset. + /// </summary> + Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs new file mode 100644 index 0000000000..3d868b5dc5 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingEndpointRowMapper.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; + +/// <summary> +/// Turns a licensing endpoint read from the source into the row the target stores, with the same key and the same +/// stand-ins for missing values as the product's own save. +/// </summary> +static class LicensingEndpointRowMapper +{ + public static LicensingEndpointEntity Map(Endpoint endpoint) + { + var sanitizedName = endpoint.SanitizedName ?? string.Empty; + + return new LicensingEndpointEntity + { + NormalizedName = LicensingDataStore.Normalize(endpoint.Id.Name), + ThroughputSource = endpoint.Id.ThroughputSource, + Name = endpoint.Id.Name, + SanitizedName = sanitizedName, + NormalizedSanitizedName = LicensingDataStore.Normalize(sanitizedName), + UserIndicator = endpoint.UserIndicator ?? string.Empty, + Scope = endpoint.Scope, + CreatedDate = endpoint.CreatedDate ?? DateTime.MinValue, + EndpointIndicators = endpoint.EndpointIndicators?.ToList() ?? [] + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs new file mode 100644 index 0000000000..299fd8912d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/LicensingThroughputRowMapper.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation; + +/// <summary> +/// Turns one endpoint's daily count read from the source into the row the target stores, keyed by the same +/// normalised name the product's own licensing store computes. +/// </summary> +static class LicensingThroughputRowMapper +{ + public static LicensingEndpointThroughputEntity Map(ThroughputDayRow row) + { + return new LicensingEndpointThroughputEntity + { + NormalizedName = LicensingDataStore.Normalize(row.Endpoint.Name), + ThroughputSource = row.Endpoint.ThroughputSource, + DateUtc = row.DateUtc, + MessageCount = row.MessageCount + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs new file mode 100644 index 0000000000..b0db77217d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MessageRedirectRowMapper.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.MessageRedirects; + +/// <summary> +/// Turns a redirect from the source into the row the product's own save would write. +/// </summary> +static class MessageRedirectRowMapper +{ + public static MessageRedirectEntity Map(MessageRedirect redirect) => new() + { + FromPhysicalAddress = redirect.FromPhysicalAddress, + ToPhysicalAddress = redirect.ToPhysicalAddress, + LastModified = redirect.LastModified + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs new file mode 100644 index 0000000000..959c40ff35 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// A writer that takes rows carrying a <typeparamref name="TDocument" />. It casts every row once, before +/// <see cref="PrepareDocuments" /> sees it, so a row from a reader that yields another type fails the batch by name. +/// </summary> +abstract class MigrationCategoryWriter<TDocument> : IMigrationCategoryWriter where TDocument : class +{ + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract int BatchSize(ServiceControlDbContext dbContext); + + public abstract Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + public Task<PreparedBatch> Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default) => + PrepareDocuments(dbContext, [.. batch.Rows.Select(row => (row, DocumentOf(row)))], cancellationToken); + + /// <summary> + /// Does what <see cref="IMigrationCategoryWriter.Prepare" /> promises, for rows already cast. + /// </summary> + protected abstract Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default); + + /// <summary> + /// Returns a skip for a row whose entity holds a text value the database would refuse, because one such value fails the whole statement the row goes into, + /// or null when every value can be stored. The reason is <see cref="MigrationSkipReason.ValueTooLong" /> for a value longer than its column allows, which is checked first, + /// or <see cref="MigrationSkipReason.MalformedValue" /> for a value holding a character the provider cannot store. + /// </summary> + protected static (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableValueSkip<TEntity>(MigrationInsert<TEntity> insert, IMigrationSqlDialect dialect, MigrationRow row, TEntity entity) where TEntity : class => + insert.ValueTooLongIn(entity, dialect) is { } tooLong + ? (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tooLong}, on document '{row.SourceId}'") + : insert.RejectedCharacterIn(entity, dialect) is { } rejected + ? (row.SourceId, MigrationSkipReason.MalformedValue, $"{rejected}, on document '{row.SourceId}'") + : null; + + TDocument DocumentOf(MigrationRow row) => + row.Document as TDocument + ?? throw new InvalidCastException($"The {CategoryId} writer takes {typeof(TDocument).FullName} documents, but row {row.SourceId} holds {row.Document?.GetType().FullName ?? "no document"}. The reader and writer for one category must agree on the document type."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs new file mode 100644 index 0000000000..326e2af514 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PendingIntegrationEventRowMapper.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.EFCore.Entities; + +static class PendingIntegrationEventRowMapper +{ + /// <summary> + /// Builds the row for one queued request, with its context stored the way the product stores it. Throws when the request has no context, so a caller must skip such a request before it calls this. + /// </summary> + public static ExternalIntegrationDispatchRequestEntity Map(ExternalIntegrationDispatchRequest request) + { + // The product's own serializer, so the dispatcher reads a copied row exactly as it reads one it stored. + var (typeName, json) = DispatchContextSerializer.Serialize(request.DispatchContext); + + return new ExternalIntegrationDispatchRequestEntity { DispatchContextTypeName = typeName, DispatchContextJson = json }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs new file mode 100644 index 0000000000..2ffa313724 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// A batch a writer has turned into rows, which the target inserts in the same transaction that saves the checkpoint. +/// Skips are rows not written, Merges are rows dropped because another row has the same key, and Changes are rows written with a value that differs from the source's. +/// The target logs each skip, merge and change as a warning after the commit. +/// </summary> +sealed record PreparedBatch( + Func<ServiceControlDbContext, CancellationToken, Task<int>> Insert, + int PreparedRowCount, + IReadOnlyList<(string SourceId, MigrationSkipReason Reason, string Detail)> Skips, + IReadOnlyList<(string SourceId, string Detail)> Merges) +{ + public IReadOnlyList<(string SourceId, string Detail)> Changes { get; init; } = []; + + /// <summary> + /// Names the rows the database gives new ids, or is null when it gives none. + /// </summary> + public string? RenumberedRows { get; init; } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs new file mode 100644 index 0000000000..400af82e6c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/RetryOperationRowMapper.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Recoverability; + +/// <summary> +/// Turns a retry operation from the source into the row the target stores. +/// Times are stored as UTC, and a time with no kind is taken as UTC already. +/// </summary> +static class RetryOperationRowMapper +{ + public static HistoricRetryOperationEntity Map(HistoricRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Originator = operation.Originator, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; + + public static UnacknowledgedRetryOperationEntity Map(UnacknowledgedRetryOperation operation) => new() + { + RequestId = operation.RequestId, + RetryType = operation.RetryType, + StartTime = operation.StartTime, + CompletionTime = operation.CompletionTime, + Last = operation.Last, + Originator = operation.Originator, + Classifier = operation.Classifier, + Failed = operation.Failed, + NumberOfMessagesProcessed = operation.NumberOfMessagesProcessed + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs new file mode 100644 index 0000000000..24f64d1de2 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Refuses a copy into a database whose schema is behind this build. The writers insert into tables by name, so +/// a missing schema migration turns into a failure part way through the first category instead of a refusal. +/// </summary> +sealed class SchemaIsCurrentCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the target schema is current"; + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + string[] pending = [.. await dbContext.Database.GetPendingMigrationsAsync(token)]; + + if (pending.Length > 0) + { + throw new Exception($"The database is missing {pending.Length} schema migration(s): {string.Join(", ", pending)}. Run ServiceControl with --setup before starting a migration."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs new file mode 100644 index 0000000000..5e43817ff8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SettingRowMapper.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text.Json; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// Builds the Settings row for a value, storing it under the given key as the JSON text that +/// <c>SettingsStore.StoreSetting</c> writes. <typeparamref name="T" /> is the type the product stores under that key. +/// </summary> +static class SettingRowMapper +{ + public static SettingEntity Map<T>(string key, T value) => new() { Key = key, Value = JsonSerializer.Serialize(value) }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs new file mode 100644 index 0000000000..734283aae7 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SubscriptionRowMapper.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// Turns a subscription from the source into the row the target stores. +/// The version is dropped, because the row has no column for it. +/// </summary> +static class SubscriptionRowMapper +{ + public static SubscriptionEntity Map(SubscriptionRow row) => new() + { + MessageType = row.MessageType, + TransportAddress = row.TransportAddress, + // The value SubscriptionStorage.Subscribe stores for a subscriber with no endpoint. + Endpoint = row.Endpoint ?? row.TransportAddress.Split('@').First() + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs new file mode 100644 index 0000000000..05db8791e5 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Refuses a copy into a database that already holds ServiceControl data. A copy into a database that ran on SQL +/// mixes the customer's rows with rows SQL already held, and a retry of a category could then delete rows the copy +/// never wrote. The check stands down once a row that is not known to be optional exists, because that means a copy +/// has started and what the tables hold is what it wrote. +/// </summary> +sealed class TargetHoldsNoServiceControlDataCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the migration target holds no ServiceControl data"; + + // One entry per mapped entity but the checkpoint: its CLR type, and a query asking whether its table holds a row. + internal static readonly IReadOnlyList<(Type Entity, Func<ServiceControlDbContext, CancellationToken, Task<bool>> HasRows)> Tables = + [ + Entry(dbContext => dbContext.CustomChecks), + Entry(dbContext => dbContext.EndpointSettings), + Entry(dbContext => dbContext.KnownEndpoints), + Entry(dbContext => dbContext.FailedMessages), + Entry(dbContext => dbContext.FailedMessageGroups), + Entry(dbContext => dbContext.GroupComments), + Entry(dbContext => dbContext.MessageRedirects), + Entry(dbContext => dbContext.RetryBatches), + Entry(dbContext => dbContext.RetryBatchNowForwarding), + Entry(dbContext => dbContext.FailedMessageRetries), + Entry(dbContext => dbContext.FailedErrorImports), + Entry(dbContext => dbContext.Settings), + Entry(dbContext => dbContext.Subscriptions), + Entry(dbContext => dbContext.EventLogItems), + Entry(dbContext => dbContext.HistoricRetryOperations), + Entry(dbContext => dbContext.UnacknowledgedRetryOperations), + Entry(dbContext => dbContext.ArchiveOperations), + Entry(dbContext => dbContext.FailedMessageEdits), + Entry(dbContext => dbContext.LicensingEndpoints), + Entry(dbContext => dbContext.LicensingEndpointThroughput), + Entry(dbContext => dbContext.ExternalIntegrationDispatchRequests) + ]; + + // Both halves of an entry come from one DbSet, so the type a test compares and the table the query reads cannot differ. + static (Type Entity, Func<ServiceControlDbContext, CancellationToken, Task<bool>> HasRows) Entry<T>(Func<ServiceControlDbContext, DbSet<T>> set) where T : class => + (typeof(T), (dbContext, cancellationToken) => set(dbContext).AnyAsync(cancellationToken)); + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var checkpointed = await dbContext.MigrationCheckpoints.Select(checkpoint => checkpoint.CategoryId).ToListAsync(token); + + if (checkpointed.Any(id => MigrationCategoryRegistry.Find(id)?.Kind != MigrationCategoryKind.Optional)) + { + return; + } + + var holdingRows = new List<string>(); + + foreach (var (entity, hasRows) in Tables) + { + if (await hasRows(dbContext, token)) + { + holdingRows.Add(dbContext.Model.FindEntityType(entity)!.GetTableName()!); + } + } + + if (holdingRows.Count > 0) + { + throw new Exception($"The database already holds ServiceControl data in {string.Join(", ", holdingRows)}, so ServiceControl has already run on it, and a migration copies only into an empty database. Create a new database, run ServiceControl with --setup against it, and run --start-data-migration before starting ServiceControl on it."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs new file mode 100644 index 0000000000..b13ab708fb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/CustomChecksWriter.cs @@ -0,0 +1,71 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the custom checks under the id the product derives from the endpoint and the check, and never replaces one the target already holds. +/// A row with SourceStorageCheck true is skipped as SourceStorageCheck, which is harmless, and a SourceStorageCheck value that is not a bool throws. +/// A row with a required text value left null is skipped as RequiredValueMissing, and one with a character the database refuses as MalformedValue. +/// </summary> +sealed class CustomChecksWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<CustomCheckDetail> +{ + public override string CategoryId => MigrationCategoryIds.CustomChecks; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<CustomCheckEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.CustomChecks.LongCountAsync(cancellationToken); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, CustomCheckDetail Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<CustomCheckEntity>.For(dbContext); + List<CustomCheckEntity> rows = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, detail) in documents) + { + if (IsSourceStorageCheck(row)) + { + skips.Add((row.SourceId, MigrationSkipReason.SourceStorageCheck, $"'{detail.CustomCheckId}' under '{detail.Category}' is a check the source's own storage runs about itself, which nothing reports once the instance runs on the target")); + continue; + } + + var column = detail.CustomCheckId is null ? nameof(CustomCheckEntity.CustomCheckId) + : detail.Category is null ? nameof(CustomCheckEntity.Category) + : detail.OriginatingEndpoint?.Name is null ? nameof(CustomCheckEntity.OriginatingEndpointName) + : detail.OriginatingEndpoint.Host is null ? nameof(CustomCheckEntity.OriginatingEndpointHost) + : null; + + if (column is not null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"CustomChecks.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = CustomCheckRowMapper.Map(detail); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); + } + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, skips, Merges: [])); + } + + static bool IsSourceStorageCheck(MigrationRow row) => + row.Metadata.GetValueOrDefault(MigrationMetadataKeys.SourceStorageCheck) switch + { + bool flagged => flagged, + null => false, + var other => throw new InvalidOperationException($"Row '{row.SourceId}' carries a {other.GetType().FullName} as its {MigrationMetadataKeys.SourceStorageCheck} value, where the source sets a {typeof(bool).FullName}") + }; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs new file mode 100644 index 0000000000..ee2ef84935 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Copies every per-endpoint setting as stored. +/// Where the name column's collation ignores case, accents, kana type or width, names that differ only in those become one row, and the dropped ones are +/// reported as merges. The row kept is the one whose name matches a known endpoint exactly. +/// </summary> +sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<EndpointSettings> +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<EndpointSettingsEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.EndpointSettings.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<EndpointSettingsEntity>.For(dbContext); + List<(string SourceId, EndpointSettingsEntity Row)> rows = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, settings) in documents) + { + var entity = new EndpointSettingsEntity { Name = settings.Name, TrackInstances = settings.TrackInstances }; + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + rows = await KnownEndpointNamesFirst(dbContext, rows, cancellationToken); + + // The insert keeps the first of any names the database treats as equal. + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } + + async Task<List<(string SourceId, EndpointSettingsEntity Row)>> KnownEndpointNamesFirst(ServiceControlDbContext dbContext, List<(string SourceId, EndpointSettingsEntity Row)> rows, CancellationToken cancellationToken) + { + var nameComparer = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); + string[] mergingNames = [.. rows.GroupBy(row => row.Row.Name, nameComparer).Where(group => group.Count() > 1).SelectMany(group => group.Select(row => row.Row.Name))]; + + if (mergingNames.Length == 0) + { + return rows; + } + + var knownNames = (await dbContext.KnownEndpoints.Where(endpoint => mergingNames.Contains(endpoint.Name)).Select(endpoint => endpoint.Name).ToListAsync(cancellationToken)).ToHashSet(StringComparer.Ordinal); + + // OrderBy is stable, so only the known names move to the front. + return [.. rows.OrderBy(row => knownNames.Contains(row.Row.Name) ? 0 : 1)]; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs new file mode 100644 index 0000000000..6e2b01156f --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedErrorImportsWriter.cs @@ -0,0 +1,146 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Abstractions; +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the failed error imports under the key a live import of the same message derives, and never replaces one the target already holds. +/// A body larger than the target keeps inline goes to body storage, under the body id the SQL import store uses for that key, before the rows are inserted. +/// A second import of the same message, in the batch or already in the target from another document, is reported as a merge. +/// </summary> +sealed class FailedErrorImportsWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter<FailedErrorImport> +{ + public override string CategoryId => MigrationCategoryIds.FailedErrorImports; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<FailedErrorImportEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.FailedErrorImports.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, FailedErrorImport Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<FailedErrorImportEntity>.For(dbContext); + List<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + List<(string SourceId, Guid UniqueMessageId, string Detail)> changes = []; + + foreach (var (row, import) in documents) + { + var column = import.Message?.Id is null ? nameof(FailedErrorImportEntity.MessageId) + : import.Message.Headers is null ? nameof(FailedErrorImportEntity.HeadersJson) + : null; + + if (column is not null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"FailedErrorImports.{column} is NOT NULL and the document has no message, or its message has no value for it")); + continue; + } + + var failedAt = row.Metadata.GetValueOrDefault(MigrationMetadataKeys.FailedAt) as DateTime? + ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} carries no {MigrationMetadataKeys.FailedAt}, so the import's failure time is unknown."); + var body = row.Body + ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} reached the writer without a body. The engine attaches a body to every row of a category that carries bodies, or skips the row."); + var entity = FailedErrorImportRowMapper.Map(import, failedAt, body.Content.Length > persisterSettings.BodyStorage.MaxBodySizeToStore); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity, body)); + + if (EmptiedValues(import) is { } emptied) + { + changes.Add((row.SourceId, entity.UniqueMessageId, emptied)); + } + } + + var stored = await FailureTimesStoredFor(dbContext, rows, cancellationToken); + var (kept, merges) = KeptAndMerged(rows, stored); + + // Every import of one message shares one body id, so only the row the insert keeps may write its body. + await WriteBodies([.. rows.Where(row => row.Row.BodyStoredExternally && !stored.ContainsKey(row.Row.UniqueMessageId) && kept[row.Row.UniqueMessageId] == row.SourceId)], cancellationToken); + + return new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, rows.Count, skips, merges) + { + Changes = [.. changes.Where(change => !stored.ContainsKey(change.UniqueMessageId) && kept[change.UniqueMessageId] == change.SourceId).Select(change => (change.SourceId, change.Detail))] + }; + } + + // FailedErrorImportRowMapper stores a missing exception or body as empty, because both columns are NOT NULL. + static string? EmptiedValues(FailedErrorImport import) => + (import.ExceptionInfo, import.Message!.Body) switch + { + (null, null) => "Stored an empty exception and an empty body, because the document has neither.", + (null, _) => "Stored an empty exception, because the document has none.", + (_, null) => "Stored an empty body, because the document's message has none.", + _ => null + }; + + static async Task<Dictionary<Guid, DateTime>> FailureTimesStoredFor(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows, CancellationToken cancellationToken) + { + var ids = rows.Select(row => row.Row.UniqueMessageId).Distinct().ToArray(); + + return await dbContext.FailedErrorImports + .AsNoTracking() + .Where(import => ids.Contains(import.UniqueMessageId)) + .ToDictionaryAsync(import => import.UniqueMessageId, import => import.FailedAt, cancellationToken); + } + + // A stored row wins over every row of the batch, and an earlier row over a later one, as in the insert. A stored row with the same failure time is this document's own row, which a retry reads again. + static (Dictionary<Guid, string?> Kept, List<(string SourceId, string Detail)> Merges) KeptAndMerged(IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> rows, Dictionary<Guid, DateTime> stored) + { + var kept = stored.ToDictionary(entry => entry.Key, entry => (string?)null); + List<(string SourceId, string Detail)> merges = []; + + foreach (var (sourceId, row, _) in rows) + { + if (!kept.TryGetValue(row.UniqueMessageId, out var keeper) || (keeper is null && SameFailureTime(stored[row.UniqueMessageId], row.FailedAt))) + { + kept[row.UniqueMessageId] = sourceId; + } + else if (keeper is null) + { + merges.Add((sourceId, $"FailedErrorImports already holds an import of the same message, '{row.MessageId}', that failed at {stored[row.UniqueMessageId]:O}, while this document failed at {row.FailedAt:O}, so it kept the stored row and dropped this one")); + } + else + { + merges.Add((sourceId, $"'{keeper}' is an import of the same message, '{row.MessageId}', so FailedErrorImports kept the row from '{keeper}' and dropped the row from '{sourceId}'")); + } + } + + return (kept, merges); + } + + // PostgreSQL keeps only whole microseconds, so the stored time of the same document can differ from a re-read by less than one. + static bool SameFailureTime(DateTime stored, DateTime read) => Math.Abs((stored - read).Ticks) < TimeSpan.TicksPerMicrosecond; + + // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. + Task WriteBodies(IReadOnlyList<(string SourceId, FailedErrorImportEntity Row, MigrationBody Body)> externalBodies, CancellationToken cancellationToken) => + Task.WhenAll(externalBodies.Select(external => WriteBody(external, cancellationToken))); + + async Task WriteBody((string SourceId, FailedErrorImportEntity Row, MigrationBody Body) external, CancellationToken cancellationToken) + { + var bodyId = FailedErrorImportEntity.ExternalBodyId(external.Row.UniqueMessageId); + + try + { + await storagePersistence.WriteBody(bodyId, external.Body.Content, external.Body.ContentType, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // The storage SDK errors do not name the body, so without this the error that stops the copy would not say which import failed. + catch (Exception ex) + { + throw new InvalidOperationException($"Writing the body of document '{external.SourceId}' in category {CategoryId} to external body storage as '{bodyId}' ({external.Body.Content.Length} bytes) failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs new file mode 100644 index 0000000000..a1f596b00c --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/FailedMessagesWriter.cs @@ -0,0 +1,199 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Abstractions; +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes failed messages, their failure group rows and a retry claim for each retry-issued message the source still +/// held one for, inserting only messages the target does not already hold and never updating one it does. The claims +/// mirror the source, so a retry by id behaves as it did before the move. A retry-issued row with no bool RetryClaimHeld +/// value fails the batch. Each category that copies failed messages derives from this and names its category id and the +/// statuses it copies. +/// </summary> +abstract class FailedMessagesWriterBase(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) : MigrationCategoryWriter<FailedMessage> +{ + // Like the claim a finished retry batch leaves, this points at no batch, so nothing stages it until it is released by message id. + public static readonly Guid MigratedRetryClaimBatchId = new("5c1b7a0e-2f4d-4e8a-9b36-7d0c4f1e8a52"); + + /// <summary> + /// The statuses this category copies. <see cref="Count" /> counts only rows with one of these statuses, because + /// both failed message categories share one table. + /// </summary> + protected abstract IReadOnlyCollection<FailedMessageStatus> Statuses { get; } + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<FailedMessageEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.FailedMessages + .Where(message => Statuses.Contains(message.Status)) + .LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, FailedMessage Document)> documents, CancellationToken cancellationToken = default) + { + var messageInsert = MigrationInsert<FailedMessageEntity>.For(dbContext); + var groupInsert = MigrationInsert<FailedMessageGroupEntity>.For(dbContext); + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + List<FailedMessageDocumentTranslation.Translated> translated = []; + HashSet<Guid> translatedIds = []; + List<(string SourceId, FailedMessageEntity Row, MigrationBody Body)> externalBodies = []; + List<(string SourceId, Guid UniqueMessageId, string Detail)> changes = []; + HashSet<Guid> heldClaims = []; + var groupIds = migrationDialect.KeyComparer(typeof(FailedMessageGroupEntity), nameof(FailedMessageGroupEntity.GroupId)); + + foreach (var (row, document) in documents) + { + switch (NamingTheDocument(row, () => FailedMessageDocumentTranslator.Translate(row))) + { + case FailedMessageDocumentTranslation.Translated translation: + var body = row.Body ?? throw new InvalidOperationException($"Row '{row.SourceId}' in category {CategoryId} reached the writer without a body. The engine attaches a body to every row of a category that carries bodies, or skips the row."); + // Set before the length check, because the stored content type is a text column with a limit. + var storeExternally = NamingTheDocument(row, () => FailedMessageDocumentTranslator.SetBody(translation.Row, document.ProcessingAttempts[^1].Headers, body, persisterSettings.BodyStorage.MaxBodySizeToStore)); + + // A message copied without one of its groups would be missing from that group, so a group that cannot be stored skips the message. + var unstorable = UnstorableValueSkip(messageInsert, migrationDialect, row, translation.Row) + ?? translation.Groups.Select(group => UnstorableValueSkip(groupInsert, migrationDialect, row, group)).FirstOrDefault(groupSkip => groupSkip is not null); + + if (unstorable is { } unstorableSkip) + { + skips.Add(unstorableSkip); + } + else + { + translated.Add(translation); + + // The insert keeps only the first document with an id, so only its body is stored under that id, only its changes are reported and only its claim is copied. + if (translatedIds.Add(translation.Row.UniqueMessageId)) + { + if (storeExternally) + { + externalBodies.Add((row.SourceId, translation.Row, body)); + } + + if (translation.Row.Status == FailedMessageStatus.RetryIssued && SourceHeldRetryClaim(row)) + { + heldClaims.Add(translation.Row.UniqueMessageId); + } + + if (ChangesIn(translation, groupIds) is { Count: > 0 } changed) + { + changes.Add((row.SourceId, translation.Row.UniqueMessageId, string.Join(" ", changed))); + } + } + } + + break; + case FailedMessageDocumentTranslation.Skipped skip: + skips.Add((row.SourceId, skip.Reason, skip.Detail)); + break; + default: + throw new InvalidOperationException($"The translator returned a translation the writer does not know for document '{row.SourceId}'."); + } + } + + // This only skips the bodies and changes of messages the target already holds; InsertMissing still decides which rows go in. + var present = externalBodies.Count == 0 && changes.Count == 0 + ? [] + : await MessagesTheTargetHolds(dbContext, [.. externalBodies.Select(external => external.Row.UniqueMessageId), .. changes.Select(change => change.UniqueMessageId)], cancellationToken); + + // All bodies are written before the transaction opens, so a crash leaves a body with no row rather than a row pointing at a body that was never stored. + await Task.WhenAll(externalBodies + .Where(external => !present.Contains(external.Row.UniqueMessageId)) + .Select(external => WriteBody(external, cancellationToken))); + + return new PreparedBatch((context, token) => InsertFailedMessages(context, translated, heldClaims, token), translated.Count, skips, Merges: []) + { + Changes = [.. changes.Where(change => !present.Contains(change.UniqueMessageId)).Select(change => (change.SourceId, change.Detail))] + }; + } + + static async Task<HashSet<Guid>> MessagesTheTargetHolds(ServiceControlDbContext dbContext, Guid[] ids, CancellationToken cancellationToken) => + (await dbContext.FailedMessages + .AsNoTracking() + .Where(message => ids.Contains(message.UniqueMessageId)) + .Select(message => message.UniqueMessageId) + .ToListAsync(cancellationToken)).ToHashSet(); + + // The group insert keeps the first of the ids its key treats as one, so a later group with such an id loses its title and type. + static IReadOnlyList<string> ChangesIn(FailedMessageDocumentTranslation.Translated translation, IEqualityComparer<string> groupIds) => + [ + .. translation.Changes, + .. translation.Groups + .GroupBy(group => group.GroupId, groupIds) + .Where(sameKey => sameKey.Count() > 1) + .Select(sameKey => $"Kept the failure group '{sameKey.First().GroupId}' titled '{sameKey.First().Title}' and dropped {string.Join(", ", sameKey.Skip(1).Select(group => $"'{group.GroupId}' titled '{group.Title}'"))}, because SQL treats their ids as one key.") + ]; + + // Throws rather than guess, because a wrong guess either blocks a retry by id or lets one send the message twice. + static bool SourceHeldRetryClaim(MigrationRow row) => + row.Metadata.GetValueOrDefault(MigrationMetadataKeys.RetryClaimHeld) switch + { + bool held => held, + null => throw new InvalidOperationException($"Row '{row.SourceId}' is retry-issued but arrived without a {MigrationMetadataKeys.RetryClaimHeld} value, which the source resolves for every retry-issued failed message"), + var other => throw new InvalidOperationException($"Row '{row.SourceId}' carries a {other.GetType().FullName} as its {MigrationMetadataKeys.RetryClaimHeld} value, where the source sets a {typeof(bool).FullName}") + }; + + async Task WriteBody((string SourceId, FailedMessageEntity Row, MigrationBody Body) external, CancellationToken cancellationToken) + { + var bodyId = external.Row.UniqueMessageId.ToString(); + + try + { + await storagePersistence.WriteBody(bodyId, external.Body.Content, external.Body.ContentType, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // The storage SDK errors do not name the body, so without this the error that stops the copy would not say which message failed. + catch (Exception ex) + { + throw new InvalidOperationException($"Writing the body of document '{external.SourceId}' in category {CategoryId} to external body storage as '{bodyId}' ({external.Body.Content.Length} bytes) failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } + + // Adds the document id to the error, because the bare error names no document and the source can hold millions. + T NamingTheDocument<T>(MigrationRow row, Func<T> step) + { + try + { + return step(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Preparing document '{row.SourceId}' in category {CategoryId} failed with {ex.GetType().Name}: {ex.Message}", ex); + } + } + + async Task<int> InsertFailedMessages(ServiceControlDbContext dbContext, IReadOnlyList<FailedMessageDocumentTranslation.Translated> translated, IReadOnlySet<Guid> heldClaims, CancellationToken cancellationToken) + { + var inserted = (await migrationDialect.InsertMissing(dbContext, [.. translated.Select(translation => translation.Row)], cancellationToken)) + .ToHashSet<FailedMessageEntity>(ReferenceEqualityComparer.Instance); + + // Matched by reference, because InsertMissing returns the row objects it kept, and a document it turned away can share a kept row's id. + // A group goes in only when the message its foreign key points at went in, so a message already in the target keeps the groups it had. + await migrationDialect.InsertMissing(dbContext, [.. translated.Where(translation => inserted.Contains(translation.Row)).SelectMany(translation => translation.Groups)], cancellationToken); + + // The claims mirror the source, so a retry by id stays blocked or goes through exactly as it would have before the move. + await migrationDialect.InsertMissing(dbContext, [.. inserted + .Where(row => heldClaims.Contains(row.UniqueMessageId)) + .Select(row => new FailedMessageRetryEntity { UniqueMessageId = row.UniqueMessageId, RetryBatchId = MigratedRetryClaimBatchId })], cancellationToken); + + return inserted.Count; + } +} + +/// <summary> +/// Writes the unresolved and retry-issued failed messages. +/// </summary> +sealed class FailedMessagesWriter(IMigrationSqlDialect migrationDialect, IBodyStoragePersistence storagePersistence, EFPersisterSettings persisterSettings) + : FailedMessagesWriterBase(migrationDialect, storagePersistence, persisterSettings) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; + + protected override IReadOnlyCollection<FailedMessageStatus> Statuses => MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs new file mode 100644 index 0000000000..3179f62c16 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/GroupCommentsWriter.cs @@ -0,0 +1,79 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Copies the group comments, one row per comment keyed on its group id, and never replaces a comment the target already holds. +/// A null or blank comment is skipped as BlankGroupComment, because the product keeps no row for one. +/// A comment whose group id the key column treats as another's is reported as a merge. +/// A copied comment on a group the target holds no failed message in is reported as a change, because the retention sweep deletes it once ServiceControl opens. +/// </summary> +sealed class GroupCommentsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<GroupComment> +{ + public override string CategoryId => MigrationCategoryIds.GroupComments; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<GroupCommentEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.GroupComments.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, GroupComment Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<GroupCommentEntity>.For(dbContext); + var rows = new List<(string SourceId, GroupCommentEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, comment) in documents) + { + if (string.IsNullOrWhiteSpace(comment.Comment)) + { + skips.Add((row.SourceId, MigrationSkipReason.BlankGroupComment, $"the comment on group '{comment.Id}' is blank, and the product keeps no row for a blank comment")); + continue; + } + + var entity = GroupCommentRowMapper.Map(comment); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + var (merges, inserted) = await KeyMerges.Split(dbContext, migrationDialect, rows, cancellationToken: cancellationToken); + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + merges) + { + Changes = await CommentsTheSweepWillDelete(dbContext, inserted, cancellationToken) + }; + } + + // Group comments copy after the unresolved failed messages, so the groups the target holds now are the groups it holds when ServiceControl opens. + async Task<IReadOnlyList<(string SourceId, string Detail)>> CommentsTheSweepWillDelete(ServiceControlDbContext dbContext, IReadOnlyList<(string SourceId, GroupCommentEntity Row)> inserted, CancellationToken cancellationToken) + { + string[] groupIds = [.. inserted.Select(comment => comment.Row.GroupId).Distinct(StringComparer.Ordinal)]; + var held = (await dbContext.FailedMessageGroups + .AsNoTracking() + .Where(group => groupIds.Contains(group.GroupId)) + .Select(group => group.GroupId) + .ToListAsync(cancellationToken)) + .ToHashSet(migrationDialect.KeyComparer(typeof(FailedMessageGroupEntity), nameof(FailedMessageGroupEntity.GroupId))); + + return [.. inserted + .Where(comment => !held.Contains(comment.Row.GroupId)) + .Select(comment => (comment.SourceId, $"The group '{comment.Row.GroupId}' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + + "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + $"The comment was: '{comment.Row.Comment}'"))]; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs new file mode 100644 index 0000000000..6650d9b5fb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KeyMerges.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Infrastructure; +using Microsoft.EntityFrameworkCore; + +/// <summary> +/// Finds the rows the insert will drop because another row, earlier in the batch or already in the table, has the same +/// key under the column's collation. A row the table already holds exactly is not a merge. +/// Returns each dropped row's source id with a line naming both keys, the source row whose key was kept, or the stored row, and the source row dropped. +/// Split also returns the rows the insert adds, and takes a describeSource that turns a source id into the words the line uses for it, quoting it by default. +/// </summary> +static class KeyMerges +{ + public static async Task<IReadOnlyList<(string SourceId, string Detail)>> In<TEntity>(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, CancellationToken cancellationToken = default) where TEntity : class => + (await Split(dbContext, dialect, rows, cancellationToken: cancellationToken)).Merges; + + public static async Task<(IReadOnlyList<(string SourceId, string Detail)> Merges, IReadOnlyList<(string SourceId, TEntity Row)> Inserted)> Split<TEntity>(ServiceControlDbContext dbContext, IMigrationSqlDialect dialect, IReadOnlyList<(string SourceId, TEntity Row)> rows, Func<string, string>? describeSource = null, CancellationToken cancellationToken = default) where TEntity : class + { + describeSource ??= sourceId => $"'{sourceId}'"; + var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; + var keyProperties = entityType.FindPrimaryKey()!.Properties; + var queryColumn = keyProperties.FirstOrDefault(property => property.ClrType == typeof(string)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no text key column, so no collation can join two of its keys."); + Func<object?, object?, bool>[] columnMatches = [.. keyProperties.Select(property => property.ClrType == typeof(string) ? TextMatch(dialect.KeyComparer(typeof(TEntity), property.Name)) : Equals)]; + + object?[] KeyOf(TEntity entity) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(entity))]; + bool SameKey(object?[] first, object?[] second) => columnMatches.Select((matches, column) => matches(first[column], second[column])).All(same => same); + + var stored = new List<TEntity>(); + string[] queryValues = [.. rows.Select(row => (string)queryColumn.GetGetter().GetClrValue(row.Row)!).Distinct(StringComparer.Ordinal)]; + + foreach (var chunk in queryValues.Chunk(dialect.RowsPerStatementFor<TEntity>(dbContext))) + { + stored.AddRange(await dbContext.Set<TEntity>().AsNoTracking().Where(entity => chunk.Contains(EF.Property<string>(entity, queryColumn.Name))).ToListAsync(cancellationToken)); + } + + // A stored key wins over every row of the batch, and an earlier row over a later one, as in the insert. + // SourceId is null for a stored row no row of the batch has claimed. + var kept = stored.Select(entity => (Key: KeyOf(entity), SourceId: (string?)null)).ToList(); + var merges = new List<(string SourceId, string Detail)>(); + var inserted = new List<(string SourceId, TEntity Row)>(); + + foreach (var (sourceId, row) in rows) + { + var key = KeyOf(row); + var match = kept.FindIndex(entry => SameKey(entry.Key, key)); + + if (match < 0) + { + kept.Add((key, sourceId)); + inserted.Add((sourceId, row)); + } + else if (kept[match].SourceId is null && kept[match].Key.SequenceEqual(key)) + { + // Claimed by this row, so a later row of the batch with the same key is a merge. + kept[match] = (key, sourceId); + } + else + { + var keptRow = kept[match].SourceId is { } keptSourceId ? $"the row from {describeSource(keptSourceId)}" : "the row the table already held"; + merges.Add((sourceId, $"{entityType.GetTableName()} treats {Describe(key)} and {Describe(kept[match].Key)} as one key, so it kept {Describe(kept[match].Key)} and dropped the row for {Describe(key)}. It kept {keptRow} and dropped the row from {describeSource(sourceId)}.")); + } + } + + return (merges, inserted); + } + + static Func<object?, object?, bool> TextMatch(IEqualityComparer<string> comparer) => (first, second) => comparer.Equals((string?)first, (string?)second); + + static string Describe(object?[] key) => string.Join(", ", key.Select(value => $"'{value}'")); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs new file mode 100644 index 0000000000..fb7e845a67 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs @@ -0,0 +1,61 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the endpoints ServiceControl has heard from. The row's key is worked out from the endpoint's own +/// details rather than carried over, so the same endpoint lands on the same row whichever side wrote it first. +/// </summary> +sealed class KnownEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<KnownEndpoint> +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<KnownEndpointEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.KnownEndpoints.LongCountAsync(cancellationToken); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, KnownEndpoint Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<KnownEndpointEntity>.For(dbContext); + var rows = new List<KnownEndpointEntity>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, endpoint) in documents) + { + if (endpoint.EndpointDetails?.Name is null || endpoint.EndpointDetails.Host is null) + { + var column = endpoint.EndpointDetails?.Name is null ? nameof(KnownEndpointEntity.Name) : nameof(KnownEndpointEntity.Host); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"KnownEndpoints.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = new KnownEndpointEntity + { + Id = endpoint.EndpointDetails.GetDeterministicId(), + Name = endpoint.EndpointDetails.Name, + HostId = endpoint.EndpointDetails.HostId, + Host = endpoint.EndpointDetails.Host, + Monitored = endpoint.Monitored + }; + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add(entity); + } + + return Task.FromResult(new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + skips, + Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs new file mode 100644 index 0000000000..1e9e702dc9 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensedEndpointDetailsWriter.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Infrastructure; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the licensed endpoint details file as one row in Settings. A row the target already holds is left as it is +/// and counted as already present. +/// </summary> +sealed class LicensedEndpointDetailsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter<LicensedEndpointDetails>(migrationDialect, SettingKeys.LicensedEndpointDetails) +{ + public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs new file mode 100644 index 0000000000..fc653b0d12 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingEndpointsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the licensing endpoints. An endpoint the target already holds, or a second one in the same batch whose key +/// the column treats as the same, is left as it is and counted as already present. +/// Each endpoint folded into another's key is also reported as a merge, naming both. +/// </summary> +sealed class LicensingEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<Endpoint> +{ + public override string CategoryId => MigrationCategoryIds.LicensingEndpoints; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<LicensingEndpointEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpoints.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, Endpoint Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<LicensingEndpointEntity>.For(dbContext); + var rows = new List<(string SourceId, LicensingEndpointEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, endpoint) in documents.OrderBy(document => document.Row.SourceId, StringComparer.Ordinal)) + { + if (endpoint.Id.Name is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "LicensingEndpoints.Name and NormalizedName are NOT NULL and the document has no endpoint name")); + continue; + } + + var entity = LicensingEndpointRowMapper.Map(endpoint); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs new file mode 100644 index 0000000000..6abd48b3d3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingReportMasksWriter.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the report masks as one row in Settings. A row the target already holds is left as it is and counted as +/// already present. +/// </summary> +sealed class LicensingReportMasksWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter<List<string>>(migrationDialect, SettingKeys.ReportMasks) +{ + public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs new file mode 100644 index 0000000000..eda0e848e0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/LicensingThroughputWriter.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Implementation; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Adds each day's message count to the stored count for that day, and the database sums endpoints whose names its key treats as one. +/// A day repeated for one source endpoint keeps its last row, and a retry deletes every copied day first. +/// It reports no merge, because the endpoints writer reports an endpoint the key folds into another. +/// Throws <see cref="InvalidOperationException" /> when a batch names an endpoint the target does not hold. +/// </summary> +sealed class LicensingThroughputWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<ThroughputDayRow>, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.LicensingThroughput; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<LicensingEndpointThroughputEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpointThroughput.LongCountAsync(cancellationToken); + + public Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.LicensingEndpointThroughput.ExecuteDeleteAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, ThroughputDayRow Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<LicensingEndpointThroughputEntity>.For(dbContext); + var mapped = new List<(EndpointIdentifier Endpoint, LicensingEndpointThroughputEntity Day)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + // Date order is the order the product's own throughput write takes row locks in. + foreach (var (row, day) in documents.OrderBy(document => document.Document.DateUtc).ThenBy(document => document.Row.SourceId, StringComparer.Ordinal)) + { + if (day.Endpoint.Name is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "LicensingEndpointThroughput.NormalizedName is NOT NULL and the day's endpoint has no name")); + continue; + } + + var entity = LicensingThroughputRowMapper.Map(day); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + mapped.Add((day.Endpoint, entity)); + } + + foreach (var endpoint in mapped.Select(day => day.Endpoint).Distinct()) + { + var normalizedName = LicensingDataStore.Normalize(endpoint.Name); + + // Checked in SQL rather than in memory, so the name compares under the column's collation as the foreign key does. + if (!await dbContext.LicensingEndpoints.AnyAsync(row => row.NormalizedName == normalizedName && row.ThroughputSource == endpoint.ThroughputSource, cancellationToken)) + { + throw new InvalidOperationException($"Endpoint {endpoint.Name} from {endpoint.ThroughputSource} does not exist in the migration target, so its daily throughput cannot be copied. LicensingThroughput copies only after LicensingEndpoints has finished."); + } + } + + List<LicensingEndpointThroughputEntity> rows = + [ + .. mapped + .GroupBy(day => (day.Endpoint.Name, day.Endpoint.ThroughputSource, day.Day.DateUtc)) + .Select(sameEndpointDay => sameEndpointDay.Last().Day) + ]; + + // Adding is safe to resume, because the batch and its checkpoint commit in one transaction, and a replayed checkpoint save is refused. + return new PreparedBatch((context, token) => migrationDialect.AddLicensingEndpointThroughput(context, rows, token), mapped.Count, skips, Merges: []); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs new file mode 100644 index 0000000000..9544efd9d9 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/MessageRedirectsWriter.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; + +/// <summary> +/// Copies the message redirects, one row per redirect. A redirect whose source address is already taken is reported as a merge. +/// A redirect that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// </summary> +sealed class MessageRedirectsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<MessageRedirect> +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<MessageRedirectEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.MessageRedirects.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, MessageRedirect Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<MessageRedirectEntity>.For(dbContext); + var rows = new List<(string SourceId, MessageRedirectEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, redirect) in documents) + { + if (redirect.FromPhysicalAddress is null || redirect.ToPhysicalAddress is null) + { + var column = redirect.FromPhysicalAddress is null ? nameof(MessageRedirectEntity.FromPhysicalAddress) : nameof(MessageRedirectEntity.ToPhysicalAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"MessageRedirects.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = MessageRedirectRowMapper.Map(redirect); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, rows, cancellationToken)); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs new file mode 100644 index 0000000000..85c0842e91 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/NotificationSettingsWriter.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Infrastructure; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the notification email settings as one Settings row. +/// </summary> +sealed class NotificationSettingsWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter<EmailNotifications>(migrationDialect, SettingKeys.NotificationEmails) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs new file mode 100644 index 0000000000..2c21cda8de --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/PendingIntegrationEventsWriter.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the waiting integration events under new keys from the database, so a retry's reset empties the table before they are read again. +/// A request with no context, a type name longer than its column or a character the database refuses is skipped as RequiredValueMissing, ValueTooLong or MalformedValue. +/// The rest of the batch still copies. +/// </summary> +sealed class PendingIntegrationEventsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<ExternalIntegrationDispatchRequest>, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.PendingIntegrationEvents; + + public override int BatchSize(ServiceControlDbContext dbContext) => StoreKeyedInsert.BatchSize; + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.ExternalIntegrationDispatchRequests.LongCountAsync(cancellationToken); + + // The whole table, because the target is refused unless it held no ServiceControl data before the first copy. + public Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.ExternalIntegrationDispatchRequests.ExecuteDeleteAsync(cancellationToken); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, ExternalIntegrationDispatchRequest Document)> documents, CancellationToken cancellationToken = default) + { + var textColumns = StoreKeyedInsert.TextColumns<ExternalIntegrationDispatchRequestEntity>(dbContext); + List<ExternalIntegrationDispatchRequest> requests = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, request) in documents) + { + if (request.DispatchContext is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "ExternalIntegrationDispatchRequests.DispatchContextTypeName and DispatchContextJson are NOT NULL and the document has no context to serialize")); + } + else if (StoreKeyedInsert.UnstorableValueSkip(textColumns, migrationDialect, row, PendingIntegrationEventRowMapper.Map(request)) is { } unstorable) + { + skips.Add(unstorable); + } + else + { + requests.Add(request); + } + } + + return Task.FromResult(new PreparedBatch( + (context, token) => StoreKeyedInsert.AddAll(context, requests, PendingIntegrationEventRowMapper.Map, token), + requests.Count, + skips, + Merges: []) + { + RenumberedRows = requests.Count == 0 ? null : "pending integration events" + }); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs new file mode 100644 index 0000000000..63440ed63d --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/RetryOperationsWriter.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Recoverability; + +/// <summary> +/// Copies the retry history into two tables. A pending operation whose key is already taken is reported as a merge. +/// Historic operations get new keys, so a retry's reset empties both tables before the history is read again. +/// An operation that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// </summary> +sealed class RetryOperationsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<RetryOperationRow>, IStoreKeyedCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<UnacknowledgedRetryOperationEntity>(dbContext); + + public override async Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.LongCountAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.LongCountAsync(cancellationToken); + + // Whole tables, because the target is refused unless it held no ServiceControl data before the first copy. + public async Task<int> DeleteCopiedRows(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + await dbContext.HistoricRetryOperations.ExecuteDeleteAsync(cancellationToken) + await dbContext.UnacknowledgedRetryOperations.ExecuteDeleteAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, RetryOperationRow Document)> documents, CancellationToken cancellationToken = default) + { + var pendingInsert = MigrationInsert<UnacknowledgedRetryOperationEntity>.For(dbContext); + var historicText = StoreKeyedInsert.TextColumns<HistoricRetryOperationEntity>(dbContext); + List<HistoricRetryOperation> historic = []; + List<(string SourceId, UnacknowledgedRetryOperationEntity Row)> unacknowledged = []; + List<(string SourceId, MigrationSkipReason Reason, string Detail)> skips = []; + + foreach (var (row, operation) in documents) + { + if (operation.Historic is { } historicOperation) + { + if (historicOperation.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it")); + } + else if (StoreKeyedInsert.UnstorableValueSkip(historicText, migrationDialect, row, RetryOperationRowMapper.Map(historicOperation)) is { } unstorable) + { + skips.Add(unstorable); + } + else + { + historic.Add(historicOperation); + } + + continue; + } + + var pending = operation.Unacknowledged!; + + if (pending.RequestId is null) + { + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it")); + continue; + } + + var entity = RetryOperationRowMapper.Map(pending); + + if (UnstorableValueSkip(pendingInsert, migrationDialect, row, entity) is { } unstorablePending) + { + skips.Add(unstorablePending); + continue; + } + + unacknowledged.Add((row.SourceId, entity)); + } + + // Reversed, because the insert keeps the first of a repeated key and the latest retry must win. + unacknowledged.Reverse(); + + return new PreparedBatch( + async (context, token) => + { + var inserted = await migrationDialect.InsertMissing(context, [.. unacknowledged.Select(pending => pending.Row)], token); + + // Reversed so the first in the document is added last and gets the highest id, which wins a tie on completion time. + // That holds only while the store hands out ids in add order. + await StoreKeyedInsert.AddAll(context, [.. Enumerable.Reverse(historic)], (HistoricRetryOperation operation) => RetryOperationRowMapper.Map(operation), token); + + return historic.Count + inserted.Count; + }, + historic.Count + unacknowledged.Count, + skips, + await KeyMerges.In(dbContext, migrationDialect, unacknowledged, cancellationToken)) + { + RenumberedRows = historic.Count == 0 ? null : "historic retry operations" + }; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs new file mode 100644 index 0000000000..8d285689bb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SettingRowWriter.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes a category that the target stores as one JSON row in Settings. +/// A row already under that key is left alone and counted as already present. +/// </summary> +abstract class SettingRowWriter<TDocument>(IMigrationSqlDialect migrationDialect, string settingKey) : MigrationCategoryWriter<TDocument> where TDocument : class +{ + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<SettingEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Settings.LongCountAsync(setting => setting.Key == settingKey, cancellationToken); + + /// <summary> + /// Builds the Settings row for one document, storing the whole document by default. + /// Override it when the document holds the value rather than being it. + /// </summary> + protected virtual SettingEntity RowFor(string key, TDocument document) => SettingRowMapper.Map(key, document); + + protected override Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default) + { + List<SettingEntity> rows = [.. documents.Select(document => RowFor(settingKey, document.Document))]; + + return Task.FromResult(new PreparedBatch(async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, rows.Count, [], Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs new file mode 100644 index 0000000000..c21ac50312 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/StoreKeyedInsert.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes rows whose primary key the database generates. +/// The model-driven insert refuses such rows, so this class adds them through EF in the caller's context, and the database assigns each key. +/// It also holds its own copy of the text check, because the base writer's check needs that insert. +/// </summary> +static class StoreKeyedInsert +{ + // EF splits its own statements under each provider's limits, so this sizes only how many rows one checkpoint covers. + public const int BatchSize = 500; + + /// <summary> + /// Returns every text column of <typeparamref name="TEntity" />, each with its name as "table.column" for a skip's detail. Read it once per batch and pass it to <see cref="UnstorableValueSkip{TEntity}" />. + /// </summary> + public static (IProperty Property, string TableAndColumn)[] TextColumns<TEntity>(ServiceControlDbContext dbContext) where TEntity : class + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity))!; + + return [.. entityType.GetProperties().Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{entityType.GetTableName()}.{property.GetColumnName()}"))]; + } + + /// <summary> + /// Returns a skip when <paramref name="entity" />, the row mapped from the source document in <paramref name="row" />, holds a text value the database would refuse, or null when every value can be stored. + /// The reason is ValueTooLong for a value longer than its column, or MalformedValue for a value holding a character the provider cannot store. The skip carries the source id of <paramref name="row" />, and its detail names the table, the column and the document. + /// </summary> + public static (string SourceId, MigrationSkipReason Reason, string Detail)? UnstorableValueSkip<TEntity>((IProperty Property, string TableAndColumn)[] textColumns, IMigrationSqlDialect dialect, MigrationRow row, TEntity entity) where TEntity : class + { + foreach (var (property, tableAndColumn) in textColumns) + { + if (property.GetGetter().GetClrValue(entity) is string { Length: var units } value && property.GetMaxLength() is { } maxLength && units > maxLength && dialect.TextLength(value) is var length && length > maxLength) + { + return (row.SourceId, MigrationSkipReason.ValueTooLong, $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}, on document '{row.SourceId}'"); + } + } + + foreach (var (property, tableAndColumn) in textColumns) + { + if (property.GetGetter().GetClrValue(entity) is string value && dialect.RejectedCharacterIn(value) is { } rejected) + { + return (row.SourceId, MigrationSkipReason.MalformedValue, $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains, on document '{row.SourceId}'"); + } + } + + return null; + } + + /// <summary> + /// Builds a new entity for each source row with <paramref name="map" />, adds them all to the caller's context in the order given, and saves inside the transaction the target already holds. + /// Returns how many rows it added, which is all of them, because the save writes every row or throws. Call it inside the target's insert, so each attempt of the execution strategy builds fresh entities. + /// </summary> + public static async Task<int> AddAll<TSource, TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TSource> sources, Func<TSource, TEntity> map, CancellationToken cancellationToken = default) where TEntity : class + { + // Mapped on every call, because a failed save can leave generated ids on the entities it tried. + dbContext.Set<TEntity>().AddRange(sources.Select(map)); + await dbContext.SaveChangesAsync(cancellationToken); + + return sources.Count; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs new file mode 100644 index 0000000000..29a57fd6c8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/SubscriptionsWriter.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Copies the subscriptions. The key leaves out the message type's version, so two versions of one type for one +/// subscriber become one row, and the second is reported as a merge. +/// A subscription that cannot be stored is skipped with a reason, and the rest of the batch still copies. +/// </summary> +sealed class SubscriptionsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter<SubscriptionRow> +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatementFor<SubscriptionEntity>(dbContext); + + public override Task<long> Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.Subscriptions.LongCountAsync(cancellationToken); + + protected override async Task<PreparedBatch> PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, SubscriptionRow Document)> documents, CancellationToken cancellationToken = default) + { + var insert = MigrationInsert<SubscriptionEntity>.For(dbContext); + var rows = new List<(string SourceId, SubscriptionEntity Row)>(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, subscription) in documents) + { + if (subscription.MessageType is null || subscription.TransportAddress is null) + { + var column = subscription.MessageType is null ? nameof(SubscriptionEntity.MessageType) : nameof(SubscriptionEntity.TransportAddress); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"Subscriptions.{column} is NOT NULL and the document has no value for it")); + continue; + } + + var entity = SubscriptionRowMapper.Map(subscription); + + if (UnstorableValueSkip(insert, migrationDialect, row, entity) is { } unstorable) + { + skips.Add(unstorable); + continue; + } + + rows.Add((row.SourceId, entity)); + } + + // The key leaves the version out, so a merge line names each row's version and endpoint as well as its source id. + var subscriptions = documents.ToDictionary(document => document.Row.SourceId, document => document.Document, StringComparer.Ordinal); + var (merges, _) = await KeyMerges.Split(dbContext, migrationDialect, rows, sourceId => $"'{sourceId}' (version {subscriptions[sourceId].Version}, endpoint '{subscriptions[sourceId].Endpoint}')", cancellationToken); + + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, [.. rows.Select(row => row.Row)], token)).Count, + rows.Count, + skips, + merges); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs new file mode 100644 index 0000000000..5e0ace65ad --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/TrialEndDateWriter.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using Entities; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Writes the trial end date as one Settings row, storing only the date, as the product does. +/// </summary> +sealed class TrialEndDateWriter(IMigrationSqlDialect migrationDialect) : SettingRowWriter<TrialMetadata>(migrationDialect, SettingKeys.TrialEndDate) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override SettingEntity RowFor(string key, TrialMetadata document) => SettingRowMapper.Map(key, document.TrialEndDate); +} diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 9ff490aa65..2b7acaa01e 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -38,6 +38,7 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont public DbSet<LicensingEndpointEntity> LicensingEndpoints { get; set; } public DbSet<LicensingEndpointThroughputEntity> LicensingEndpointThroughput { get; set; } public DbSet<ExternalIntegrationDispatchRequestEntity> ExternalIntegrationDispatchRequests { get; set; } + public DbSet<MigrationCheckpointEntity> MigrationCheckpoints { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); @@ -80,6 +81,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new LicensingEndpointConfiguration()); modelBuilder.ApplyConfiguration(new LicensingEndpointThroughputConfiguration()); modelBuilder.ApplyConfiguration(new ExternalIntegrationDispatchRequestConfiguration()); + modelBuilder.ApplyConfiguration(new MigrationCheckpointConfiguration()); } public abstract bool IsDuplicateKeyException(DbUpdateException exception); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs new file mode 100644 index 0000000000..2f43e79e78 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +using ServiceControl.Persistence.DataMigration; + +public class MigrationCheckpointEntity +{ + public required string CategoryId { get; set; } + public MigrationCategoryState State { get; set; } + public string? Cursor { get; set; } + public long CopiedCount { get; set; } + public long SkippedCount { get; set; } + public long? SourceTotal { get; set; } + public IReadOnlyDictionary<MigrationSkipReason, long>? SkipReasons { get; set; } + public DateTime? StartedAt { get; set; } + public DateTime? LastProgressAt { get; set; } + public DateTime? SettledAt { get; set; } + public string? LastError { get; set; } + public long AlreadyPresentCount { get; set; } + public long Version { get; set; } + public long? StartedWindowSeconds { get; set; } + + internal MigrationCheckpoint ToCheckpoint() => new( + CategoryId, State, Cursor, + CopiedCount, SkippedCount, SourceTotal, SkipReasons, + StartedAt, LastProgressAt, SettledAt, LastError, + AlreadyPresentCount, Version, StartedWindowSeconds); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs new file mode 100644 index 0000000000..9cc35d77f5 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using System.Text.Json; +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using ServiceControl.Persistence.DataMigration; + +class MigrationCheckpointConfiguration : IEntityTypeConfiguration<MigrationCheckpointEntity> +{ + public void Configure(EntityTypeBuilder<MigrationCheckpointEntity> builder) + { + builder.HasKey(e => e.CategoryId); + builder.Property(e => e.CategoryId).HasMaxLength(ColumnLengths.ShortTextLength).ValueGeneratedNever(); + builder.Property(e => e.Version).IsConcurrencyToken(); + + // Null never reaches a converter: EF Core stores NULL for it, so a checkpoint with no skips has no JSON. + builder.Property(e => e.SkipReasons).HasConversion( + reasons => ToJson(reasons!), + json => FromJson(json), + new ValueComparer<IReadOnlyDictionary<MigrationSkipReason, long>>( + (left, right) => left!.Count == right!.Count && !left.Except(right).Any(), + reasons => reasons.Aggregate(0, (hash, entry) => HashCode.Combine(hash, entry.Key, entry.Value)), + reasons => reasons.ToDictionary(entry => entry.Key, entry => entry.Value))); + } + + // Keyed by name, so a stored count keeps its meaning whatever order the enum's members are in. + static string ToJson(IReadOnlyDictionary<MigrationSkipReason, long> reasons) => + JsonSerializer.Serialize(reasons.ToDictionary(entry => entry.Key.ToString(), entry => entry.Value)); + + // A name this build has no member for buckets into Unknown rather than throwing, because ReadAll is what + // decides whether the host may start; grouped because several unknown names collapse onto the one member. + static IReadOnlyDictionary<MigrationSkipReason, long> FromJson(string json) => + JsonSerializer.Deserialize<Dictionary<string, long>>(json)! + .GroupBy(entry => Enum.TryParse<MigrationSkipReason>(entry.Key, out var reason) && Enum.IsDefined(reason) ? reason : MigrationSkipReason.Unknown) + .ToDictionary(group => group.Key, group => group.Sum(entry => entry.Value)); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs new file mode 100644 index 0000000000..3f343f44e3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +// Every method opens its own scope and context through DataStoreBase, so none of them can join a +// caller's transaction. Inside one, call ServiceControlDbContext.UpsertCheckpoint instead. +public class EFMigrationCheckpointStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationCheckpointStore +{ + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entities = await dbContext.MigrationCheckpoints.AsNoTracking().ToListAsync(token); + return (IReadOnlyList<MigrationCheckpoint>)[.. entities.Select(entity => entity.ToCheckpoint())]; + }, cancellationToken); + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entity = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(e => e.CategoryId == categoryId, token); + return entity?.ToCheckpoint(); + }, cancellationToken); + + public Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => dbContext.UpsertCheckpoint(checkpoint, token), cancellationToken); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs index 08553a6686..9cf5f6e032 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/LicensingDataStore.cs @@ -362,5 +362,5 @@ static Endpoint ToEndpoint(LicensingEndpointEntity row, DateOnly? lastCollectedD LastCollectedDate = lastCollectedDate ?? default }; - static string Normalize(string name) => name.ToLowerInvariant(); + internal static string Normalize(string name) => name.ToLowerInvariant(); } diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs index 22f620a47a..6b3785277d 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/MessageBodyClassifier.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using System.Text; +using NServiceBus; using ServiceControl.Persistence.Infrastructure; /// <summary> @@ -19,7 +20,8 @@ public static (string? BodyText, bool StoreExternally) Classify(IReadOnlyDiction return (null, false); } - if (headers.IsBinary()) + // A null content type says no more than a missing one, which IsBinary treats as binary, but IsBinary throws on null. + if (headers.GetValueOrDefault(Headers.ContentType) is null || headers.IsBinary()) { return (null, true); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs new file mode 100644 index 0000000000..fefcc7ee24 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +/// <summary> +/// The migration's provider-specific SQL. Statements run inside the caller's transaction. +/// </summary> +public interface IMigrationSqlDialect +{ + /// <summary> + /// Reads, once, whatever the statements need from the schema. On SQL Server that is the collation of every + /// string column of every primary key, composite keys included. + /// </summary> + Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + /// <summary> + /// How the database compares one string key column, so a caller can work out which of its own rows the insert will treat as one. The insert compares in the database itself; this only predicts what it will find. + /// Call <see cref="Open"/> first, and name a property that is a string column of the entity's primary key. + /// </summary> + IEqualityComparer<string> KeyComparer(Type entityType, string propertyName); + + /// <summary> + /// How many rows the provider will take in one statement, for rows carrying this many values each. + /// </summary> + int RowsPerStatement(int parametersPerRow); + + /// <summary> + /// The length of a text value as the database counts it when checking a column's maximum length. + /// </summary> + int TextLength(string value); + + /// <summary> + /// Finds a character the database refuses to store in any text column, which would fail the whole statement the value goes into. + /// Returns that character, or null when the value has none. + /// </summary> + char? RejectedCharacterIn(string value); + + /// <summary> + /// Inserts the rows whose key the table does not hold, merges any whose keys the database treats as one, and returns the rows it inserted. + /// </summary> + Task<IReadOnlyList<TEntity>> InsertMissing<TEntity>(ServiceControlDbContext dbContext, IReadOnlyList<TEntity> rows, CancellationToken cancellationToken = default) where TEntity : class; + + /// <summary> + /// Adds each day's message count to the count the table holds for that day, and inserts a day the table does not hold. + /// Rows whose keys the key columns' collation treats as one are summed in the statement, so the database decides which names are one day. + /// The rows go to the database in the order given. Every row's endpoint must already be in the table, or the foreign key fails the statement. + /// Returns how many rows the statements inserted or updated. It runs on the caller's open transaction, and throws <see cref="InvalidOperationException"/> when there is none or when a returned key matches no row given. + /// </summary> + Task<int> AddLicensingEndpointThroughput(ServiceControlDbContext dbContext, IReadOnlyList<LicensingEndpointThroughputEntity> rows, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs new file mode 100644 index 0000000000..b53b885fa9 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs @@ -0,0 +1,206 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using System.Data.Common; +using System.Reflection; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; + +/// <summary> +/// The table, columns and primary key <typeparamref name="TEntity"/> maps to, and the run of one provider's insert statement over a chunk of those rows. +/// </summary> +public sealed class MigrationInsert<TEntity> where TEntity : class +{ + static readonly ExactKey KeyEquality = new(); + + readonly IReadOnlyList<IProperty> properties; + readonly IReadOnlyList<IProperty> keyProperties; + readonly IReadOnlyList<MethodInfo> keyReaders; + readonly IReadOnlyList<(IProperty Property, string TableAndColumn, int MaxLength)> boundedText; + readonly IReadOnlyList<(IProperty Property, string TableAndColumn)> text; + + MigrationInsert(string tableName, string quotedTable, IReadOnlyList<IProperty> properties, IReadOnlyList<IProperty> keyProperties, Func<IProperty, string> columnName, Func<string, string> quote) + { + Table = quotedTable; + this.properties = properties; + this.keyProperties = keyProperties; + keyReaders = [.. keyProperties.Select(KeyReader)]; + Columns = [.. properties.Select(property => quote(columnName(property)))]; + KeyColumns = [.. keyProperties.Select(property => quote(columnName(property)))]; + text = [.. properties.Where(property => property.ClrType == typeof(string)).Select(property => (property, $"{tableName}.{columnName(property)}"))]; + boundedText = + [ + .. text + .Where(column => column.Property.GetMaxLength() is not null) + .Select(column => (column.Property, column.TableAndColumn, column.Property.GetMaxLength()!.Value)) + ]; + } + + /// <summary> + /// The table name, quoted for this provider and carrying its schema. + /// </summary> + public string Table { get; } + + /// <summary> + /// Every column the insert writes, quoted, in the order the parameters of one row follow. + /// </summary> + public IReadOnlyList<string> Columns { get; } + + /// <summary> + /// The primary key columns, quoted, in the order the statement has to return them. + /// </summary> + public IReadOnlyList<string> KeyColumns { get; } + + /// <summary> + /// Reads the table, columns and key out of the EF Core model for this entity. + /// </summary> + public static MigrationInsert<TEntity> For(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not an entity in the EF Core model, so the migration cannot tell which table it lands in."); + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not mapped to a table."); + var key = entityType.FindPrimaryKey() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no primary key, so there is nothing for a row to be absent by."); + IProperty[] properties = [.. entityType.GetProperties()]; + + if (properties.Any(property => property.ValueGenerated != ValueGenerated.Never) + || entityType.GetComplexProperties().Any() + || entityType.GetNavigations().Any(navigation => navigation.TargetEntityType.IsOwned())) + { + throw new InvalidOperationException($"{typeof(TEntity).Name} has a store-generated, complex or owned property, whose columns InsertMissing does not write. Add these rows through EF in the caller's context instead."); + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var sql = dbContext.GetService<ISqlGenerationHelper>(); + + return new MigrationInsert<TEntity>( + tableName, + sql.DelimitIdentifier(tableName, entityType.GetSchema()), + properties, + key.Properties, + property => property.GetColumnName(storeObject) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name}.{property.Name} has no column in {tableName}."), + sql.DelimitIdentifier); + } + + /// <summary> + /// Finds the first text value in a row that is longer than its column allows, which would fail the whole statement the row goes into. + /// Returns a message naming the column and both lengths, or null when every value fits. + /// </summary> + public string? ValueTooLongIn(TEntity row, IMigrationSqlDialect dialect) + { + foreach (var (property, tableAndColumn, maxLength) in boundedText) + { + // No provider counts a string as longer than its UTF-16 length, so a value within that length fits and skips the dialect's own count. + if (property.GetGetter().GetClrValue(row) is string { Length: var units } value && units > maxLength && dialect.TextLength(value) is var length && length > maxLength) + { + return $"{tableAndColumn} takes at most {maxLength} characters and the value holds {length}"; + } + } + + return null; + } + + /// <summary> + /// Finds the first text value in a row holding a character the provider refuses to store, which would fail the whole statement the row goes into. + /// Returns a message naming the column and the character, or null when there is none. + /// </summary> + public string? RejectedCharacterIn(TEntity row, IMigrationSqlDialect dialect) + { + foreach (var (property, tableAndColumn) in text) + { + if (property.GetGetter().GetClrValue(row) is string value && dialect.RejectedCharacterIn(value) is { } rejected) + { + return $"{tableAndColumn} cannot hold the character U+{(int)rejected:X4}, which the value contains"; + } + } + + return null; + } + + /// <summary> + /// Runs one insert statement over a chunk of rows and returns the rows of that chunk it inserted. + /// The statement reads parameters named @p0 upward, row after row in <see cref="Columns"/> order, and must return the <see cref="KeyColumns"/> of each row it inserted, in that order. + /// It runs on the caller's open transaction, and throws when there is none or when a returned key matches no row of the chunk. + /// Each parameter is built from EF Core's type mapping and then passed, with its property, to <paramref name="configureParameter"/> when one is given, so a provider can change how it is declared. + /// </summary> + public async Task<IReadOnlyList<TEntity>> Execute(ServiceControlDbContext dbContext, string sql, TEntity[] chunk, Action<IProperty, DbParameter>? configureParameter, CancellationToken cancellationToken = default) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("InsertMissing must run inside the caller's transaction, so its rows commit with the checkpoint.")).GetDbTransaction(); + command.CommandText = sql; + // A raw command starts at the provider's own 30 seconds, where SaveChangesAsync took the configured value. + command.CommandTimeout = dbContext.Database.GetCommandTimeout() ?? command.CommandTimeout; + + var index = 0; + foreach (var row in chunk) + { + foreach (var property in properties) + { + // The mapping applies the value converter and the provider type, as EF's own inserts do. + var parameter = property.GetRelationalTypeMapping().CreateParameter(command, $"@p{index++}", property.GetGetter().GetClrValue(row), property.IsNullable); + configureParameter?.Invoke(property, parameter); + command.Parameters.Add(parameter); + } + } + + var rowsByKey = new Dictionary<object?[], TEntity>(KeyEquality); + foreach (var row in chunk) + { + // A key repeated exactly is one row to every database, so the first row given carries it. + rowsByKey.TryAdd(KeyOf(row), row); + } + + var inserted = new List<TEntity>(chunk.Length); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + object?[] key = [.. keyProperties.Select((property, column) => FromProvider(property, keyReaders[column].Invoke(reader, BindingFlags.DoNotWrapExceptions, null, [column], null)!))]; + + // A returned key that no row of the chunk holds exactly would be a bare KeyNotFoundException with a plain lookup, so it is refused by name. + inserted.Add(rowsByKey.TryGetValue(key, out var row) + ? row + : throw new InvalidOperationException($"The {Table} insert returned the key [{string.Join(", ", key.Select(Describe))}], which matches no row in the batch that produced it. Compare it against the key types the batch carried: [{string.Join(", ", keyProperties.Select(property => property.ClrType.Name))}].")); + } + + return inserted; + } + + object?[] KeyOf(TEntity row) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(row))]; + + static string Describe(object? value) => value is null ? "null" : $"{value} ({value.GetType().Name})"; + + // SQL Server's GetValue returns a date column as DateTime, which no DateOnly key equals, so each key is read as the type its mapping stores. + static MethodInfo KeyReader(IProperty property) + { + var mapping = property.GetRelationalTypeMapping(); + + return typeof(DbDataReader).GetMethod(nameof(DbDataReader.GetFieldValue))!.MakeGenericMethod(mapping.Converter?.ProviderClrType ?? mapping.ClrType); + } + + static object? FromProvider(IProperty property, object value) => + property.GetRelationalTypeMapping().Converter is { } converter ? converter.ConvertFromProvider(value) : value; + + // Exact, because the database decided which keys were one before the statement ran, and OUTPUT and RETURNING echo the value inserted. + sealed class ExactKey : IEqualityComparer<object?[]> + { + public bool Equals(object?[]? x, object?[]? y) => x!.SequenceEqual(y!); + + public int GetHashCode(object?[] key) + { + var hash = new HashCode(); + + foreach (var value in key) + { + hash.Add(value); + } + + return hash.ToHashCode(); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs new file mode 100644 index 0000000000..5ebc22df43 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationSqlDialectExtensions.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; + +static class MigrationSqlDialectExtensions +{ + /// <summary> + /// How many <typeparamref name="TEntity"/> rows fit in one of <see cref="IMigrationSqlDialect.InsertMissing{TEntity}"/>'s statements, sized from the same column list the statement chunks by. + /// </summary> + public static int RowsPerStatementFor<TEntity>(this IMigrationSqlDialect dialect, ServiceControlDbContext dbContext) where TEntity : class => + dialect.RowsPerStatement(MigrationInsert<TEntity>.For(dbContext).Columns.Count); +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index e1b11c0057..4e958292ad 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -11,4 +11,7 @@ static class SettingKeys public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string LicensedEndpointDetailsAmendment = "LicensedEndpointDetailsAmendment"; public const string NotificationEmails = "NotificationEmails"; + // Written the first time a host starts on a database a migration has already written to. Once it is set, + // going back to RavenDB loses everything ServiceControl has written here since. + public const string MigrationHostOpenedOnTarget = "Migration/HostOpenedOnTarget"; } diff --git a/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs new file mode 100644 index 0000000000..db12c9b895 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.EFCore; + +using DbContexts; +using Entities; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +public static class MigrationCheckpointExtensions +{ + /// <summary> + /// Saves a migration checkpoint on the caller's own context, so it commits with that context's transaction + /// and flushes whatever else that context is tracking, and returns it carrying the version the save landed on. + /// Throws <see cref="MigrationCheckpointConflictException"/> when the stored row no longer holds + /// <see cref="MigrationCheckpoint.Version"/>. + /// </summary> + public static async Task<MigrationCheckpoint> UpsertCheckpoint(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + try + { + return await dbContext.UpsertCheckpointCore(checkpoint, cancellationToken); + } + catch (DbUpdateConcurrencyException e) + { + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, which the stored row no longer holds. Another writer advanced it, or a retried save is meeting its own committed write after a lost acknowledgement.", e); + } + } + + // Deliberately not UpsertAsync: its duplicate-key fallback reloads and saves again, which inside a + // caller-opened PostgreSQL transaction runs after the failed insert aborted it and fails with 25P02. + static async Task<MigrationCheckpoint> UpsertCheckpointCore(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken) + { + var entity = await dbContext.FindAsync<MigrationCheckpointEntity>([checkpoint.CategoryId], cancellationToken: cancellationToken); + + if (entity is null) + { + if (checkpoint.Version != 0) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but no row exists for it."); + } + + var inserted = Insert(checkpoint); + dbContext.Add(inserted); + + try + { + await dbContext.SaveChangesAsync(cancellationToken); + } + catch (DbUpdateException e) when (dbContext.IsDuplicateKeyException(e)) + { + // Another writer created this category's first checkpoint between the read and the insert. Not + // retried: PostgreSQL has already aborted the caller's transaction, so a retry cannot run. + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved as a first write, but another writer had already created it.", e); + } + + return inserted.ToCheckpoint(); + } + + // The expected version goes in the UPDATE's WHERE clause, so a save from an out-of-date copy matches no row. + dbContext.Entry(entity).Property(e => e.Version).OriginalValue = checkpoint.Version; + entity.Version = checkpoint.Version + 1; + Apply(checkpoint, entity); + + await dbContext.SaveChangesAsync(cancellationToken); + + return entity.ToCheckpoint(); + } + + static MigrationCheckpointEntity Insert(MigrationCheckpoint checkpoint) + { + var entity = new MigrationCheckpointEntity { CategoryId = checkpoint.CategoryId, Version = 1 }; + + Apply(checkpoint, entity); + + return entity; + } + + // One place both branches copy the row through, so a column added to the checkpoint cannot be written on + // an update and forgotten on an insert. + static void Apply(MigrationCheckpoint checkpoint, MigrationCheckpointEntity entity) + { + entity.State = checkpoint.State; + entity.Cursor = checkpoint.Cursor; + entity.CopiedCount = checkpoint.CopiedCount; + entity.SkippedCount = checkpoint.SkippedCount; + entity.SourceTotal = checkpoint.SourceTotal; + entity.SkipReasons = checkpoint.SkipReasons; + entity.StartedAt = checkpoint.StartedAt; + entity.LastProgressAt = checkpoint.LastProgressAt; + entity.SettledAt = checkpoint.SettledAt; + entity.LastError = checkpoint.LastError; + entity.AlreadyPresentCount = checkpoint.AlreadyPresentCount; + entity.StartedWindowSeconds = checkpoint.StartedWindowSeconds; + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs new file mode 100644 index 0000000000..3f32f8227f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/FailedMessageStream.cs @@ -0,0 +1,138 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client; +using Raven.Client.Documents.Commands; +using ServiceControl.MessageFailures; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Turns the failed message documents RavenDB streams into migration rows. +/// </summary> +class FailedMessageStream(RavenReadOnlySourceLifecycle lifecycle) +{ + /// <summary> + /// Builds the function that turns each streamed failed message into a row. The function returns null for a message whose + /// status is not in <paramref name="statuses" />, and throws <see cref="InvalidOperationException" /> for a status that neither + /// failed message category copies. + /// </summary> + public Func<StreamResult<FailedMessage>, MigrationRow?> Project(IReadOnlyCollection<FailedMessageStatus> statuses) => + result => + { + var status = result.Document.Status; + + if (!MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Contains(status) && !MigrationCategoryRegistry.ArchivedAndResolvedStatuses.Contains(status)) + { + throw new InvalidOperationException($"The failed message document '{result.Id}' has the status value {(int)status}, which neither failed message category copies, so it would be left behind without a skip to show for it."); + } + + if (!statuses.Contains(status)) + { + return null; + } + + RestoreMetadataTypes(result.Document); + + return new MigrationRow( + result.Id, + result.Document, + new Dictionary<string, object?> + { + [MigrationMetadataKeys.StatusChangedAt] = StatusChangedAt(result.Document, ReadMetadataDate(result, Constants.Documents.Metadata.Expires), lifecycle.Settings.ErrorRetentionPeriod), + [MigrationMetadataKeys.LastModified] = ReadMetadataDate(result, Constants.Documents.Metadata.LastModified) + }); + }; + + /// <summary> + /// Returns the batch with every retry-issued row carrying <see cref="MigrationMetadataKeys.RetryClaimHeld" />, true when + /// RavenDB still holds a retry claim for the message. It looks up every claim in the batch with one request, and returns a + /// batch with no retry-issued row as it is. + /// </summary> + public async Task<MigrationBatch> ResolveRetryClaims(MigrationBatch batch, CancellationToken cancellationToken = default) + { + string[] claimIds = [.. batch.Rows.Where(IsRetryIssued).Select(ClaimId)]; + + if (claimIds.Length == 0) + { + return batch; + } + + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + var claims = await session.LoadAsync<FailedMessageRetry>(claimIds.Distinct(StringComparer.OrdinalIgnoreCase), cancellationToken); + var held = claims.Where(claim => claim.Value is not null).Select(claim => claim.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); + + return batch with + { + Rows = [.. batch.Rows.Select(row => IsRetryIssued(row) + ? row with { Metadata = new Dictionary<string, object?>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = held.Contains(ClaimId(row)) } } + : row)] + }; + } + + static bool IsRetryIssued(MigrationRow row) => ((FailedMessage)row.Document).Status == FailedMessageStatus.RetryIssued; + + static string ClaimId(MigrationRow row) => RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(((FailedMessage)row.Document).UniqueMessageId); + + // The translator checks TimeSent, SendingEndpoint and ReceivingEndpoint by exact type, but RavenDB can hand them back as text or untyped objects. + // Only the newest attempt is fixed because the translator reads no other, and a TimeSent that is not a date is left as it is for the translator to skip. + internal static void RestoreMetadataTypes(FailedMessage document) + { + if (document.ProcessingAttempts is not { Count: > 0 } || document.ProcessingAttempts[^1] is not { MessageMetadata: { } metadata } newest) + { + return; + } + + if (metadata.GetValueOrDefault("TimeSent") is string timeSent + && DateTime.TryParse(timeSent, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + metadata["TimeSent"] = parsed; + } + + if (newest.Headers is not { } headers) + { + return; + } + + RestoreEndpoint(metadata, "SendingEndpoint", () => MigrationEndpointDetailsParser.SendingEndpoint(headers)); + RestoreEndpoint(metadata, "ReceivingEndpoint", () => MigrationEndpointDetailsParser.ReceivingEndpoint(headers)); + } + + static void RestoreEndpoint(Dictionary<string, object> metadata, string key, Func<EndpointDetails?> parseHeaders) + { + if (metadata.GetValueOrDefault(key) is null or EndpointDetails) + { + return; + } + + // Storing null is deliberate, because the translator reads it as no endpoint, the same as when ingestion stored none. + metadata[key] = parseHeaders()!; + } + + // For an archived or resolved message RavenDB stores only when it expires, so the time its status + // changed is that expiry less the ErrorRetentionPeriod the source set it with. + internal static DateTime? StatusChangedAt(FailedMessage document, DateTime? expires, TimeSpan sourceErrorRetentionPeriod) + { + ArgumentOutOfRangeException.ThrowIfLessThan(sourceErrorRetentionPeriod, TimeSpan.Zero, "ErrorRetentionPeriod"); + + return document.Status is FailedMessageStatus.Resolved or FailedMessageStatus.Archived && expires is { } deadline + ? AsUtc(deadline) - sourceErrorRetentionPeriod + : document.ProcessingAttempts?.Where(attempt => attempt is not null).MaxBy(attempt => attempt.AttemptedAt) is { } newest ? AsUtc(newest.AttemptedAt) : null; + } + + // RavenDB stores times in UTC, so a time read back with no kind is already UTC and must not be shifted by the local time zone. + static DateTime AsUtc(DateTime value) => + value.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(value, DateTimeKind.Utc) : value.ToUniversalTime(); + + internal static DateTime? ReadMetadataDate<TDocument>(StreamResult<TDocument> result, string key) => + result.Metadata.TryGetValue(key, out var value) && value != null + ? AsUtc(DateTime.Parse(value.ToString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)) + : null; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs new file mode 100644 index 0000000000..8c05a27f97 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs @@ -0,0 +1,25 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Which documents one category is made of, and how they become rows. Read keeps the contract +/// <see cref="IMigrationSource.Read"/> sets out, for this one category. +/// </summary> +interface IMigrationCategoryReader +{ + /// <summary>The category this reader handles, named as in <see cref="MigrationCategoryIds" />.</summary> + string CategoryId { get; } + + /// <summary> + /// The type of <see cref="MigrationRow.Document" /> in every row <see cref="Read" /> yields. + /// </summary> + Type DocumentType { get; } + + IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs new file mode 100644 index 0000000000..bd26b24b74 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs @@ -0,0 +1,50 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using Raven.Client.Documents.Commands; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// A reader whose every row carries a <typeparamref name="TDocument" />, which is the type its category's writer +/// must take. Reading through <see cref="WholeDocuments" /> ties the documents RavenDB streams to that type. +/// </summary> +abstract class MigrationCategoryReader<TDocument>(RavenReadOnlySourceLifecycle lifecycle) : IMigrationCategoryReader where TDocument : class +{ + /// <summary> + /// The source connection, for a reader that streams or opens sessions itself instead of using <see cref="WholeDocuments" />. + /// </summary> + protected RavenReadOnlySourceLifecycle Lifecycle { get; } = lifecycle; + + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); + + /// <summary> + /// Streams the primary database's documents whose id starts with <paramref name="prefix" />, each one whole as a row. + /// </summary> + protected IAsyncEnumerable<MigrationBatch> WholeDocuments(string prefix, string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + Documents(prefix, resumeAfter, batchSize, RavenDocumentStream.WholeDocument, cancellationToken); + + /// <summary> + /// Streams the primary database's documents whose id starts with <paramref name="prefix" />, in id order, through + /// <see cref="RavenDocumentStream.ByPrefix{TDocument}" /> with this category's id and the walk limit every reader shares, + /// <see cref="RavenDocumentStream.MaxDocumentsWalkedPerBatch" />. + /// </summary> + protected IAsyncEnumerable<MigrationBatch> Documents(string prefix, string? resumeAfter, int batchSize, Func<StreamResult<TDocument>, MigrationRow?> project, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix( + Lifecycle, + CategoryId, + Lifecycle.Settings.DatabaseName, + prefix, + resumeAfter, + batchSize, + RavenDocumentStream.MaxDocumentsWalkedPerBatch, + project, + cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs new file mode 100644 index 0000000000..37a67848cb --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationEndpointDetailsParser.cs @@ -0,0 +1,95 @@ +#nullable enable +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using NServiceBus; +using NServiceBus.Faults; +using ServiceControl.Operations; +using ServiceControl.Persistence.Infrastructure; + +/// <summary> +/// Rebuilds the sending and receiving endpoint of a failed message from its headers exactly as ingestion would have stored them, +/// so a message ingested with an odd header copies with the same endpoint details it would have had. +/// </summary> +static class MigrationEndpointDetailsParser +{ + public static EndpointDetails? SendingEndpoint(IReadOnlyDictionary<string, string> headers) + { + var endpoint = new EndpointDetails + { + Name = ValueOf(headers, Headers.OriginatingEndpoint) ?? string.Empty, + Host = ValueOf(headers, Headers.OriginatingMachine) ?? string.Empty, + HostId = Guid.TryParse(ValueOf(headers, Headers.OriginatingHostId), out var hostId) ? hostId : Guid.Empty + }; + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + if (ValueOf(headers, Headers.OriginatingAddress) is not { } address) + { + return null; + } + + var (queue, machine) = ExtractQueueAndMachineName(address); + endpoint.Name = queue; + endpoint.Host = machine!; + return endpoint; + } + + public static EndpointDetails? ReceivingEndpoint(IReadOnlyDictionary<string, string> headers) + { + var endpoint = new EndpointDetails + { + Name = ValueOf(headers, Headers.ProcessingEndpoint) ?? string.Empty, + Host = ValueOf(headers, Headers.HostDisplayName) ?? ValueOf(headers, Headers.ProcessingMachine) ?? string.Empty, + HostId = Guid.TryParse(ValueOf(headers, Headers.HostId), out var hostId) ? hostId : Guid.Empty + }; + + if (!string.IsNullOrEmpty(endpoint.Name) && !string.IsNullOrEmpty(endpoint.Host)) + { + return endpoint; + } + + if (ValueOf(headers, FaultsHeaderKeys.FailedQ) is not { } address) + { + return null; + } + + var (queue, machine) = ExtractQueueAndMachineName(address); + + if (string.IsNullOrEmpty(endpoint.Name)) + { + endpoint.Name = queue; + } + + if (string.IsNullOrEmpty(endpoint.Host)) + { + endpoint.Host = machine ?? "unknown"; + } + + if (string.IsNullOrEmpty(endpoint.Name)) + { + return null; + } + + if (endpoint.HostId == Guid.Empty) + { + endpoint.HostId = DeterministicGuid.MakeId(endpoint.Name, endpoint.Host); + } + + return endpoint; + } + + static string? ValueOf(IReadOnlyDictionary<string, string> headers, string key) => + headers.TryGetValue(key, out var value) ? value : null; + + static (string Queue, string? Machine) ExtractQueueAndMachineName(string address) + { + var atIndex = address.IndexOf('@', StringComparison.InvariantCulture); + + return atIndex > -1 ? (address[..atIndex], address[(atIndex + 1)..]) : (address, null); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs new file mode 100644 index 0000000000..cfa5726b66 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs @@ -0,0 +1,127 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Session; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads RavenDB documents for a migration category in id order and hands them back in batches, most often every +/// document whose id starts with a prefix. Id order is what lets a copy resume, because a resume asks RavenDB to +/// start after the last id it copied rather than to skip a count. +/// </summary> +static class RavenDocumentStream +{ + /// <summary> + /// The most documents one batch reads before it is handed back, even if it holds few rows or none. Every batch is + /// one commit, so a long run of documents that make no row costs one commit per this many documents rather than one + /// per batch size, while the cursor is still saved every few seconds and a resume reads at most this many documents again. + /// </summary> + public const int MaxDocumentsWalkedPerBatch = 10_000; + + /// <summary> + /// Streams the documents whose id starts with <paramref name="prefix" />, after <paramref name="resumeAfter" />, and yields a batch as soon as it holds <paramref name="batchSize" /> rows, or once it has read <paramref name="maxDocumentsWalkedPerBatch" /> documents, even if it holds few rows or none. A batch never holds more than <paramref name="batchSize" /> rows, but when <paramref name="project" /> returns null for some documents, one batch can cover more documents than that. If any documents were read after the last such batch, it yields one more batch for them, even an empty one, so the cursor past them is saved. When <paramref name="project" /> never returns null and <paramref name="batchSize" /> is no more than <paramref name="maxDocumentsWalkedPerBatch" />, every batch but the last holds exactly <paramref name="batchSize" /> rows. Throws <see cref="InvalidOperationException" /> when the source holds no document with the id in <paramref name="resumeAfter" />. + /// </summary> + public static async IAsyncEnumerable<MigrationBatch> ByPrefix<TDocument>( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string databaseName, + string prefix, + string? resumeAfter, + int batchSize, + int maxDocumentsWalkedPerBatch, + Func<StreamResult<TDocument>, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken = default) where TDocument : class + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxDocumentsWalkedPerBatch); + + using var session = lifecycle.OpenSession(databaseName); + + // RavenDB starts a stream after whatever id it is given, even one that does not exist, so a stale cursor would skip rows nothing has copied. + await RefuseMissingCursor(session, categoryId, databaseName, resumeAfter, cancellationToken); + + await using var enumerator = await session.Advanced.StreamAsync<TDocument>( + prefix, startAfter: resumeAfter, token: cancellationToken); + + await foreach (var batch in Batches(enumerator, resumeAfter, batchSize, maxDocumentsWalkedPerBatch, project)) + { + yield return batch; + } + } + + /// <summary> + /// Checks that the cursor names a document the source still holds. The cursor comes from the target's checkpoint, so + /// if the source has no document with that id, either the copy is now reading a different database or the document was + /// deleted since, and resuming after it could skip rows that were never copied. Throws + /// <see cref="InvalidOperationException" /> when <paramref name="resumeAfter" /> is not null and the source holds no document with that id. + /// </summary> + public static async Task RefuseMissingCursor(IAsyncDocumentSession session, string categoryId, string databaseName, string? resumeAfter, CancellationToken cancellationToken = default) + { + if (resumeAfter is not null && !await session.Advanced.ExistsAsync(resumeAfter, cancellationToken)) + { + throw new InvalidOperationException( + $"The migration cannot resume the '{categoryId}' category after '{resumeAfter}', because the RavenDB database '{databaseName}' no longer holds a document with that id. The likeliest cause is that the document expired and RavenDB deleted it while the copy was interrupted, which can happen to any archived or resolved message. Run --migration-retry {categoryId} to copy the category again from the start. If RavenDB was instead restored from a backup or the migration now reads a different database, point it back at the database this copy started from and restart."); + } + } + + /// <summary> + /// Groups the documents from a RavenDB stream into batches, by the rules <see cref="ByPrefix{TDocument}" /> describes: + /// a batch goes out once it holds <paramref name="batchSize" /> rows or has read <paramref name="maxDocumentsWalkedPerBatch" /> + /// documents, and one last batch goes out for any documents read after that, even if it holds no rows. The stream must return + /// documents in id order, or a resume would start in the wrong place. <paramref name="resumeAfter" /> is the id the stream + /// started after, or null. <paramref name="project" /> turns a document into a row, or returns null for a document the category + /// does not copy, and the cursor still moves past that document. Each batch's cursor is the id of the last document it read. + /// An empty stream yields nothing. + /// </summary> + public static async IAsyncEnumerable<MigrationBatch> Batches<TDocument>( + IAsyncEnumerator<StreamResult<TDocument>> enumerator, + string? resumeAfter, + int batchSize, + int maxDocumentsWalkedPerBatch, + Func<StreamResult<TDocument>, MigrationRow?> project) + { + var rows = new List<MigrationRow>(batchSize); + var cursor = resumeAfter; + var lastYielded = resumeAfter; + var walkedSinceYield = 0; + + while (await enumerator.MoveNextAsync()) + { + // The cursor moves on every document the stream hands back, even one that becomes no row, so a resume never walks it again. + cursor = enumerator.Current.Id; + walkedSinceYield++; + + if (project(enumerator.Current) is { } row) + { + rows.Add(row); + } + + // Handing back a batch at the document cap saves progress through a long run of documents that make no row. + if (rows.Count == batchSize || walkedSinceYield == maxDocumentsWalkedPerBatch) + { + yield return new MigrationBatch(rows, cursor); + rows = new List<MigrationRow>(batchSize); + lastYielded = cursor; + walkedSinceYield = 0; + } + } + + // A tail whose documents all became no row still goes out, empty, so that the cursor past them is saved. + if (rows.Count > 0 || cursor != lastYielded) + { + yield return new MigrationBatch(rows, cursor!); + } + } + + public static MigrationRow WholeDocument<TDocument>(StreamResult<TDocument> result) where TDocument : class => + new(result.Id, result.Document, NoMetadata); + + static readonly IReadOnlyDictionary<string, object?> NoMetadata = ReadOnlyDictionary<string, object?>.Empty; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs new file mode 100644 index 0000000000..a77fe39612 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -0,0 +1,136 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Operations; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Operations.BodyStorage.RavenAttachments; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +/// <summary> +/// Reads a migration out of RavenDB. It holds one reader per category and knows nothing about any of them +/// beyond that, so a category this build cannot read is simply absent from <see cref="SupportedCategoryIds" /> +/// and never reaches the engine. +/// </summary> +sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMigrationSource +{ + readonly FrozenDictionary<string, IMigrationCategoryReader> readers = new IMigrationCategoryReader[] + { + new KnownEndpointsReader(lifecycle), + new EndpointSettingsReader(lifecycle), + new FailedMessagesReader(lifecycle), + new LicensingEndpointsReader(lifecycle), + new LicensingThroughputReader(lifecycle), + new LicensingReportMasksReader(lifecycle), + new LicensedEndpointDetailsReader(lifecycle), + new MessageRedirectsReader(lifecycle), + new SubscriptionsReader(lifecycle), + new NotificationSettingsReader(lifecycle), + new TrialEndDateReader(lifecycle), + new RetryOperationsReader(lifecycle), + new PendingIntegrationEventsReader(lifecycle), + new CustomChecksReader(lifecycle), + new FailedErrorImportsReader(lifecycle), + new GroupCommentsReader(lifecycle) + }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); + + public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); + + public IReadOnlyList<IMigrationStartupCheck> ContributedChecks() => []; + + public async Task<MigrationSourceDescription> Describe(CancellationToken cancellationToken = default) + { + var settings = lifecycle.Settings; + var build = await lifecycle.DocumentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + var server = settings.UseEmbeddedServer + ? new MigrationSourceFact("Server", settings.ServerUrl, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseMaintenancePortKey}") + : new MigrationSourceFact("Server", settings.ConnectionString, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.ConnectionStringKey}"); + + return new MigrationSourceDescription(build.ProductVersion, + [ + new MigrationSourceFact("Mode", settings.UseEmbeddedServer ? "embedded" : "external"), + server, + new MigrationSourceFact("Primary database", settings.DatabaseName, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseNameKey}"), + new MigrationSourceFact("Throughput database", settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}") + ]); + } + + public async Task<IReadOnlyList<MigrationSourceInventoryEntry>> Inventory(CancellationToken cancellationToken = default) + { + var entries = new List<MigrationSourceInventoryEntry>(); + + foreach (var databaseName in new[] { lifecycle.Settings.DatabaseName, lifecycle.Settings.ThroughputDatabaseName }) + { + var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + entries.AddRange(statistics.Collections.Select(collection => new MigrationSourceInventoryEntry(databaseName, collection.Key, collection.Value))); + } + + return entries; + } + + // Counted through the reader the copy uses, not from RavenDB's collection statistics, because a collection does not map one to one onto a category. + public async Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) + { + var total = 0L; + + await foreach (var batch in Read(category, resumeAfter: null, batchSize: CountBatchSize, cancellationToken)) + { + total += batch.Rows.Count; + } + + return total; + } + + public IAsyncEnumerable<MigrationBatch> Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default) => + readers.TryGetValue(category.Id, out var reader) + ? reader.Read(resumeAfter, batchSize, cancellationToken) + : throw new NotSupportedException($"The migration source cannot yet read the '{category.Id}' category."); + + public async Task<MigrationBody?> ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + // Another category may keep its body inside the document, where a body attachment read would find nothing and skip every row as BodyMissing. + if (category.Id != MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) + { + throw new NotSupportedException($"The migration source does not read bodies as attachments for the '{category.Id}' category."); + } + + using var session = lifecycle.OpenSession(lifecycle.Settings.DatabaseName); + + var attachment = await session.Advanced.Attachments.GetAsync(sourceId, RavenAttachmentsBodyStorage.AttachmentName, cancellationToken); + + if (attachment == null) + { + return null; + } + + await using var stream = attachment.Stream; + + var content = new MemoryStream((int)attachment.Details.Size); + await stream.CopyToAsync(content, cancellationToken); + + return new MigrationBody(content.ToArray(), attachment.Details.ContentType); + } + + public IReadOnlyCollection<string> SupportedCategoryIds => readers.Keys; + + public IReadOnlyDictionary<string, Type> DocumentTypes => readers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); + + // Counting only adds up row counts, so this size changes nothing but how often the stream stops to hand one back. + const int CountBatchSize = 1024; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs new file mode 100644 index 0000000000..2500437e43 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -0,0 +1,267 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Diagnostics; +using System.Net.Http; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Newtonsoft.Json; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions.Database; +using Raven.Client.Exceptions.Security; +using Raven.Client.Json.Serialization.NewtonsoftJson; +using ServiceControl.Configuration; +using ServiceControl.Contracts.Operations; +using ServiceControl.RavenDB; + +/// <summary> +/// The connection to the RavenDB database a migration reads from, and the embedded server it starts when that +/// database is a data directory rather than a URL. Nothing here writes, so the old database stays exactly as it +/// was and the copy can still be thrown away. +/// </summary> +sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings, SettingsRootNamespace settingsRoot) : IAsyncDisposable +{ + public RavenPersisterSettings Settings => settings; + + public SettingsRootNamespace SettingsRoot => settingsRoot; + + public IDocumentStore DocumentStore => documentStore ?? throw new InvalidOperationException($"The migration source is not open. Call {nameof(Open)} first."); + + public async Task Open(CancellationToken cancellationToken = default) + { + if (documentStore is not null) + { + throw new InvalidOperationException("The migration source is already open. Opening it twice would abandon the first server without stopping it."); + } + + try + { + var serverUrl = settings.UseEmbeddedServer ? await StartEmbedded(cancellationToken) : settings.ConnectionString; + documentStore = Connect(serverUrl); + + if (!settings.UseEmbeddedServer) + { + // Only an external server can be older than the client; an embedded one ships beside it. + await StartupChecks.EnsureServerVersion(documentStore, cancellationToken); + } + + await EnsureReadable(settings.DatabaseName, $"{settingsRoot}/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); + await EnsureReadable(settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}", cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await DisposeAsync(); + throw; + } + catch (Exception) + { + await DisposeAsync(); + throw; + } + } + + public IAsyncDocumentSession OpenSession(string databaseName) => + DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName, NoTracking = true }); + + async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) + { + var elapsed = Stopwatch.StartNew(); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + return; + } + catch (DatabaseLoadTimeoutException e) when (settings.UseEmbeddedServer && elapsed.Elapsed >= EmbeddedLoadBudget) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but it did not finish loading within {EmbeddedLoadBudget.TotalMinutes:N0} minutes. A data directory held by another process, or one that is corrupt, is the usual cause.", e); + } + catch (DatabaseLoadTimeoutException) when (settings.UseEmbeddedServer) + { + await Task.Delay(EmbeddedLoadRetryDelay, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (DatabaseDoesNotExistException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is.", e); + } + catch (AuthorizationException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} refused its client certificate access to the database '{databaseName}'. Grant that certificate Read access to '{databaseName}', or supply one that has it in '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}'. If '{databaseName}' is the wrong name, correct the '{settingKey}' setting instead: RavenDB refuses a certificate that has no access to a database whether or not that database exists.", e); + } + catch (Exception e) when (e is not DatabaseLoadTimeoutException) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it: {e.Message}", e); + } + } + } + + string Located() => Located(settings, settingsRoot); + + internal static string Located(RavenPersisterSettings sourceSettings, SettingsRootNamespace root) => sourceSettings.UseEmbeddedServer + ? $"{sourceSettings.ServerUrl} (embedded, data directory '{sourceSettings.DatabasePath}', from '{root}/{RavenBootstrapper.DatabasePathKey}')" + : sourceSettings.ConnectionString; + + async Task<string> StartEmbedded(CancellationToken cancellationToken) + { + // A dynamic query is a POST to /queries, which the read-only guard allows, and it builds an auto-index on the + // customer's database. Nothing on the client can tell those queries apart, so the server is told to refuse them. + var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode) { DisableAutoIndexes = true }; + + embedded = EmbeddedDatabase.Start(configuration, lifetime); + + try + { + return await embedded.WaitUntilReady(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + throw new InvalidOperationException($"The RavenDB migration source could not start a server for the embedded database at {Located()}: {e.Message} A ServiceControl instance still holding that data directory is one cause, and stopping it lets the report run; a port already in use or a missing RavenDB server are others, which the message above tells apart.", e); + } + } + + IDocumentStore Connect(string serverUrl) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = [serverUrl], + Conventions = new DocumentConventions + { + SaveEnumsAsIntegers = true, + Serialization = new NewtonsoftJsonSerializationConventions { CustomizeJsonDeserializer = serializer => serializer.Converters.Add(new FailureDetailsConverter()) } + } + }; + + if (!settings.UseEmbeddedServer) + { + store.Certificate = RavenClientCertificate.FindClientCertificate(settings); + RefuseUnusableCertificate(store.Certificate); + } + + store.OnBeforeRequest += RefuseWrite; + + return store.Initialize(); + } + + // An unusable certificate otherwise surfaces on the first request as a refused connection, which says nothing about the cause. + void RefuseUnusableCertificate(X509Certificate2? certificate) + { + if (certificate is null) + { + if (settings.ConnectionString.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"The RavenDB migration source at '{settings.ConnectionString}' is secured but no client certificate is configured. Set '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}'."); + } + + return; + } + + // X509Certificate2 reports both bounds in local time, so they move to UTC before the comparison. + var notBefore = certificate.NotBefore.ToUniversalTime(); + var notAfter = certificate.NotAfter.ToUniversalTime(); + var now = DateTime.UtcNow; + + if (now < notBefore || now > notAfter) + { + throw new InvalidOperationException($"The RavenDB client certificate '{certificate.Subject}' is valid from {notBefore:u} to {notAfter:u}, which does not include now."); + } + } + + static void RefuseWrite(object? sender, BeforeRequestEventArgs e) + { + if (IsRead(e.Request.Method, new Uri(e.Url).AbsolutePath)) + { + return; + } + + throw new InvalidOperationException($"The RavenDB migration source is open read-only and refused a {e.Request.Method} to '{e.Url}'. Nothing in a migration may write to the database it is reading."); + } + + static bool IsRead(HttpMethod method, string path) + { + if (method == HttpMethod.Get || method == HttpMethod.Head) + { + // HiLo persists the id range it hands out, so it writes despite being a GET. + return !path.Contains(HiLoPathSegment, StringComparison.OrdinalIgnoreCase); + } + + return method == HttpMethod.Post && Array.Exists(ReadOnlyPostPaths, suffix => path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); + } + + public async ValueTask DisposeAsync() + { + documentStore?.Dispose(); + documentStore = null; + + if (embedded is not null) + { + // Stop force-kills only once this token cancels; EmbeddedDatabase sets the graceful wait to an hour. + using var shutdown = new CancellationTokenSource(EmbeddedShutdownTimeout); + await embedded.Stop(shutdown.Token); + embedded.Dispose(); + embedded = null; + } + } + + const string HiLoPathSegment = "/hilo/"; + static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; + static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); + static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); + // Generous because one DatabaseLoadTimeoutException already means RavenDB waited its own load timeout. + static readonly TimeSpan EmbeddedLoadBudget = TimeSpan.FromMinutes(5); + + IDocumentStore? documentStore; + EmbeddedDatabase? embedded; + readonly SourceLifetime lifetime = new(); + + sealed class SourceLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + + public void StopApplication() + { + } + } + + // The FailureDetails constructor sets TimeOfFailure to now. Starting from the default instead lets the translator see a stored attempt that has no time. + sealed class FailureDetailsConverter : JsonConverter<FailureDetails> + { + public override FailureDetails? ReadJson(JsonReader reader, Type objectType, FailureDetails? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var details = new FailureDetails { AddressOfFailingEndpoint = string.Empty, TimeOfFailure = default }; + serializer.Populate(reader, details); + + return details; + } + + public override void WriteJson(JsonWriter writer, FailureDetails? value, JsonSerializer serializer) => + throw new NotSupportedException("The migration source only reads, so it never writes a FailureDetails."); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs new file mode 100644 index 0000000000..979be4d817 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/CustomChecksReader.cs @@ -0,0 +1,38 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the custom check results out of the primary database, each one turned into the shared <see cref="CustomCheckDetail" />, a type the target can reference. +/// A null value in the stored document stays null, so the writer can name the column it would fill. +/// A result of one of the RavenDB persister's own checks carries the SourceStorageCheck metadata value, so the target can skip it. +/// </summary> +sealed class CustomChecksReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<CustomCheckDetail>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.CustomChecks; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix<CustomCheck>(Lifecycle, CategoryId, Lifecycle.Settings.DatabaseName, $"{RavenCustomCheckDataStore.CollectionName}/", resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, + static result => new MigrationRow(result.Id, ToDetail(result.Document), MetadataFor(result.Document)), cancellationToken); + + static IReadOnlyDictionary<string, object?> MetadataFor(CustomCheck document) => + document.CustomCheckId is { } id && document.Category is { } category && RavenStorageCheckIds.All.Contains((id, category)) + ? new Dictionary<string, object?> { [MigrationMetadataKeys.SourceStorageCheck] = true } + : ReadOnlyDictionary<string, object?>.Empty; + + static CustomCheckDetail ToDetail(CustomCheck document) => new() + { + OriginatingEndpoint = document.OriginatingEndpoint!, + CustomCheckId = document.CustomCheckId!, + ReportedAt = document.ReportedAt, + Category = document.Category!, + HasFailed = document.Status == Status.Fail, + FailureReason = document.FailureReason + }; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs new file mode 100644 index 0000000000..087194c67c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the per-endpoint settings, whole, out of the primary database. The collection holds one document per +/// endpoint plus one with an empty name, which carries the default for every endpoint. +/// </summary> +sealed class EndpointSettingsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<EndpointSettings>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(EndpointSettingsStore.CollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs new file mode 100644 index 0000000000..c45b0e0eac --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedErrorImportsReader.cs @@ -0,0 +1,30 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using Raven.Client; +using Raven.Client.Documents.Commands; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the messages ServiceControl took off the error queue but could not store, each with its body and the time it failed. +/// </summary> +sealed class FailedErrorImportsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<FailedErrorImport>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.FailedErrorImports; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + Documents($"{FailedErrorImportDataStore.CollectionName}/", resumeAfter, batchSize, ToRow, cancellationToken); + + // A document with no message still gets an empty body, because the engine would otherwise ask ReadBody, which refuses this category. + static MigrationRow ToRow(StreamResult<FailedErrorImport> result) => new( + result.Id, + result.Document, + new Dictionary<string, object?> { [MigrationMetadataKeys.FailedAt] = FailedMessageStream.ReadMetadataDate(result, Constants.Documents.Metadata.LastModified) }, + new MigrationBody( + result.Document.Message?.Body ?? [], + result.Document.Message?.Headers?.GetValueOrDefault(NServiceBus.Headers.ContentType) ?? "application/octet-stream")); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs new file mode 100644 index 0000000000..1d32916de9 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/FailedMessagesReader.cs @@ -0,0 +1,197 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions; +using ServiceControl.Infrastructure; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the failed messages that are still waiting on somebody: unresolved, or with a retry issued. This category is +/// copied while ServiceControl is closed, and the archived and resolved messages in the same collection can outnumber it +/// many times, so it queries <see cref="FailedMessageViewIndex" /> for just these statuses instead of reading them all. +/// </summary> +sealed class FailedMessagesReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<FailedMessage>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages; + + /// <summary> + /// Waits for <see cref="FailedMessageViewIndex" /> to catch up, then streams the unresolved and retry issued messages in id + /// order, starting after <paramref name="resumeAfter" />. When the index is missing, errored or disabled, or does not hold + /// every failed message with a status some category copies, it logs a warning and reads the whole collection instead. + /// Both ways use the document id as the cursor, so a copy can switch from one to the other between runs. + /// Throws <see cref="InvalidOperationException" /> when the source holds no document with the id in <paramref name="resumeAfter" />, + /// or when a failed message has a status no category copies. + /// </summary> + public override async IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var databaseName = Lifecycle.Settings.DatabaseName; + var stream = new FailedMessageStream(Lifecycle); + var project = stream.Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses); + + var batches = await IndexHoldsEveryCopiedFailedMessage(databaseName, cancellationToken) + ? ThroughIndex(databaseName, resumeAfter, batchSize, project, cancellationToken) + : Documents($"{FailedMessageIdGenerator.CollectionName}/", resumeAfter, batchSize, project, cancellationToken); + + await foreach (var batch in batches) + { + yield return await stream.ResolveRetryClaims(batch, cancellationToken); + } + } + + async IAsyncEnumerable<MigrationBatch> ThroughIndex( + string databaseName, + string? resumeAfter, + int batchSize, + Func<StreamResult<FailedMessage>, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + await RavenDocumentStream.RefuseMissingCursor(session, CategoryId, databaseName, resumeAfter, cancellationToken); + + var query = Query(session).WhereIn(nameof(FailedMessage.Status), StatusValues(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses)); + + if (resumeAfter is not null) + { + query = query.AndAlso().WhereGreaterThan(DocumentId, resumeAfter); + } + + await using var enumerator = await session.Advanced.StreamAsync(query.OrderBy(DocumentId).OfType<FailedMessage>(), cancellationToken); + + await foreach (var batch in RavenDocumentStream.Batches(enumerator, resumeAfter, batchSize, RavenDocumentStream.MaxDocumentsWalkedPerBatch, project)) + { + yield return batch; + } + } + + // A full read passes a document RavenDB could not index to the target to skip, and refuses a status no category copies, + // but a query on the index sees neither, so either kind of document sends the copy back to a full read. + async Task<bool> IndexHoldsEveryCopiedFailedMessage(string databaseName, CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(databaseName); + + while (true) + { + var index = await WaitForIndex(databaseName, cancellationToken); + + if (index is null || index.State is IndexState.Error or IndexState.Disabled) + { + logger.LogWarning("The RavenDB index {IndexName} in {DatabaseName} is {IndexState}, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer", IndexName, databaseName, index?.State.ToString() ?? "missing", CategoryId); + return false; + } + + // Expiration keeps deleting archived and resolved messages during the copy, so the collection is counted before and + // after the index count, because a deletion in between could hide a document the index left out. A stale index count + // means something changed after the wait, so it waits and counts again. + var storedBefore = await StoredFailedMessages(databaseName, cancellationToken); + int indexed; + QueryStatistics statistics; + try + { + indexed = await Query(session).Statistics(out statistics).WhereIn(nameof(FailedMessage.Status), StatusValues(CopiedStatuses)).CountAsync(cancellationToken); + } + catch (RavenException) + { + // RavenDB can mark the index errored after the wait saw it working, and then refuses the query. The next pass falls back to a full read. + if ((await WaitForIndex(databaseName, cancellationToken))?.State is IndexState.Error or IndexState.Disabled) + { + continue; + } + + throw; + } + + var storedAfter = await StoredFailedMessages(databaseName, cancellationToken); + + if (statistics.IsStale) + { + continue; + } + + if (indexed == storedBefore && indexed == storedAfter) + { + return true; + } + + logger.LogWarning("The RavenDB database {DatabaseName} held {StoredBefore} and then {StoredAfter} failed messages while its index {IndexName} held {Indexed} with a status a category copies, so copying {CategoryId} reads every failed message instead, archived and resolved ones included, which keeps ServiceControl closed for longer. The counts differ when RavenDB could not index a document, which the index errors in RavenDB Studio name, when a document has a status no category copies, or when expiration deleted documents during the count", databaseName, storedBefore, storedAfter, IndexName, indexed, CategoryId); + return false; + } + } + + async Task<long> StoredFailedMessages(string databaseName, CancellationToken cancellationToken) => + (await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken)) + .Collections.GetValueOrDefault(FailedMessageIdGenerator.CollectionName); + + // A query on an index that is behind leaves out what it has not indexed yet, which here means unresolved messages nothing + // would copy. It waits rather than halts, because halting would need an operator to retry for a wait that is normally seconds. + async Task<IndexInformation?> WaitForIndex(string databaseName, CancellationToken cancellationToken) + { + var waiting = Stopwatch.StartNew(); + var nextLog = TimeSpan.Zero; + + while (true) + { + var statistics = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + var index = statistics.Indexes.SingleOrDefault(candidate => candidate.Name == IndexName); + + // A missing, errored or disabled index never catches up, so waiting for it would keep ServiceControl closed for good. + if (index is null || !index.IsStale || index.State is IndexState.Error or IndexState.Disabled) + { + return index; + } + + if (waiting.Elapsed >= nextLog) + { + await LogWait(databaseName, waiting.Elapsed, cancellationToken); + nextLog += LogEvery; + } + + await Task.Delay(PollEvery, cancellationToken); + } + } + + // A paused index stays stale with its state still Normal, so without this the log reads exactly like an index that is merely slow. + async Task LogWait(string databaseName, TimeSpan waited, CancellationToken cancellationToken) + { + var indexing = await Lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexingStatusOperation(), cancellationToken); + var paused = indexing.Status == IndexRunningStatus.Paused + || indexing.Indexes.Any(index => index.Name == IndexName && index.Status == IndexRunningStatus.Paused); + + if (paused) + { + logger.LogWarning("Copying {CategoryId} is waiting for the RavenDB index {IndexName} in {DatabaseName}, but indexing is paused, so ServiceControl stays closed until it is resumed in RavenDB Studio. Waited {Waited} so far", CategoryId, IndexName, databaseName, waited); + return; + } + + logger.LogInformation("Waiting for the RavenDB index {IndexName} in {DatabaseName} to catch up before copying {CategoryId}, waited {Waited} so far", IndexName, databaseName, CategoryId, waited); + } + + static IAsyncDocumentQuery<FailedMessageViewIndex.SortAndFilterOptions> Query(IAsyncDocumentSession session) => + session.Advanced.AsyncDocumentQuery<FailedMessageViewIndex.SortAndFilterOptions, FailedMessageViewIndex>(); + + static object[] StatusValues(IEnumerable<FailedMessageStatus> statuses) => statuses.Select(status => (object)(int)status).ToArray(); + + static readonly FailedMessageStatus[] CopiedStatuses = [.. MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses, .. MigrationCategoryRegistry.ArchivedAndResolvedStatuses]; + static readonly string IndexName = new FailedMessageViewIndex().IndexName; + static readonly TimeSpan PollEvery = TimeSpan.FromSeconds(1); + static readonly TimeSpan LogEvery = TimeSpan.FromSeconds(30); + static readonly ILogger<FailedMessagesReader> logger = LoggerUtil.CreateStaticLogger<FailedMessagesReader>(); + + const string DocumentId = "id()"; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs new file mode 100644 index 0000000000..1397a35a4d --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/GroupCommentsReader.cs @@ -0,0 +1,23 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; + +/// <summary> +/// Reads the comments operators wrote on failure groups. Each row carries the bare group id, because SQL keys a comment on the group id while RavenDB stores it under its own document id. +/// </summary> +sealed class GroupCommentsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<GroupComment>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.GroupComments; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + Documents(GroupsDataStore.CommentIdPrefix, resumeAfter, batchSize, + static result => new MigrationRow(result.Id, new GroupComment { Id = result.Id[GroupsDataStore.CommentIdPrefix.Length..], Comment = result.Document.Comment }, ReadOnlyDictionary<string, object?>.Empty), + cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs new file mode 100644 index 0000000000..3372368c5f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs @@ -0,0 +1,18 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the endpoints ServiceControl has heard from, whole, out of the primary database. +/// </summary> +sealed class KnownEndpointsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<KnownEndpoint>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(RavenMonitoringDataStore.KnownEndpointsCollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs new file mode 100644 index 0000000000..f5b8d3f66e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensedEndpointDetailsReader.cs @@ -0,0 +1,22 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; + +/// <summary> +/// Reads the licensed endpoint details document, which becomes one Settings row. A read gives one batch holding the +/// whole document, or no rows when the document does not exist. The batch's cursor is the document's id, and a read +/// resumed after that id gives nothing. +/// </summary> +sealed class LicensedEndpointDetailsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<LicensedEndpointDetails, LicensedEndpointDetails>(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.LicencedEndpointDetailsDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.LicensedEndpointDetails; + + protected override IEnumerable<(string SourceId, LicensedEndpointDetails Document)> RowsFrom(LicensedEndpointDetails document) => + [(DocumentId, document)]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs new file mode 100644 index 0000000000..0a0b9629e1 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingEndpointsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the licensing endpoints out of the throughput database, skipping every document outside the EndpointDocuments collection. +/// </summary> +sealed class LicensingEndpointsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<Endpoint>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.LicensingEndpoints; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ThroughputDocuments.Endpoints(Lifecycle, CategoryId, resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs new file mode 100644 index 0000000000..47597f08cd --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingReportMasksReader.cs @@ -0,0 +1,22 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +/// <summary> +/// Reads the report masks document, whose list of masked strings becomes one Settings row. A read gives one batch +/// holding the list, or no rows when the document does not exist, and a list stored as null reads as an empty one. +/// The batch's cursor is the document's id, and a read resumed after that id gives nothing. +/// </summary> +sealed class LicensingReportMasksReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<ReportConfigurationDocument, List<string>>(lifecycle, lifecycle.Settings.ThroughputDatabaseName, LicensingDataStore.ReportMasksDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.LicensingReportMasks; + + protected override IEnumerable<(string SourceId, List<string> Document)> RowsFrom(ReportConfigurationDocument document) => + [(DocumentId, document.MaskedStrings ?? [])]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs new file mode 100644 index 0000000000..a77b53067c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/LicensingThroughputReader.cs @@ -0,0 +1,55 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Session.TimeSeries; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; + +/// <summary> +/// Reads each licensing endpoint's daily message counts out of the throughput database, one batch per endpoint in +/// ordinal document-id order, with the batch's cursor on that endpoint's document id. +/// </summary> +sealed class LicensingThroughputReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<ThroughputDayRow>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.LicensingThroughput; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadThroughputDays(resumeAfter, cancellationToken); + + async IAsyncEnumerable<MigrationBatch> ReadThroughputDays(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // One endpoint per batch, because the batch size counts rows and one endpoint can bring a year or more of days. + await foreach (var endpoints in ThroughputDocuments.Endpoints(Lifecycle, CategoryId, resumeAfter, batchSize: 1, cancellationToken)) + { + var days = new List<MigrationRow>(); + + foreach (var endpoint in endpoints.Rows) + { + var id = ((Endpoint)endpoint.Document).Id; + + foreach (var point in await ReadThroughputSeries(endpoint.SourceId, cancellationToken)) + { + days.Add(new MigrationRow($"{endpoint.SourceId}|{point.Timestamp.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)}", new ThroughputDayRow(id, DateOnly.FromDateTime(point.Timestamp), (long)point.Value), ReadOnlyDictionary<string, object?>.Empty)); + } + } + + yield return new MigrationBatch(days, endpoints.Cursor); + } + } + + // Each series read is a request, and one session refuses more than MaxNumberOfRequestsPerSession of them. + async Task<TimeSeriesEntry[]> ReadThroughputSeries(string documentId, CancellationToken cancellationToken) + { + using var session = Lifecycle.OpenSession(Lifecycle.Settings.ThroughputDatabaseName); + return await session.IncrementalTimeSeriesFor(documentId, LicensingDataStore.ThroughputTimeSeriesName).GetAsync(token: cancellationToken) ?? []; + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs new file mode 100644 index 0000000000..12795ddbba --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/MessageRedirectsReader.cs @@ -0,0 +1,22 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.RavenDB.MessageRedirects; + +/// <summary> +/// Reads the message redirects document as one row per redirect, in the document's order. +/// </summary> +sealed class MessageRedirectsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<MessageRedirectsCollection, MessageRedirect>(lifecycle, lifecycle.Settings.DatabaseName, MessageRedirectsDataStore.CollectionId) +{ + public override string CategoryId => MigrationCategoryIds.MessageRedirects; + + // The writer skips a redirect with no source address, and that skip still needs an id to name. + protected override IEnumerable<(string SourceId, MessageRedirect Document)> RowsFrom(MessageRedirectsCollection document) => + document.ToRedirects().Select((redirect, index) => (redirect.FromPhysicalAddress ?? $"{DocumentId}#{index}", redirect)); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs new file mode 100644 index 0000000000..9a0ae42aea --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/NotificationSettingsReader.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Editing; + +/// <summary> +/// Reads the notification settings document, whose email settings become one Settings row. +/// </summary> +sealed class NotificationSettingsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<NotificationsSettingsDocument, EmailNotifications>(lifecycle, lifecycle.Settings.DatabaseName, NotificationsDataStore.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.NotificationSettings; + + protected override IEnumerable<(string SourceId, EmailNotifications Document)> RowsFrom(NotificationsSettingsDocument document) => + document.Email is { } email ? [(DocumentId, email)] : []; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs new file mode 100644 index 0000000000..e1e9775e64 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/PendingIntegrationEventsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the integration events the old instance had not yet dispatched, whole, out of the primary database. +/// </summary> +sealed class PendingIntegrationEventsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader<ExternalIntegrationDispatchRequest>(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.PendingIntegrationEvents; + + public override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(ExternalIntegrationRequestsDataStore.KeyPrefix + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs new file mode 100644 index 0000000000..bd92c28dbf --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RavenStorageCheckIds.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Frozen; +using System.Collections.Generic; + +/// <summary> +/// The six custom checks the RavenDB persister runs about its own database, each as its id and the category it reports under. +/// The custom checks reader flags a result only when both match, so a result that reuses one of these ids under another category is not flagged, such as the audit instance's "RavenDB dirty memory" under "ServiceControl.Audit Health". +/// </summary> +static class RavenStorageCheckIds +{ + // Each check passes its id and category to the CustomCheck base constructor as literals, so there is nothing to reference. RavenStorageCheckIdsTests fails if the two lists differ. + public static IReadOnlySet<(string Id, string Category)> All { get; } = new[] + { + ("Error Database Index Errors", "ServiceControl Health"), + ("Error Database Index Lag", "ServiceControl Health"), + ("ServiceControl database", "Storage space"), + ("Message Ingestion Process", "ServiceControl Health"), + ("RavenDB dirty memory", "ServiceControl Health"), + ("Error Database Search Engine", "ServiceControl Health") + }.ToFrozenSet(); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs new file mode 100644 index 0000000000..72280d9697 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/RetryOperationsReader.cs @@ -0,0 +1,24 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; +using ServiceControl.Recoverability; + +/// <summary> +/// Reads the retry history document as rows, historic operations first, then unacknowledged ones. +/// </summary> +sealed class RetryOperationsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<RetryHistory, RetryOperationRow>(lifecycle, lifecycle.Settings.DatabaseName, RetryHistoryDataStore.DocumentId) +{ + public override string CategoryId => MigrationCategoryIds.RetryOperations; + + protected override IEnumerable<(string SourceId, RetryOperationRow Document)> RowsFrom(RetryHistory document) => + [ + .. document.HistoricOperations.Select(operation => ($"historic|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(operation, null))), + .. document.UnacknowledgedOperations.Select(operation => ($"unacknowledged|{operation.RequestId}|{operation.RetryType}|{operation.CompletionTime:O}", new RetryOperationRow(null, operation))) + ]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs new file mode 100644 index 0000000000..7f61ffa043 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/SubscriptionsReader.cs @@ -0,0 +1,27 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the subscriptions document as one row per subscriber of each message type and version. +/// </summary> +sealed class SubscriptionsReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<Subscriptions, SubscriptionRow>(lifecycle, lifecycle.Settings.DatabaseName, Subscriptions.SingleDocumentId) +{ + public override string CategoryId => MigrationCategoryIds.Subscriptions; + + protected override IEnumerable<(string SourceId, SubscriptionRow Document)> RowsFrom(Subscriptions document) + { + foreach (var subscription in document.All.Values) + { + foreach (var subscriber in subscription.Subscribers) + { + yield return ($"{subscription.Id}|{subscriber.TransportAddress}", new SubscriptionRow(subscription.MessageType.TypeName, subscription.MessageType.Version, subscriber.TransportAddress, subscriber.Endpoint)); + } + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs new file mode 100644 index 0000000000..3724963f67 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/TrialEndDateReader.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the trial metadata document, whose end date becomes one Settings row. +/// </summary> +sealed class TrialEndDateReader(RavenReadOnlySourceLifecycle lifecycle) + : SingleDocumentReader<TrialMetadata, TrialMetadata>(lifecycle, lifecycle.Settings.DatabaseName, TrialLicenseDataProvider.TrialMetadataId) +{ + public override string CategoryId => MigrationCategoryIds.TrialEndDate; + + protected override IEnumerable<(string SourceId, TrialMetadata Document)> RowsFrom(TrialMetadata document) => [(DocumentId, document)]; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs new file mode 100644 index 0000000000..aac5d66c70 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/SingleDocumentReader.cs @@ -0,0 +1,54 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads a category that RavenDB keeps in one document with a fixed id. +/// The document is one batch, and its id is the cursor. A missing document gives a batch with no rows. +/// </summary> +abstract class SingleDocumentReader<TStored, TDocument>(RavenReadOnlySourceLifecycle lifecycle, string databaseName, string documentId) : MigrationCategoryReader<TDocument>(lifecycle) + where TStored : class + where TDocument : class +{ + /// <summary> + /// The id of the one document this category reads, and the cursor of its only batch. + /// </summary> + protected string DocumentId { get; } = documentId; + + public sealed override IAsyncEnumerable<MigrationBatch> Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + ReadDocument(resumeAfter, cancellationToken); + + /// <summary> + /// Turns the loaded document into rows, in the order the writer gets them. Called only when the document exists. + /// Leave out a missing value rather than return a null document. + /// </summary> + protected abstract IEnumerable<(string SourceId, TDocument Document)> RowsFrom(TStored document); + + async IAsyncEnumerable<MigrationBatch> ReadDocument(string? resumeAfter, [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (resumeAfter == DocumentId) + { + yield break; + } + + if (resumeAfter is not null) + { + throw new InvalidOperationException($"The migration cannot resume the '{CategoryId}' category after '{resumeAfter}', because its only cursor is '{DocumentId}'. Run --migration-retry {CategoryId} to copy the category again from the start."); + } + + using var session = Lifecycle.OpenSession(databaseName); + var document = await session.LoadAsync<TStored>(DocumentId, cancellationToken); + + yield return new MigrationBatch( + document is null ? [] : [.. RowsFrom(document).Select(row => new MigrationRow(row.SourceId, row.Document, ReadOnlyDictionary<string, object?>.Empty))], + DocumentId); + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs new file mode 100644 index 0000000000..77701e8a65 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/ThroughputDocuments.cs @@ -0,0 +1,80 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Conventions; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +/// <summary> +/// What the throughput database's readers share: which documents are endpoints, and how an endpoint document becomes a row. +/// </summary> +static class ThroughputDocuments +{ + static readonly string EndpointCollection = DocumentConventions.DefaultGetCollectionName(typeof(EndpointDocument)); + + /// <summary> + /// Reads the endpoints in ordinal document-id order, after <paramref name="resumeAfter" />, in batches of at most + /// <paramref name="batchSize" /> rows, each with its cursor on the last endpoint's id. Throws + /// <see cref="InvalidOperationException" /> when the source holds no document with the id in <paramref name="resumeAfter" />. + /// </summary> + public static async IAsyncEnumerable<MigrationBatch> Endpoints( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var endpoints = (await AllEndpoints(lifecycle, categoryId, resumeAfter, cancellationToken)) + .Where(row => resumeAfter is null || string.CompareOrdinal(row.SourceId, resumeAfter) > 0) + .OrderBy(row => row.SourceId, StringComparer.Ordinal); + + foreach (var chunk in endpoints.Chunk(batchSize)) + { + yield return new MigrationBatch(chunk, chunk[^1].SourceId); + } + } + + // With no id prefix RavenDB streams in reverse etag order and ignores startAfter, so callers order the rows and apply the cursor. + static async Task<List<MigrationRow>> AllEndpoints(RavenReadOnlySourceLifecycle lifecycle, string categoryId, string? resumeAfter, CancellationToken cancellationToken) + { + var databaseName = lifecycle.Settings.ThroughputDatabaseName; + var endpoints = new List<MigrationRow>(); + + using var session = lifecycle.OpenSession(databaseName); + await RavenDocumentStream.RefuseMissingCursor(session, categoryId, databaseName, resumeAfter, cancellationToken); + + await using var stream = await session.Advanced.StreamAsync<EndpointDocument>(string.Empty, token: cancellationToken); + + while (await stream.MoveNextAsync()) + { + if (Endpoint(stream.Current) is { } row) + { + endpoints.Add(row); + } + } + + return endpoints; + } + + /// <summary> + /// Turns one streamed document into a row that carries an <see cref="Particular.LicensingComponent.Contracts.Endpoint" />. Returns null when the document is not in the EndpointDocuments collection, which is the only place the product reads endpoints from. + /// The throughput database gives its documents no id prefix, so the stream also hands back its singleton documents and anything else stored there. + /// </summary> + public static MigrationRow? Endpoint(StreamResult<EndpointDocument> entry) => + IsEndpoint(entry) ? new MigrationRow(entry.Id, entry.Document.ToEndpoint(), ReadOnlyDictionary<string, object?>.Empty) : null; + + // RavenDB matches collection names ignoring case. + static bool IsEndpoint(StreamResult<EndpointDocument> entry) => + entry.Metadata.TryGetValue(Constants.Documents.Metadata.Collection, out var collection) && string.Equals(collection?.ToString(), EndpointCollection, StringComparison.OrdinalIgnoreCase); +} diff --git a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs index c801d834fa..1aefbe8e67 100644 --- a/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Editing/NotificationsDataStore.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Persistence.RavenDB.Editing class NotificationsDataStore(IRavenSessionProvider sessionProvider) : INotificationsDataStore { - const string SingleDocumentId = "NotificationsSettings/All"; + internal const string SingleDocumentId = "NotificationsSettings/All"; public async Task<NotificationsSettings> LoadSettings(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs index 0792bcd8ff..5c88f09c5a 100644 --- a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs @@ -47,5 +47,5 @@ public async Task UpdateEndpointSettings(EndpointSettings settings, Cancellation await session.SaveChangesAsync(cancellationToken); } - const string CollectionName = "EndpointSettings"; + internal const string CollectionName = "EndpointSettings"; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs index d9848ccb28..c1365095bc 100644 --- a/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ExternalIntegrationRequestsDataStore.cs @@ -39,7 +39,7 @@ public ExternalIntegrationRequestsDataStore( ); } - const string KeyPrefix = "ExternalIntegrationDispatchRequests"; + internal const string KeyPrefix = "ExternalIntegrationDispatchRequests"; public async Task StoreDispatchRequest(IEnumerable<ExternalIntegrationDispatchRequest> dispatchRequests, CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs b/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs index e9cfd8f89d..acbc5081eb 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenCustomCheckDataStore.cs @@ -44,7 +44,9 @@ public async Task<CheckStateChange> UpdateCustomCheckStatus(CustomCheckDetail de return status; } - static string MakeId(Guid id) => $"CustomChecks/{id}"; + internal const string CollectionName = "CustomChecks"; + + static string MakeId(Guid id) => $"{CollectionName}/{id}"; public async Task<QueryResult<IList<CustomCheckView>>> GetStats(PagingInfo paging, string status = null, CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 7dfaeb715c..234f9924f5 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -5,11 +5,13 @@ namespace ServiceControl.Persistence.RavenDB using System.Reflection; using Configuration; using CustomChecks; + using DataMigration; using Microsoft.Extensions.Logging; using Particular.LicensingComponent.Contracts; using ServiceControl.Infrastructure; + using ServiceControl.Persistence.DataMigration; - class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration + class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration, IMigrationSourceFactory { public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; @@ -89,5 +91,8 @@ public IPersistence Create(PersistenceSettings settings) var specificSettings = (RavenPersisterSettings)settings; return new RavenPersistence(specificSettings); } + + public IMigrationSource CreateSource(SettingsRootNamespace settingsRoot) => + new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)CreateSettings(settingsRoot), settingsRoot)); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs index 92fcfdf898..284700ead9 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/GroupsDataStore.cs @@ -146,6 +146,8 @@ public async Task DeleteComment(string groupId, CancellationToken cancellationTo await session.SaveChangesAsync(cancellationToken); } - public static string MakeId(string groupId) => $"GroupComment/{groupId}"; + internal const string CommentIdPrefix = "GroupComment/"; + + public static string MakeId(string groupId) => $"{CommentIdPrefix}{groupId}"; } } diff --git a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs index 93f55c7ef1..2df9d1af09 100644 --- a/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Recoverability/RetryHistoryDataStore.cs @@ -9,7 +9,7 @@ class RetryHistoryDataStore(IRavenSessionProvider sessionProvider) : IRetryHistoryDataStore { - const string DocumentId = "RetryOperations/History"; + internal const string DocumentId = "RetryOperations/History"; public async Task<QueryResult<RetryHistory>> GetRetryHistory(CancellationToken cancellationToken = default) { diff --git a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs index 55364bb14c..046b871336 100644 --- a/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/Throughput/LicensingDataStore.cs @@ -22,11 +22,11 @@ class LicensingDataStore( TimeProvider timeProvider) : ILicensingDataStore { internal const string ThroughputTimeSeriesName = "INC: throughput data"; - const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; + internal const string AuditServiceMetadataDocumentId = "AuditServiceMetadata"; const string AuditEnvironmentMetadataDocumentId = "AuditEnvironmentMetadata"; - const string BrokerMetadataDocumentId = "BrokerMetadata"; - const string ReportMasksDocumentId = "ReportMasks"; - const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; + internal const string BrokerMetadataDocumentId = "BrokerMetadata"; + internal const string ReportMasksDocumentId = "ReportMasks"; + internal const string LicencedEndpointDetailsDocumentId = "LicensedEndpointDetails"; const string LicencedEndpointDetailsAmendmentDocumentId = "LicensedEndpointDetailsAmendment"; static readonly AuditServiceMetadata DefaultAuditServiceMetadata = new([], []); diff --git a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs index f0feaa2aaf..a3d51910a1 100644 --- a/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs +++ b/src/ServiceControl.Persistence.RavenDB/TrialLicenseDataProvider.cs @@ -23,6 +23,6 @@ public async Task StoreTrialEndDate(DateOnly trialEndDate, CancellationToken can await session.SaveChangesAsync(cancellationToken); } - static string TrialMetadataId = "metadata/trialinformation"; + internal const string TrialMetadataId = "metadata/trialinformation"; } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs index a23b266f1b..dc2bcb5fc6 100644 --- a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs @@ -24,6 +24,8 @@ public Task Setup(IHostApplicationBuilder hostBuilder) return Task.CompletedTask; } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public Task PostSetup(IHost host) => Task.CompletedTask; public Task TearDown() => Task.CompletedTask; diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs new file mode 100644 index 0000000000..2c38adfcb1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/MigrationValueLengthTests.cs @@ -0,0 +1,163 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationValueLengthTests : PersistenceTestBase +{ + static readonly string OutsideTheBasicPlane = char.ConvertFromUtf32(0x1F600); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task A_value_is_measured_in_code_points_as_varchar_counts_them() + { + var fits = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(450), HostId = Guid.NewGuid(), Host = "SALES01" } }; + var tooLong = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(451), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", fits, new Dictionary<string, object>()), + new MigrationRow("KnownEndpoints/2", tooLong, new Dictionary<string, object>()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "450 code points fit a varchar(450) column though they are 900 UTF-16 units, and a skip here would be permanent"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2" })); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { Repeat(450) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<KnownEndpointEntity>(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 451, on document 'KnownEndpoints/2'" + }), "the length reported is the one PostgreSQL counts"); + Assert.That(ServiceProvider.GetRequiredService<IMigrationSqlDialect>().TextLength(Repeat(451)), Is.LessThanOrEqualTo(Repeat(451).Length), "the check skips measuring a value whose UTF-16 length fits, which is safe only while no count exceeds it"); + } + } + + [Test] + public async Task A_value_holding_a_NUL_character_is_skipped_as_malformed_naming_the_column() + { + var clean = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SALES01" } }; + var withNul = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SAL\u0000ES" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", clean, new Dictionary<string, object>()), + new MigrationRow("KnownEndpoints/2", withNul, new Dictionary<string, object>()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<KnownEndpointEntity>(ServiceProvider, nameof(KnownEndpointEntity.Host))} cannot hold the character U+0000, which the value contains, on document 'KnownEndpoints/2'" + })); + } + } + + [Test] + public async Task A_custom_check_whose_failure_reason_holds_a_NUL_character_is_skipped_as_malformed_and_the_rest_copy() + { + var endpoint = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SALES01" }; + var batch = new MigrationBatch( + [ + new MigrationRow("CustomChecks/1", CustomCheck(endpoint, "Sales.HealthCheck", "timed out"), new Dictionary<string, object>()), + new MigrationRow("CustomChecks/2", CustomCheck(endpoint, "Sales.DiskCheck", "disk \u0000 full"), new Dictionary<string, object>()), + new MigrationRow("CustomChecks/3", CustomCheck(endpoint, "Sales.QueueCheck", null), new Dictionary<string, object>()) + ], + "CustomChecks/3"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(CustomChecksCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.CustomChecks, MigrationCategoryState.InProgress, "CustomChecks/3", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<CustomCheckEntity>(ServiceProvider, nameof(CustomCheckEntity.FailureReason))} cannot hold the character U+0000, which the value contains, on document 'CustomChecks/2'" + })); + Assert.That(await Target.Count(CustomChecksCategory), Is.EqualTo(2)); + } + } + + [Test] + public async Task A_failed_error_import_whose_exception_info_holds_a_NUL_character_is_skipped_as_malformed_and_the_rest_copy() + { + var batch = new MigrationBatch( + [ + FailedErrorImportRow("native-1", "Deserialization failed"), + FailedErrorImportRow("native-2", "Deserialization \u0000 failed"), + FailedErrorImportRow("native-3", "Deserialization failed") + ], + "FailedErrorImports/native-3"); + var writer = new FailedErrorImportsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), ServiceProvider.GetRequiredService<IBodyStoragePersistence>(), ServiceProvider.GetRequiredService<EFPersisterSettings>()); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, writer, batch); + var result = await Target.Write(FailedErrorImportsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.FailedErrorImports, MigrationCategoryState.InProgress, "FailedErrorImports/native-3", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "PostgreSQL refuses U+0000 in text, which would fail the whole insert and halt the category"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 1 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<FailedErrorImportEntity>(ServiceProvider, nameof(FailedErrorImportEntity.ExceptionInfo))} cannot hold the character U+0000, which the value contains, on document 'FailedErrorImports/native-2'" + })); + Assert.That(await Target.Count(FailedErrorImportsCategory), Is.EqualTo(2)); + } + } + + static readonly MigrationCategory FailedErrorImportsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + // The NUL goes in ExceptionInfo, because the headers are written as JSON, which escapes it. + static MigrationRow FailedErrorImportRow(string nativeMessageId, string exceptionInfo) => new( + $"FailedErrorImports/{nativeMessageId}", + new FailedErrorImport + { + Id = $"FailedErrorImports/{nativeMessageId}", + Message = new FailedTransportMessage { Id = nativeMessageId, Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = nativeMessageId }, Body = [1, 2, 3] }, + ExceptionInfo = exceptionInfo + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc) }, + new MigrationBody(new byte[] { 1, 2, 3 }, "application/octet-stream")); + + static readonly MigrationCategory CustomChecksCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; + + static CustomCheckDetail CustomCheck(EndpointDetails endpoint, string customCheckId, string failureReason) => new() + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = "Ping", + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + HasFailed = failureReason is not null, + FailureReason = failureReason + }; + + static string Repeat(int codePoints) => string.Concat(Enumerable.Repeat(OutsideTheBasicPlane, codePoints)); +} diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs index 4ce91496d5..b99a642473 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs @@ -51,7 +51,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton<TimeProvider>(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -59,6 +59,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService<IDatabaseMigrator>().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs new file mode 100644 index 0000000000..75d1b0bc09 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/CustomCheckSourceTests.cs @@ -0,0 +1,76 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +class CustomCheckSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_a_custom_check_as_the_neutral_detail_type_with_its_id_recoverable() + { + var endpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + var expectedId = await SeedCustomCheck(endpoint, "Sales.HealthCheck", "Ping", failed: true, reason: "timed out"); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "CustomChecks"); + + await using var source = await OpenMigrationSource(); + + var detail = (CustomCheckDetail)(await CollectBatches(source, category)).SelectMany(batch => batch.Rows).Single().Document; + + using (Assert.EnterMultipleScope()) + { + Assert.That(detail.GetDeterministicId(), Is.EqualTo(expectedId), "the id must be recoverable from the same fields RavenDB used to generate it"); + Assert.That(detail.CustomCheckId, Is.EqualTo("Sales.HealthCheck")); + Assert.That(detail.HasFailed, Is.True); + Assert.That(detail.FailureReason, Is.EqualTo("timed out")); + } + } + + [Test] + public async Task Flags_only_a_row_whose_id_and_category_are_one_of_the_persisters_own_checks() + { + var endpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + await SeedCustomCheck(endpoint, "Error Database Index Lag", "ServiceControl Health", failed: false, reason: null); + await SeedCustomCheck(new EndpointDetails { Name = "Billing", Host = "MACHINE", HostId = Guid.NewGuid() }, "Error Database Index Lag", "Another category", failed: false, reason: null); + await SeedCustomCheck(endpoint, "Sales.HealthCheck", "Ping", failed: false, reason: null); + await SeedCustomCheck(endpoint, "Error Message Ingestion Process", "ServiceControl Health", failed: false, reason: null); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "CustomChecks"); + + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToDictionary(row => ((CustomCheckDetail)row.Document).CustomCheckId + "|" + ((CustomCheckDetail)row.Document).Category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows["Error Database Index Lag|ServiceControl Health"].Metadata[MigrationMetadataKeys.SourceStorageCheck], Is.True); + Assert.That(rows["Error Database Index Lag|Another category"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck), "the same id under another category is not the persister's check"); + Assert.That(rows["Error Message Ingestion Process|ServiceControl Health"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck), "another id under the persister's category is not the persister's check"); + Assert.That(rows["Sales.HealthCheck|Ping"].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.SourceStorageCheck)); + } + } + + async Task<Guid> SeedCustomCheck(EndpointDetails endpoint, string customCheckId, string category, bool failed, string reason) + { + var detail = new CustomCheckDetail + { + OriginatingEndpoint = endpoint, + CustomCheckId = customCheckId, + Category = category, + ReportedAt = Now, + HasFailed = failed, + FailureReason = reason + }; + + await CustomChecks.UpdateCustomCheckStatus(detail); + + return detail.GetDeterministicId(); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs new file mode 100644 index 0000000000..27a23e01bf --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedErrorImportSourceTests.cs @@ -0,0 +1,48 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +class FailedErrorImportSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Every_import_arrives_with_its_body_attached_and_the_time_RavenDB_last_modified_it() + { + await FailedImportStore.StoreFailedErrorImport(new FailedErrorImport + { + Id = "native-1", + Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }); + await FailedImportStore.StoreFailedErrorImport(new FailedErrorImport { Id = "native-2", Message = null, ExceptionInfo = "Deserialization failed" }); + await CompleteDatabaseOperation(); + + DateTime lastModified; + + using (var session = await SessionProvider.OpenSession()) + { + lastModified = session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedErrorImport>(FailedErrorImportDataStore.MakeDocumentId("native-1"))).Value; + } + + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports))).SelectMany(batch => batch.Rows).ToDictionary(row => row.SourceId); + var withMessage = rows[FailedErrorImportDataStore.MakeDocumentId("native-1")]; + var withoutMessage = rows[FailedErrorImportDataStore.MakeDocumentId("native-2")]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(withMessage.Body!.Content.ToArray(), Is.EqualTo(new byte[] { 1, 2, 3 }), "attached in Read, so the engine never asks ReadBody, which refuses this category"); + Assert.That(withMessage.Body.ContentType, Is.EqualTo("application/octet-stream"), "the default the SQL import store gives a message with no content type header"); + Assert.That(withMessage.Metadata[MigrationMetadataKeys.FailedAt], Is.EqualTo(lastModified.ToUniversalTime()), "RavenDB keeps no failure time, only the stamp it wrote when the import was stored"); + Assert.That(((DateTime)withMessage.Metadata[MigrationMetadataKeys.FailedAt]!).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(withoutMessage.Body?.Content.Length, Is.Zero, "an empty body rather than none, so the writer skips the row instead of the engine stopping on ReadBody"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs new file mode 100644 index 0000000000..beaee964fe --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageBodySourceTests.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class FailedMessageBodySourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory FailedMessages = + MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + [Test] + public async Task Reads_the_attachment_and_its_own_content_type() + { + var content = Encoding.UTF8.GetBytes("<order>1</order>"); + var documentId = await SeedWithAttachment(content, "application/json"); + + await using var source = await OpenMigrationSource(); + var body = await source.ReadBody(FailedMessages, documentId); + + Assert.That(body, Is.Not.Null); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Content.ToArray(), Is.EqualTo(content)); + Assert.That(body.ContentType, Is.EqualTo("application/json"), "the message header says text/xml, the attachment's own type wins"); + } + } + + [Test] + public async Task Reads_a_zero_length_attachment_as_an_empty_body() + { + var documentId = await SeedWithAttachment([], "text/plain"); + + await using var source = await OpenMigrationSource(); + var body = await source.ReadBody(FailedMessages, documentId); + + Assert.That(body, Is.Not.Null, "an empty payload is a body, and is not the same as a missing one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(body.Content.Length, Is.Zero); + Assert.That(body.ContentType, Is.EqualTo("text/plain")); + } + } + + [Test] + public async Task Returns_null_when_the_document_has_no_attachment() + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + + await using var source = await OpenMigrationSource(); + + Assert.That(await source.ReadBody(FailedMessages, message.Id), Is.Null); + } + + [Test] + public async Task Returns_null_for_a_document_that_does_not_exist() + { + await using var source = await OpenMigrationSource(); + + Assert.That(await source.ReadBody(FailedMessages, "FailedMessages/no-such-message"), Is.Null, + "a message RavenDB expired during the copy has to be a BodyMissing skip, not three failed reads and a BodyUnreadable one"); + } + + [Test] + public async Task Refuses_a_category_it_does_not_read_body_attachments_for() + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + var failedErrorImports = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.FailedErrorImports); + + await using var source = await OpenMigrationSource(); + + var exception = Assert.ThrowsAsync<NotSupportedException>(() => source.ReadBody(failedErrorImports, message.Id), + "a failed error import keeps its body inside the document, so reading an attachment would skip every one of them as BodyMissing"); + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.FailedErrorImports)); + } + + async Task<string> SeedWithAttachment(byte[] content, string contentType) + { + var message = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + + using var session = await SessionProvider.OpenSession(); + session.Advanced.Attachments.Store(message.Id, "body", new MemoryStream(content), contentType); + await session.SaveChangesAsync(); + + return message.Id; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs new file mode 100644 index 0000000000..4aee73b24b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/FailedMessageStatusChangedAtTests.cs @@ -0,0 +1,114 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration +{ + using System; + using NUnit.Framework; + using ServiceControl.MessageFailures; + using ServiceControl.Persistence.RavenDB.DataMigration; + + [TestFixture] + class FailedMessageStatusChangedAtTests + { + static readonly TimeSpan ErrorRetentionPeriod = TimeSpan.FromDays(10); + + [TestCase(FailedMessageStatus.Resolved)] + [TestCase(FailedMessageStatus.Archived)] + public void Reconstructs_the_status_change_instant_from_the_expiry_stamp(FailedMessageStatus status) + { + var statusChangedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(status), statusChangedAt + ErrorRetentionPeriod, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(statusChangedAt)); + } + + [TestCase(FailedMessageStatus.Unresolved)] + [TestCase(FailedMessageStatus.RetryIssued)] + public void Records_the_newest_attempt_for_a_status_that_never_expires(FailedMessageStatus status) + { + var failure = new IngestedFailure(); + var stamp = new DateTime(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(failure.ToFailedMessage(status), stamp, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(failure.AttemptedAt), "a stamp on a status that never expires is not its status change"); + } + + [Test] + public void Falls_back_to_the_newest_attempt_for_an_archived_message_with_no_stamp() + { + var failure = new IngestedFailure(); + + var resolved = FailedMessageStream.StatusChangedAt(failure.ToFailedMessage(FailedMessageStatus.Archived), null, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(failure.AttemptedAt)); + } + + [Test] + public void Returns_null_for_a_document_with_no_attempts_and_no_stamp() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + Assert.That(FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod), Is.Null); + } + + [Test] + public void Returns_null_for_a_document_whose_attempt_list_is_null_and_has_no_stamp() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + + Assert.That(FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod), Is.Null, "the translator skips a message with no attempts, so this must not throw and halt the category"); + } + + [Test] + public void Reconstructs_an_instant_already_past_the_retention_cutoff_faithfully() + { + var expires = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived), expires, ErrorRetentionPeriod); + + Assert.That(resolved, Is.EqualTo(expires - ErrorRetentionPeriod), "the value the source implies, not one adjusted to survive the sweep"); + } + + [TestCase(DateTimeKind.Unspecified)] + [TestCase(DateTimeKind.Utc)] + public void Returns_a_UTC_instant_whatever_kind_the_attempt_was_read_with(DateTimeKind kind) + { + var document = new IngestedFailure().ToFailedMessage(); + var attemptedAt = new DateTime(2026, 9, 1, 8, 0, 0, kind); + document.ProcessingAttempts[^1].AttemptedAt = attemptedAt; + + var resolved = FailedMessageStream.StatusChangedAt(document, null, ErrorRetentionPeriod); + + using (Assert.EnterMultipleScope()) + { + Assert.That(resolved.Value.Kind, Is.EqualTo(DateTimeKind.Utc), "MigrationMetadataKeys promises a UTC DateTime and DateTime equality ignores Kind"); + Assert.That(resolved.Value.Ticks, Is.EqualTo(attemptedAt.Ticks)); + } + } + + [Test] + public void Reads_an_expiry_stamp_with_no_kind_as_UTC() + { + var statusChangedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Unspecified); + + var resolved = FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived), statusChangedAt + ErrorRetentionPeriod, ErrorRetentionPeriod); + + using (Assert.EnterMultipleScope()) + { + Assert.That(resolved.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(resolved.Value.Ticks, Is.EqualTo(statusChangedAt.Ticks), "an unspecified instant is already UTC and must not be shifted by the local offset"); + } + } + + [Test] + public void Refuses_a_negative_source_retention_period() + { + var ex = Assert.Throws<ArgumentOutOfRangeException>(() => + FailedMessageStream.StatusChangedAt(new IngestedFailure().ToFailedMessage(), null, TimeSpan.FromDays(-1))); + + Assert.That(ex.Message, Does.Contain("ErrorRetentionPeriod")); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs new file mode 100644 index 0000000000..af5f2d510d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/GroupCommentsSourceTests.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Recoverability; + +class GroupCommentsSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments); + + [Test] + public async Task Reads_each_stored_comment_keyed_by_its_group_id_a_cleared_one_included() + { + await GroupsStore.EditComment("group-1", "Known issue, ignore until Thursday"); + await GroupsStore.EditComment("group-2", ""); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + var rows = (await CollectBatches(source, Category)).SelectMany(batch => batch.Rows).ToList(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { GroupsDataStore.MakeId("group-1"), GroupsDataStore.MakeId("group-2") }), "the cursor and every skip name the RavenDB document id"); + Assert.That(rows.Select(row => ((GroupComment)row.Document).Id), Is.EqualTo(new[] { "group-1", "group-2" }), "SQL keys a comment on the bare group id, so that is what crosses"); + Assert.That(rows.Select(row => ((GroupComment)row.Document).Comment), Is.EqualTo(new[] { "Known issue, ignore until Thursday", "" }), "a cleared comment is read, so the writer counts its skip instead of it vanishing uncounted"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs new file mode 100644 index 0000000000..66b7214ec9 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +class KnownEndpointSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_known_endpoints_by_the_plural_prefix_in_batches_with_their_monitored_flag() + { + using (var session = await SessionProvider.OpenSession()) + { + foreach (var (name, monitored) in new[] { ("Sales.Orders", true), ("Billing", false), ("Shipping", false) }) + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }, + HostDisplayName = "HOST01", + Monitored = monitored + }; + + await session.StoreAsync(endpoint, $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"); + } + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var batches = await CollectBatches(source, category, batchSize: 2); + + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 1 })); + Assert.That(batches.SelectMany(batch => batch.Rows).Count(row => ((KnownEndpoint)row.Document).Monitored), Is.EqualTo(1)); + } + + // KnownEndpointsWriter skips these as RequiredValueMissing, which it can only do if the reader hands them over despite the required members. + [Test] + public async Task Reads_a_known_endpoint_missing_a_required_value_with_that_value_null() + { + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(new { EndpointDetails = new { HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = false }, "KnownEndpoints/no-name"); + await session.StoreAsync(new { EndpointDetails = new { Name = "Sales.Orders", HostId = Guid.NewGuid() }, Monitored = false }, "KnownEndpoints/no-host"); + await session.StoreAsync(new { Monitored = false }, "KnownEndpoints/no-details"); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var endpoints = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToDictionary(row => row.SourceId, row => (KnownEndpoint)row.Document); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints["KnownEndpoints/no-name"].EndpointDetails.Name, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-host"].EndpointDetails.Host, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-details"].EndpointDetails, Is.Null); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs new file mode 100644 index 0000000000..5dc3de34e7 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingEndpointsSourceTests.cs @@ -0,0 +1,95 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +class LicensingEndpointsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_endpoints_and_skips_the_four_singleton_documents() + { + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)) + { + SanitizedName = "sales.orders" + }, "Sales.Orders/Broker"); + + await session.StoreAsync(new { }, RavenLicensingDataStore.AuditServiceMetadataDocumentId); + await session.StoreAsync(new { MaskedStrings = new[] { "secret" } }, RavenLicensingDataStore.ReportMasksDocumentId); + await session.StoreAsync(new { }, RavenLicensingDataStore.BrokerMetadataDocumentId); + await session.StoreAsync(new { LicenseId = "abc" }, RavenLicensingDataStore.LicencedEndpointDetailsDocumentId); + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingEndpoints"); + + var batches = await CollectBatches(source, category); + var rows = batches.SelectMany(batch => batch.Rows).ToList(); + + Assert.That(rows, Has.Count.EqualTo(1)); + var endpoint = (Endpoint)rows[0].Document; + Assert.That(endpoint.Id.Name, Is.EqualTo("Sales.Orders")); + } + + [Test] + public async Task Reads_endpoints_in_id_order_and_resumes_after_the_cursor_whatever_order_they_were_stored_in() + { + foreach (var name in new[] { "Z", "A", "M" }) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)) { SanitizedName = name.ToLowerInvariant() }, $"{name}/Broker"); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingEndpoints); + + var batches = await CollectBatches(source, category, batchSize: 1); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor, batchSize: 1); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "A/Broker", "M/Broker", "Z/Broker" }), "each batch cursors on its last endpoint, in id order rather than the order RavenDB stored them"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { "M/Broker", "Z/Broker" }), "a resume reads only the endpoints after the cursor"); + } + } + + [Test] + public async Task A_document_of_another_collection_is_neither_read_nor_counted() + { + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)) + { + SanitizedName = "sales.orders" + }, "Sales.Orders/Broker"); + + var stray = new { Setting = "left by an older version" }; + await session.StoreAsync(stray, "LegacySettings/1"); + session.Advanced.GetMetadataFor(stray)["@collection"] = "LegacySettings"; + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingEndpoints); + + var rows = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToList(); + var count = await source.Count(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Has.None.StartsWith("LegacySettings/"), "the product reads endpoints only from the EndpointDocuments collection, so a document of any other kind is not an endpoint"); + Assert.That(rows, Has.Count.EqualTo(1)); + Assert.That(count, Is.EqualTo(rows.Count), "the count reads what the copy reads, so --migration-verify expects no row for the stray document"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs new file mode 100644 index 0000000000..fa228033bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingSettingsSourceTests.cs @@ -0,0 +1,115 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.Throughput.Models; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; + +class LicensingSettingsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_the_report_masks_document_as_its_list_of_masked_strings() + { + await Seed(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso", "fabrikam"] }); + + var batches = await Read(MigrationCategoryIds.LicensingReportMasks); + + var row = batches.SelectMany(batch => batch.Rows).Single(); + using (Assert.EnterMultipleScope()) + { + Assert.That(row.SourceId, Is.EqualTo("ReportMasks")); + Assert.That((List<string>)row.Document, Is.EqualTo(new[] { "contoso", "fabrikam" })); + Assert.That(batches.Last().Cursor, Is.EqualTo("ReportMasks")); + } + } + + [Test] + public async Task Reads_the_licensed_endpoint_details_document_whole() + { + var seeded = Details; + await Seed(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, seeded); + + var batches = await Read(MigrationCategoryIds.LicensedEndpointDetails); + + var details = (LicensedEndpointDetails)batches.SelectMany(batch => batch.Rows).Single().Document; + using (Assert.EnterMultipleScope()) + { + Assert.That(JsonSerializer.Serialize(details), Is.EqualTo(JsonSerializer.Serialize(seeded)), "every field, including the endpoint, queue and product arrays, must survive the read"); + Assert.That(batches.Last().Cursor, Is.EqualTo("LicensedEndpointDetails")); + } + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails, "LicensedEndpointDetails")] + public async Task An_absent_document_gives_one_empty_batch_that_still_records_the_cursor(string categoryId, string documentId) + { + var batches = await Read(categoryId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { documentId })); + } + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks, "ReportMasks")] + public async Task A_read_resumed_after_the_documents_own_id_gives_nothing(string categoryId, string documentId) + { + await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); + + var batches = await Read(categoryId, documentId); + + Assert.That(batches, Is.Empty); + } + + [TestCase(MigrationCategoryIds.LicensingReportMasks)] + public async Task A_read_resumed_after_a_cursor_the_reader_never_issued_is_refused(string categoryId) + { + await Seed("ReportMasks", new ReportConfigurationDocument { MaskedStrings = ["contoso"] }); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Read(categoryId, "Sales.Orders/Broker")); + + Assert.That(exception!.Message, Does.Contain(categoryId).And.Contain("'Sales.Orders/Broker'")); + } + + [Test] + public async Task A_report_masks_document_whose_list_is_null_reads_as_an_empty_list() + { + await Seed(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = null }); + + var batches = await Read(MigrationCategoryIds.LicensingReportMasks); + + var row = batches.SelectMany(batch => batch.Rows).Single(); + Assert.That(row.Document, Is.InstanceOf<List<string>>().And.Empty, "the product shows a null list as no masks, and a null document halts the writer with a reader and writer mismatch"); + } + + static LicensedEndpointDetails Details => new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + async Task Seed(string id, object document) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(document, id); + await session.SaveChangesAsync(); + } + + async Task<List<MigrationBatch>> Read(string categoryId, string resumeAfter = null) + { + await using var source = await OpenMigrationSource(); + return await CollectBatches(source, MigrationCategoryRegistry.All.Single(entry => entry.Id == categoryId), resumeAfter); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs new file mode 100644 index 0000000000..7497fef6f8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/LicensingThroughputSourceTests.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Conventions; +using ServiceControl.Persistence.DataMigration; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; +using ServiceControl.Persistence.RavenDB.Throughput.Models; + +class LicensingThroughputSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_one_endpoints_whole_time_series_as_one_batch_and_cursors_on_its_document_id() + { + var documentId = "Sales.Orders/Broker"; + + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName) + .Increment(new DateTime(2026, 1, 1), 100); + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName) + .Increment(new DateTime(2026, 1, 2), 250); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + + Assert.That(batches, Has.Count.EqualTo(1)); + Assert.That(batches[0].Cursor, Is.EqualTo(documentId)); + Assert.That(batches[0].Rows.Select(row => row.Document), Is.EqualTo(new[] + { + new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100), + new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 2), 250) + }), "each row carries the endpoint its count belongs to, because EndpointDailyThroughput does not, and the day and total RavenDB holds"); + Assert.That(batches[0].Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "Sales.Orders", "Sales.Orders" }), "the endpoint's name keeps its case, which the record's equality ignores"); + } + + [Test] + public async Task A_resume_reads_only_the_endpoints_after_the_cursor_whatever_order_they_were_stored_in() + { + foreach (var name in new[] { "Z", "A", "M" }) + { + var documentId = $"{name}/Broker"; + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName).Increment(new DateTime(2026, 1, 1), 100); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "A/Broker", "M/Broker", "Z/Broker" }), "one endpoint per batch, in id order rather than the order RavenDB stored them"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "M", "Z" }), "a resume reads only the days of endpoints after the cursor"); + } + } + + [Test] + public async Task Endpoints_whose_names_differ_only_in_case_and_trailing_spaces_come_in_separate_batches() + { + foreach (var name in new[] { "Sales.Orders", "Sales.Orders.Archive", "SALES.ORDERS " }) + { + var documentId = $"{name}/Broker"; + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier(name, ThroughputSource.Broker)), documentId); + await session.SaveChangesAsync(); + + session.IncrementalTimeSeriesFor(documentId, RavenLicensingDataStore.ThroughputTimeSeriesName).Increment(new DateTime(2026, 1, 1), 100); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + var resumed = await CollectBatches(source, category, resumeAfter: batches[0].Cursor); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { "SALES.ORDERS /Broker", "Sales.Orders.Archive/Broker", "Sales.Orders/Broker" }), "the source groups by its own document id only, and which names are one key is the target's to decide"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name).Single()), Is.EqualTo(new[] { "SALES.ORDERS ", "Sales.Orders.Archive", "Sales.Orders" }), "each batch holds the days of the one endpoint its cursor names"); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => ((ThroughputDayRow)row.Document).Endpoint.Name), Is.EqualTo(new[] { "Sales.Orders.Archive", "Sales.Orders" }), "a resume reads only the endpoints after the cursor, even one whose name differs from the cursor's only in case and spaces"); + } + } + + [Test] + public async Task Reads_more_endpoints_than_one_session_accepts_requests() + { + var endpointCount = new DocumentConventions().MaxNumberOfRequestsPerSession + 1; + + using (var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName)) + { + for (var index = 0; index < endpointCount; index++) + { + await session.StoreAsync(new EndpointDocument(new EndpointIdentifier($"Endpoint{index}", ThroughputSource.Broker)), $"Endpoint{index}/Broker"); + } + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == "LicensingThroughput"); + + var batches = await CollectBatches(source, category); + + Assert.That(batches, Has.Count.EqualTo(endpointCount), "each endpoint's series is its own request, and one session refuses this many"); + Assert.That(await source.Count(category), Is.Zero); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs new file mode 100644 index 0000000000..ebd380e1bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MessageRedirectsSourceTests.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using RavenMessageRedirectsDataStore = Persistence.RavenDB.MessageRedirects.MessageRedirectsDataStore; + +class MessageRedirectsSourceTests : RavenMigrationSourceTestBase +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects); + + [Test] + public async Task A_document_whose_last_redirect_was_removed_gives_one_empty_batch() + { + var redirect = new MessageRedirect { FromPhysicalAddress = "old1", ToPhysicalAddress = "new1", LastModified = DateTime.UtcNow }; + await MessageRedirectsDataStore.AddRedirect(redirect); + await MessageRedirectsDataStore.RemoveRedirect(redirect); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { RavenMessageRedirectsDataStore.CollectionId })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs new file mode 100644 index 0000000000..9fb6081bb3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration; + +[TestFixture] +class MigrationCategoryReadersTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Every_reader_claims_a_category_the_registry_knows() + { + await using var source = (RavenMigrationSource)await OpenMigrationSource(); + + var unknown = source.SupportedCategoryIds.Where(id => MigrationCategoryRegistry.Find(id) is null).ToArray(); + + Assert.That(unknown, Is.Empty, "a reader for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_reader_derives_from_the_typed_base() + { + var readers = typeof(IMigrationCategoryReader).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryReader).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(readers, Is.Not.Empty, "the test proves nothing if it finds no reader"); + Assert.That(readers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryReader<>))).Select(type => type.Name), Is.Empty, + "a reader that implements the interface directly can declare a DocumentType it does not yield, which the pairing test cannot see"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs new file mode 100644 index 0000000000..0cbf6a8e4d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationEndpointDetailsParserParityTests.cs @@ -0,0 +1,106 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using NServiceBus; +using NServiceBus.Faults; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.RavenDB.DataMigration; +// Aliased because the RavenDB persister's own EndpointDetailsParser would otherwise win from this namespace. +using IngestionParser = Contracts.Operations.EndpointDetailsParser; + +[TestFixture] +class MigrationEndpointDetailsParserParityTests +{ + [TestCaseSource(nameof(HeaderSets))] + public void Rebuilds_both_endpoints_exactly_as_ingestion_does(Dictionary<string, string> headers) + { + using (Assert.EnterMultipleScope()) + { + Assert.That(Shape(MigrationEndpointDetailsParser.SendingEndpoint(headers)), Is.EqualTo(Shape(IngestionParser.SendingEndpoint(headers))), + "a migrated message whose sending endpoint differs from a newly ingested one is grouped apart from it in ServicePulse"); + Assert.That(Shape(MigrationEndpointDetailsParser.ReceivingEndpoint(headers)), Is.EqualTo(Shape(IngestionParser.ReceivingEndpoint(headers))), + "a migrated message whose receiving endpoint differs from a newly ingested one is grouped apart from it in ServicePulse"); + } + } + + static (string Name, string Host, Guid HostId)? Shape(EndpointDetails endpoint) => + endpoint is null ? null : (endpoint.Name, endpoint.Host, endpoint.HostId); + + static IEnumerable<TestCaseData> HeaderSets() + { + var sendingHostId = Guid.Parse("6a3a9b1c-2d4e-4f50-8a61-7b8c9d0e1f20").ToString(); + var receivingHostId = Guid.Parse("0f1e2d3c-4b5a-4968-8776-655443322110").ToString(); + + yield return Case("full headers", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = sendingHostId, + [Headers.ProcessingEndpoint] = "Billing", + [Headers.HostDisplayName] = "Billing host", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = receivingHostId + }); + + yield return Case("non-GUID host ids", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = "not-a-guid", + [Headers.ProcessingEndpoint] = "Billing", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = "not-a-guid" + }); + + yield return Case("null-valued host ids", new() + { + [Headers.OriginatingEndpoint] = "Sales", + [Headers.OriginatingMachine] = "SALES-01", + [Headers.OriginatingHostId] = null, + [Headers.ProcessingEndpoint] = "Billing", + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = null + }); + + yield return Case("null-valued host display name", new() + { + [Headers.ProcessingEndpoint] = "Billing", + [Headers.HostDisplayName] = null, + [Headers.ProcessingMachine] = "BILLING-01", + [Headers.HostId] = receivingHostId + }); + + yield return Case("addresses with a machine", new() + { + [Headers.OriginatingAddress] = "Sales@SALES-01", + [FaultsHeaderKeys.FailedQ] = "Billing@BILLING-01" + }); + + yield return Case("addresses without a machine", new() + { + [Headers.OriginatingAddress] = "Sales", + [FaultsHeaderKeys.FailedQ] = "Billing" + }); + + yield return Case("failed queue with a non-GUID host id", new() + { + [FaultsHeaderKeys.FailedQ] = "Billing@BILLING-01", + [Headers.HostId] = "not-a-guid" + }); + + yield return Case("null-valued addresses", new() + { + [Headers.OriginatingAddress] = null, + [FaultsHeaderKeys.FailedQ] = null + }); + + yield return Case("no endpoint headers", new() + { + [Headers.MessageId] = "message-1" + }); + } + + static TestCaseData Case(string name, Dictionary<string, string> headers) => new TestCaseData(headers).SetArgDisplayNames(name); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs new file mode 100644 index 0000000000..e5afca8d87 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/PendingIntegrationEventsSourceTests.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; + +class PendingIntegrationEventsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_each_queued_request_with_its_context_as_the_concrete_type() + { + await ExternalIntegrationStore.StoreDispatchRequest( + [ + new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "one" } }, + new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "two" } } + ]); + await CompleteDatabaseOperation(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.PendingIntegrationEvents); + + await using var source = await OpenMigrationSource(); + var contexts = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).Select(row => ((ExternalIntegrationDispatchRequest)row.Document).DispatchContext).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(contexts, Has.All.InstanceOf<TestDispatchContext>(), "an untyped context would serialize as the JSON library's own type and never dispatch"); + Assert.That(contexts.Cast<TestDispatchContext>().Select(context => context.Value), Is.EquivalentTo(new[] { "one", "two" })); + Assert.That(await source.Count(category), Is.EqualTo(2)); + } + } + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs new file mode 100644 index 0000000000..f940d6f8ba --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs @@ -0,0 +1,69 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +abstract class RavenMigrationSourceTestBase : RavenPersistenceTestBase +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + static readonly object SettingsGate = new(); + + protected string ThroughputDatabaseName => ((RavenPersisterSettings)PersistenceSettings).ThroughputDatabaseName; + + protected async Task<IMigrationSource> OpenMigrationSource() + { + var settings = (RavenPersisterSettings)PersistenceSettings; + (string Name, string Value)[] variables = + [ + ("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", settings.ConnectionString), + ("SERVICECONTROL_RAVENDB_DATABASENAME", settings.DatabaseName), + ("SERVICECONTROL_ERRORRETENTIONPERIOD", settings.ErrorRetentionPeriod.ToString()), + ("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", settings.ThroughputDatabaseName) + ]; + + IMigrationSource source; + + // Environment variables are process wide and these tests run in parallel, so without the gate a source + // reads whichever test set them last and opens another test's database. + lock (SettingsGate) + { + try + { + foreach (var (name, value) in variables) + { + Environment.SetEnvironmentVariable(name, value); + } + + source = new RavenPersistenceConfiguration().CreateSource(SettingsRoot); + } + finally + { + foreach (var (name, _) in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + } + } + + await source.Open(); + + return source; + } + + protected static async Task<List<MigrationBatch>> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List<MigrationBatch>(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs new file mode 100644 index 0000000000..21502d2d23 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -0,0 +1,202 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.Unicast.Subscriptions; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Notifications; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.MessageRedirects; +using ServiceControl.Persistence.RavenDB.Throughput.Models; +using RavenLicensingDataStore = Persistence.RavenDB.Throughput.LicensingDataStore; + +class RavenMigrationSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_endpoint_settings_in_document_id_order_in_batches_of_the_requested_size() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var batches = await CollectBatches(source, EndpointSettingsCategory, batchSize: 2); + + Assert.Multiple(() => + { + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 2, 1 }), "A batch never exceeds the requested size, and the trailing partial batch is still yielded."); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.Ordered); + }); + } + + [Test] + public async Task Every_row_is_identified_by_the_document_id_the_persister_wrote_it_under() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, EndpointSettingsCategory)).SelectMany(batch => batch.Rows).ToList(); + + Assert.Multiple(() => + { + Assert.That(rows, Has.Count.EqualTo(5)); + Assert.That(rows.Select(row => row.SourceId), Has.All.StartWith("EndpointSettings/"), "The cursor the engine checkpoints is a source document id, so a row identified by anything else cannot be resumed after."); + }); + } + + [Test] + public async Task Resuming_after_a_cursor_yields_only_what_follows_it() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var firstBatch = (await CollectBatches(source, EndpointSettingsCategory, batchSize: 2))[0]; + var resumed = (await CollectBatches(source, EndpointSettingsCategory, resumeAfter: firstBatch.Cursor, batchSize: 2)).SelectMany(batch => batch.Rows).ToList(); + + Assert.That(resumed, Has.Count.EqualTo(3)); + Assert.That(resumed.Select(row => row.SourceId), Has.No.Member(firstBatch.Rows[0].SourceId)); + Assert.That(resumed.Select(row => row.SourceId), Is.Ordered); + } + + [Test] + public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_instead_of_starting_past_it() + { + // The cursor is saved in the target database, so a restored backup or a corrected database name leaves + // one naming a document this source does not have. + await SeedEndpointSettings(3); + await using var source = await OpenMigrationSource(); + + var refusal = Assert.ThrowsAsync<InvalidOperationException>(() => CollectBatches(source, EndpointSettingsCategory, resumeAfter: "EndpointSettings/zzz-from-another-database")); + + Assert.Multiple(() => + { + Assert.That(refusal.Message, Does.Contain(EndpointSettingsCategory.Id)); + Assert.That(refusal.Message, Does.Contain("EndpointSettings/zzz-from-another-database")); + Assert.That(refusal.Message, Does.Contain(DatabaseName), "the operator has to be told which database the cursor was looked for in"); + Assert.That(refusal.Message, Does.Contain("expired"), "an archived or resolved document can expire during an interrupted copy, on any source"); + Assert.That(refusal.Message, Does.Contain($"--migration-retry {EndpointSettingsCategory.Id}"), "a retry re-reads the category from the start, so it is the way out"); + }); + } + + [Test] + public async Task Counts_every_row_the_category_would_read() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.EqualTo(5)); + } + + [Test] + public async Task Counts_a_category_with_nothing_in_it_as_zero() + { + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.Zero, "A category with nothing to copy has to be told apart from one this source declines to count."); + } + + [Test] + public async Task Reading_every_category_creates_no_index() + { + await using var source = await OpenMigrationSource(); + + Assert.That(source.SupportedCategoryIds, Is.SubsetOf(Seeds.Keys), "A category read with nothing in it cannot show whether its reader builds an index, so a new reader needs its seed adding here."); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await Seeds[categoryId](this); + } + + var before = await IndexNames(); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await CollectBatches(source, MigrationCategoryRegistry.Find(categoryId)); + } + + var after = await IndexNames(); + + Assert.Multiple(() => + { + Assert.That(after, Is.EquivalentTo(before), "A reader that queries without naming a static index has RavenDB build one and index the whole collection on the customer's live database."); + Assert.That(after, Has.None.StartWith("Auto/")); + }); + } + + // Keyed by category id so a reader added without seed data fails the index test by name instead of passing + // over an empty collection. + static readonly Dictionary<string, Func<RavenMigrationSourceTests, Task>> Seeds = new() + { + [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), + [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }), + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = tests => tests.SeedFailedMessage(new IngestedFailure().ToFailedMessage()), + [MigrationCategoryIds.LicensingEndpoints] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), + [MigrationCategoryIds.LicensingThroughput] = tests => tests.SeedThroughputDocument("Sales.Orders/Broker", new EndpointDocument(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker))), + [MigrationCategoryIds.LicensingReportMasks] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.ReportMasksDocumentId, new ReportConfigurationDocument { MaskedStrings = ["contoso"] }), + [MigrationCategoryIds.LicensedEndpointDetails] = tests => tests.SeedThroughputDocument(RavenLicensingDataStore.LicencedEndpointDetailsDocumentId, new LicensedEndpointDetails { LicenseId = "abc-123" }), + [MigrationCategoryIds.MessageRedirects] = tests => tests.MessageRedirectsDataStore.AddRedirect(new MessageRedirect { FromPhysicalAddress = "Sales@OLDHOST", ToPhysicalAddress = "Sales@NEWHOST", LastModified = DateTime.UtcNow }), + [MigrationCategoryIds.Subscriptions] = tests => tests.SeedSubscription(), + [MigrationCategoryIds.NotificationSettings] = tests => tests.NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + [MigrationCategoryIds.TrialEndDate] = tests => tests.ServiceProvider.GetRequiredService<ITrialLicenseDataProvider>().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + [MigrationCategoryIds.RetryOperations] = tests => tests.RetryHistoryStore.RecordRetryOperationCompleted("group-1", RetryType.FailureGroup, DateTime.UtcNow, DateTime.UtcNow, "Sales.Orders", "Exception Type", false, 1, DateTime.UtcNow, 10), + [MigrationCategoryIds.PendingIntegrationEvents] = tests => tests.ExternalIntegrationStore.StoreDispatchRequest([new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "seed" } }]), + [MigrationCategoryIds.CustomChecks] = tests => tests.CustomChecks.UpdateCustomCheckStatus(new CustomCheckDetail { OriginatingEndpoint = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }, CustomCheckId = "Sales.HealthCheck", Category = "Ping" }), + [MigrationCategoryIds.FailedErrorImports] = tests => tests.FailedImportStore.StoreFailedErrorImport(new FailedErrorImport { Id = "native-1", Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, ExceptionInfo = "Deserialization failed" }), + [MigrationCategoryIds.GroupComments] = tests => tests.GroupsStore.EditComment("group-1", "Known issue, ignore until Thursday") + }; + + string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; + + // Both databases, because the licensing readers read the throughput database. + async Task<string[]> IndexNames() => + [.. await IndexNamesIn(DatabaseName), .. await IndexNamesIn(ThroughputDatabaseName)]; + + Task<string[]> IndexNamesIn(string database) => + DocumentStore.Maintenance.ForDatabase(database).SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); + + const int IndexNamePageSize = 1024; + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.Find("EndpointSettings"); + + async Task SeedThroughputDocument(string id, object document) + { + using var session = DocumentStore.OpenAsyncSession(ThroughputDatabaseName); + await session.StoreAsync(document, id); + await session.SaveChangesAsync(); + } + + async Task SeedSubscription() + { + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)), + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using var session = await SessionProvider.OpenSession(); + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + + async Task SeedEndpointSettings(int count) + { + for (var index = 0; index < count; index++) + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = $"Endpoint{index}", TrackInstances = true }); + } + } + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs new file mode 100644 index 0000000000..eefcf35846 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenStorageCheckIdsTests.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using NServiceBus.CustomChecks; +using NUnit.Framework; +using ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +class RavenStorageCheckIdsTests : RavenPersistenceTestBase +{ + [Test] + public void Lists_exactly_the_checks_the_RavenDB_persister_registers_with_their_categories() => + Assert.That(ServiceProvider.GetServices<ICustomCheck>().Select(check => (check.Id, check.Category)), Is.EquivalentTo(RavenStorageCheckIds.All), + "the custom checks reader flags exactly these pairs, so a check the persister adds, renames or moves to another category would be copied to SQL, where nothing reports it again"); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs new file mode 100644 index 0000000000..ba3f892ffe --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -0,0 +1,380 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Session; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Configuration; +using ServiceControl.MessageFailures; +using ServiceControl.Operations.BodyStorage.RavenAttachments; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +class ReadOnlySourceLifecycleTests +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + + string databaseName; + IDocumentStore bootstrapStore; + RavenPersisterSettings sourceSettings; + + [SetUp] + public async Task SetUp() + { + var embeddedServer = await SharedEmbeddedServer.GetInstance(); + databaseName = Guid.NewGuid().ToString("n"); + + bootstrapStore = new DocumentStore + { + Urls = [embeddedServer.ServerUrl], + Database = databaseName, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(databaseName))); + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord($"{databaseName}-throughput"))); + + using (var session = bootstrapStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessage { UniqueMessageId = "abc", Status = FailedMessageStatus.Archived }, "FailedMessages/abc"); + await session.SaveChangesAsync(); + } + + sourceSettings = new RavenPersisterSettings + { + DatabaseName = databaseName, + ThroughputDatabaseName = $"{databaseName}-throughput", + ConnectionString = embeddedServer.ServerUrl, + ErrorRetentionPeriod = TimeSpan.FromDays(10) + }; + } + + [TearDown] + public void TearDown() => bootstrapStore?.Dispose(); + + [Test] + public async Task Opening_the_source_creates_no_index() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + + Assert.That(statistics.CountOfIndexes, Is.Zero, "Opening a migration source must not create the fifteen ServiceControl indexes on it."); + } + [Test] + public async Task Opening_the_source_creates_no_database() + { + var absentThroughput = $"{databaseName}-absent"; + sourceSettings.ThroughputDatabaseName = absentThroughput; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentThroughput).And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentThroughput)), Is.Null, "Opening a migration source must not create a database that was missing."); + } + + [Test] + public async Task Opening_the_source_refuses_a_missing_primary_database() + { + var absentPrimary = $"{databaseName}-absent"; + sourceSettings.DatabaseName = absentPrimary; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentPrimary).And.Contain("ServiceControl/RavenDB/DatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentPrimary)), Is.Null, "Opening a migration source must not create a primary database that was missing."); + } + + [Test] + public async Task Opening_the_source_writes_no_database_settings() + { + var before = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)) + { + await lifecycle.Open(); + } + + var after = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + Assert.That(after, Is.EquivalentTo(before), "Opening a migration source must not rewrite the settings of the database it reads."); + } + + [Test] + public async Task Opening_the_source_configures_no_expiration() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Opening a migration source must not enable RavenDB document expiry against it, or the customer's fallback data is deleted while they are migrating."); + } + + [Test] + public async Task Writing_through_the_source_store_throws() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def"); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => await session.SaveChangesAsync()); + + Assert.That(exception.Message, Does.Contain("open read-only")); + await AssertAbsent("FailedMessages/def"); + } + + [Test] + public async Task A_source_session_cannot_even_stage_a_write() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def")); + + Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the RefuseWrite request hook. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); + } + + [Test] + public async Task Failed_message_status_reads_back_as_stored() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + var loaded = await session.LoadAsync<FailedMessage>("FailedMessages/abc"); + + Assert.That(loaded.Status, Is.EqualTo(FailedMessageStatus.Archived), "Without SaveEnumsAsIntegers every message status is misread, and it looks like data corruption rather than a missing convention."); + } + + [Test] + public async Task A_failed_open_leaves_no_store_behind() + { + sourceSettings.ThroughputDatabaseName = $"{databaseName}-absent"; + + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + Assert.ThrowsAsync<InvalidOperationException>(async () => await lifecycle.Open()); + + var afterFailure = Assert.Throws<InvalidOperationException>(() => _ = lifecycle.DocumentStore); + + Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed Open must dispose what it created. The caller never received a source, so nothing else can, and on the embedded path what leaks is a live RavenDB server process holding the customer's data directory."); + + await lifecycle.DisposeAsync(); + } + [Test] + public async Task A_source_describes_itself_as_facts_naming_the_setting_each_came_from() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var description = await source.Describe(); + var facts = description.Facts.ToDictionary(fact => fact.Label); + + Assert.Multiple(() => + { + Assert.That(description.Version, Does.StartWith("6.")); + Assert.That(facts["Mode"].Value, Is.EqualTo("external")); + Assert.That(facts["Primary database"].Value, Is.EqualTo(databaseName)); + Assert.That(facts["Primary database"].SettingKey, Is.EqualTo("ServiceControl/RavenDB/DatabaseName")); + Assert.That(facts["Throughput database"].Value, Is.EqualTo($"{databaseName}-throughput")); + }); + } + + [Test] + public async Task An_unopened_source_refuses_to_describe_itself() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => await source.Describe()); + + Assert.That(exception.Message, Does.Contain("is not open"), "Separating construction from Open is what lets a host register a source in its container, and it buys a state where nothing is connected yet. That state has to fail loudly rather than return an empty description."); + } + + [Test] + public async Task The_source_inventories_every_collection_by_database() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var inventory = await source.Inventory(); + + Assert.That(inventory.Single(entry => entry.Scope == databaseName && entry.Name == "FailedMessages").Count, Is.EqualTo(1)); + } + + [Test] + public void An_embedded_source_names_the_data_directory_setting() + { + sourceSettings.ConnectionString = null; + + var location = RavenReadOnlySourceLifecycle.Located(sourceSettings, SettingsRoot); + + Assert.That(location, Does.Contain("'ServiceControl/DbPath'"), "An operator told the embedded source cannot be read needs the setting that points at its data directory."); + } + + [Test] + public async Task Opening_the_source_twice_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain("already open"), "A second Open would abandon the first store, and on the embedded path a running server process with it."); + } + + [Test] + public async Task Generating_an_id_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + + Assert.CatchAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "hilo", Status = FailedMessageStatus.Unresolved })); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo reserves an id range with a request of its own before SaveChanges, so only the request hook can refuse it."); + } + + [Test] + public async Task A_patch_against_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( + new PatchOperation("FailedMessages/abc", null, new PatchRequest { Script = "this.Status = 1;" }))); + + Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch is a request of its own with no session, so only the request hook can refuse it; the RavenDB persister writes this way."); + } + + [Test] + public async Task A_bulk_insert_into_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + { + await using var bulk = lifecycle.DocumentStore.BulkInsert(databaseName); + await bulk.StoreAsync(new FailedMessage { UniqueMessageId = "bulk", Status = FailedMessageStatus.Unresolved }, "FailedMessages/bulk"); + }); + + await AssertAbsent("FailedMessages/bulk"); + } + + [Test] + public async Task Reconfiguring_expiry_on_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync( + new ConfigureExpirationOperation(new ExpirationConfiguration { Disabled = false, DeleteFrequencyInSec = 60 }))); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Enabling expiry on the source would start deleting the customer fallback data."); + } + + [Test] + public async Task Deleting_an_untracked_document_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + session.Delete("FailedMessages/abc"); + + Assert.CatchAsync(async () => await session.SaveChangesAsync()); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "A delete by id of an untracked document only goes out at SaveChanges, as a batch request the hook must refuse."); + } + + [Test] + public async Task Reading_the_source_still_works() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var loaded = await session.LoadAsync<FailedMessage>("FailedMessages/abc"); + var queried = await session.Query<FailedMessage>().ToListAsync(); + + Assert.Multiple(() => + { + Assert.That(loaded, Is.Not.Null, "A guard that fails closed still has to let every read the migration needs through."); + Assert.That(queried, Has.Count.EqualTo(1)); + }); + } + + [Test] + public async Task A_source_session_reads_a_message_body_attachment() + { + using (var session = bootstrapStore.OpenAsyncSession()) + { + using var stored = new MemoryStream(Encoding.UTF8.GetBytes("<order>1</order>")); + session.Advanced.Attachments.Store("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName, stored, "text/xml"); + await session.SaveChangesAsync(); + } + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var sourceSession = lifecycle.OpenSession(databaseName); + using var attachment = await sourceSession.Advanced.Attachments.GetAsync("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName); + using var read = new MemoryStream(); + await attachment.Stream.CopyToAsync(read); + + Assert.Multiple(() => + { + Assert.That(Encoding.UTF8.GetString(read.ToArray()), Is.EqualTo("<order>1</order>"), "The body copy reads every attachment through this no-tracking session, so a read that needed tracking would need a second kind of source session."); + Assert.That(attachment.Details.ContentType, Is.EqualTo("text/xml")); + }); + } + + async Task AssertAbsent(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync<FailedMessage>(documentId); + + Assert.That(found, Is.Null, $"The source refused the write, so '{documentId}' must not be in the database. Asserting the exception alone tests the guard, not the guarantee."); + } + + async Task<long> CountDocuments() + { + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + return statistics.CountOfDocuments; + } + + async Task<FailedMessageStatus> LoadStatus(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync<FailedMessage>(documentId); + return found.Status; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs new file mode 100644 index 0000000000..a45824203a --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/SubscriptionsSourceTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Unicast.Subscriptions; +using NUnit.Framework; +using ServiceControl.Infrastructure.RavenDB.Subscriptions; +using ServiceControl.Persistence.DataMigration; + +class SubscriptionsSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Flattens_the_dictionary_of_subscriptions_into_one_row_per_subscriber() + { + var eventV1 = new MessageType("MyMessages.OrderPlaced", new Version(1, 0, 0)); + var eventV2 = new MessageType("MyMessages.OrderPlaced", new Version(2, 0, 0)); + + var document = new Subscriptions(); + document.All["v1"] = new Subscription + { + Id = "v1", + MessageType = eventV1, + Subscribers = [ + new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }, + new SubscriptionClient { Endpoint = "Shipping", TransportAddress = "Shipping@SHIP01" } + ] + }; + document.All["v2"] = new Subscription + { + Id = "v2", + MessageType = eventV2, + Subscribers = [new SubscriptionClient { Endpoint = "Billing", TransportAddress = "Billing@BILLING01" }] + }; + + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(document, Subscriptions.SingleDocumentId); + await session.SaveChangesAsync(); + } + + var batches = await Read(); + + Assert.That(batches.Last().Cursor, Is.EqualTo(Subscriptions.SingleDocumentId)); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => (row.SourceId, (SubscriptionRow)row.Document)), Is.EquivalentTo(new[] + { + ("v1|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("v1|Shipping@SHIP01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping")), + ("v2|Billing@BILLING01", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")) + }), "each row carries every value its subscriber holds, under an id naming the subscription and the address"); + } + + async Task<List<MigrationBatch>> Read() + { + await using var source = await OpenMigrationSource(); + return await CollectBatches(source, MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions)); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs new file mode 100644 index 0000000000..9faec057e1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/UnresolvedFailedMessageSourceTests.cs @@ -0,0 +1,939 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Configuration; +using ServiceControl.MessageFailures; +using ServiceControl.MessageFailures.Api; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.DataMigration; + +[TestFixture] +class UnresolvedFailedMessageSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_both_unresolved_statuses_and_neither_of_the_others() + { + var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + var retryIssued = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Resolved)); + await CompleteDatabaseOperation(); + + var rows = await ReadAll(); + + Assert.That( + rows.Select(row => ((FailedMessage)row.Document).UniqueMessageId), + Is.EquivalentTo(new[] { unresolved.UniqueMessageId, retryIssued.UniqueMessageId })); + } + + [Test] + public async Task Counts_only_the_documents_this_category_copies_not_the_whole_collection() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(Category), Is.EqualTo(2), "the archived message shares the collection but belongs to another category"); + } + + [Test] + public async Task Refuses_a_document_whose_status_neither_failed_message_category_copies() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage((FailedMessageStatus)99)); + await CompleteDatabaseOperation(); + + var refusal = Assert.ThrowsAsync<InvalidOperationException>(ReadAll); + + Assert.That(refusal.Message, Does.Contain(stored.Id).And.Contain("99"), "a status no category copies would otherwise be left behind with nothing to show for it"); + } + + [Test] + public async Task Hands_over_attempt_times_as_UTC() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(numberOfAttempts: 2)); + await CompleteDatabaseOperation(); + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + Assert.That(attempts.SelectMany(attempt => new[] { attempt.AttemptedAt.Kind, attempt.FailureDetails.TimeOfFailure.Kind }), Is.All.EqualTo(DateTimeKind.Utc)); + } + + [Test] + public async Task Resolves_StatusChangedAt_to_the_newest_attempt_when_it_is_neither_first_nor_last() + { + var failure = new IngestedFailure(); + var message = failure.ToFailedMessage(FailedMessageStatus.Unresolved, numberOfAttempts: 3); + var attempts = message.ProcessingAttempts; + message.ProcessingAttempts = [attempts[1], attempts[2], attempts[0]]; + await SeedFailedMessage(message); + await CompleteDatabaseOperation(); + + var row = (await ReadAll()).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.EqualTo(failure.AttemptedAt)); + Assert.That(((DateTime)row.Metadata[MigrationMetadataKeys.StatusChangedAt]).Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public async Task Sends_only_the_migration_metadata_keys_across_the_seam() + { + await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata.Keys, Is.EquivalentTo(new[] { MigrationMetadataKeys.StatusChangedAt, MigrationMetadataKeys.LastModified }), "a store's own metadata, such as @expires, never travels"); + } + + [Test] + public async Task A_retry_issued_message_says_whether_the_source_still_holds_its_retry_claim() + { + var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var released = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + var unresolved = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + using (var session = DocumentStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = unresolved.Id, RetryBatchId = "RetryBatches/staging" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(unresolved.UniqueMessageId)); + await session.SaveChangesAsync(); + } + + await CompleteDatabaseOperation(); + + var rows = (await ReadAll()).ToDictionary(row => ((FailedMessage)row.Document).UniqueMessageId); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows[claimed.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True); + Assert.That(rows[released.UniqueMessageId].Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.False); + Assert.That(rows[unresolved.UniqueMessageId].Metadata, Does.Not.ContainKey(MigrationMetadataKeys.RetryClaimHeld), "only a retry-issued message gets a claim in SQL, even when the source holds one for another"); + } + } + + [Test] + public async Task Carries_the_documents_own_last_modified() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + await CompleteDatabaseOperation(); + + DateTime lastModified; + + using (var session = await SessionProvider.OpenSession()) + { + lastModified = session.Advanced.GetLastModifiedFor(await session.LoadAsync<FailedMessage>(stored.Id)).Value; + } + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.LastModified], Is.EqualTo(lastModified.ToUniversalTime()), "the value RavenDB shows as the message's LastModified, not the copy time"); + } + + [Test] + public async Task A_run_of_dropped_documents_yields_an_empty_batch_only_once_the_walk_limit_is_reached() + { + var ids = await SeedInIdOrder(Enumerable.Repeat(FailedMessageStatus.Archived, 7).ToArray()); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 3); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[2], ids[5], ids[6] }), "an empty stretch commits once per walk limit, plus the tail so the cursor past it is saved"); + Assert.That(batches.SelectMany(batch => batch.Rows), Is.Empty); + } + } + + [Test] + public async Task The_category_reads_no_archived_or_resolved_document() + { + await SeedInIdOrder(FailedMessageStatus.Archived, FailedMessageStatus.Resolved, FailedMessageStatus.Archived); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category, batchSize: 2); + + Assert.That(batches, Is.Empty, "the copy runs while ServiceControl is closed, so reading the archive only to drop it makes the outage grow with the archive"); + } + + [Test] + public async Task Resuming_after_a_cursor_reads_only_the_unresolved_messages_after_it() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.RetryIssued, + FailedMessageStatus.Unresolved, FailedMessageStatus.Resolved, FailedMessageStatus.Unresolved); + + await using var source = await OpenMigrationSource(); + var first = (await CollectBatches(source, Category, batchSize: 2))[0]; + var resumed = await CollectBatches(source, Category, resumeAfter: first.Cursor, batchSize: 2); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { ids[0], ids[2] })); + Assert.That(first.Cursor, Is.EqualTo(ids[2])); + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[3], ids[5] }), "a resume neither repeats a row nor skips one"); + } + } + + [Test] + public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_instead_of_starting_past_it() + { + await SeedInIdOrder(FailedMessageStatus.Unresolved, FailedMessageStatus.Unresolved); + const string cursor = "FailedMessages/zzz-from-another-database"; + + await using var source = await OpenMigrationSource(); + var refusal = Assert.ThrowsAsync<InvalidOperationException>(() => CollectBatches(source, Category, resumeAfter: cursor)); + + Assert.That(refusal.Message, Does.Contain(Category.Id).And.Contain(cursor), "a cursor from a restored backup or another database would otherwise resume part way through and skip rows"); + } + + [Test] + public async Task Waits_for_the_index_to_catch_up_so_no_unresolved_message_is_missed() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)); + await CompleteDatabaseOperation(); + await DocumentStore.Maintenance.SendAsync(new StopIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + + // The stopped index still counts the message as archived, so its count matches the collection, and only the wait for the index keeps the read from finding nothing. + using (var session = DocumentStore.OpenAsyncSession()) + { + (await session.LoadAsync<FailedMessage>(stored.Id, TestContext.CurrentContext.CancellationToken)).Status = FailedMessageStatus.Unresolved; + await session.SaveChangesAsync(TestContext.CurrentContext.CancellationToken); + } + + var reading = ReadAll(); + var finishedWhileStale = await Task.WhenAny(reading, Task.Delay(TimeSpan.FromSeconds(3), TestContext.CurrentContext.CancellationToken)) == reading; + + await DocumentStore.Maintenance.SendAsync(new StartIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var rows = await reading.WaitAsync(TimeSpan.FromSeconds(60), TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finishedWhileStale, Is.False, "a query on an index that has not caught up leaves out the messages it has not indexed yet"); + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id })); + } + } + + [Test] + public async Task Resumes_after_a_cursor_that_names_an_archived_message() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved); + + await using var source = await OpenMigrationSource(); + var resumed = await CollectBatches(source, Category, resumeAfter: ids[1]); + + Assert.That(resumed.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(new[] { ids[2], ids[4] }), "a run that read the whole collection can save an archived message as its cursor, and the next run may query the index"); + } + + [Test] + public async Task Reads_through_a_Corax_index_as_through_a_Lucene_one() + { + await UseCorax(); + var ids = await SeedInIdOrder(FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.RetryIssued, FailedMessageStatus.Resolved); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category); + + Assert.That(batches.Select(batch => (batch.Cursor, batch.Rows.Select(row => row.SourceId))), Is.EqualTo(new[] { (ids[2], new[] { ids[0], ids[2] }) }), "a full read would walk on to the resolved message and save its cursor past it"); + } + + [Test] + public async Task Resumes_through_a_Corax_index_with_no_row_repeated_or_skipped() + { + await UseCorax(); + var statuses = Enumerable.Range(0, 60).Select(position => (FailedMessageStatus)((position % 4) + 1)).ToArray(); + var ids = await SeedInIdOrder(statuses); + var expected = ids.Where((_, position) => statuses[position] is FailedMessageStatus.Unresolved or FailedMessageStatus.RetryIssued).ToList(); + + await using var source = await OpenMigrationSource(); + var read = new List<string>(); + string cursor = null; + + // Every batch is read by a fresh call, as a restart after each commit would. + while ((await CollectBatches(source, Category, resumeAfter: cursor, batchSize: 7)).FirstOrDefault() is { } batch) + { + Assert.That(batch.Cursor, Is.Not.EqualTo(cursor), "a resume that hands back the batch it resumed after never ends"); + read.AddRange(batch.Rows.Select(row => row.SourceId)); + cursor = batch.Cursor; + } + + var afterArchived = await CollectBatches(source, Category, resumeAfter: ids[Array.IndexOf(statuses, FailedMessageStatus.Archived)]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(read, Is.EqualTo(expected), "Corax has to order and compare id() as the cursor expects"); + Assert.That(afterArchived.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.EqualTo(expected.SkipWhile(id => string.CompareOrdinal(id, ids[Array.IndexOf(statuses, FailedMessageStatus.Archived)]) < 0))); + } + } + + [Test] + public async Task Reads_the_whole_collection_when_the_index_is_missing() + { + await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + var claimed = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)); + + using (var session = DocumentStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessageRetry { FailedMessageId = claimed.Id, RetryBatchId = "RetryBatches/forwarded" }, RetryDocumentDataStore.MakeFailedMessageRetriesDocumentId(claimed.UniqueMessageId)); + await session.SaveChangesAsync(); + } + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { stored.Id, claimed.Id }), "the copy cannot build the index on a source it only reads, so reading everything is the only way through"); + Assert.That(rows.Single(row => row.SourceId == claimed.Id).Metadata[MigrationMetadataKeys.RetryClaimHeld], Is.True, "the whole-collection read must resolve claims too, or the writer stops on the first retry-issued row"); + } + } + + [Test] + public async Task Reads_the_whole_collection_instead_of_waiting_for_a_disabled_index() + { + await DocumentStore.Maintenance.SendAsync(new DisableIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { stored.Id }), "a disabled index never catches up, so waiting for it would keep ServiceControl closed for good"); + } + + [Test] + public async Task Reads_the_whole_collection_instead_of_waiting_for_an_errored_index() + { + var unindexable = await SeedFailedMessage(FailedMessageWithATimeSentThatIsNotADate()); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + Assert.That(await IndexStateNow(), Is.EqualTo(IndexState.Error), "one document that fails out of one attempt errors the index"); + + // An errored index stops part way, so this one stays unindexed and the index stays stale. + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)); + + var rows = await ReadAll().WaitAsync(TimeSpan.FromSeconds(30), TestContext.CurrentContext.CancellationToken); + + Assert.That(rows.Select(row => row.SourceId), Is.EquivalentTo(new[] { unindexable.Id, stored.Id }), "an errored index never catches up, so waiting for it would keep ServiceControl closed for good"); + } + + [Test] + public async Task Reads_the_whole_collection_when_the_index_leaves_out_a_document_it_could_not_index() + { + await using (var bulkInsert = DocumentStore.BulkInsert(token: TestContext.CurrentContext.CancellationToken)) + { + foreach (var message in Enumerable.Range(0, 200).Select(_ => new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived))) + { + await bulkInsert.StoreAsync(message, FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId)); + } + } + + var unindexable = await SeedFailedMessage(FailedMessageWithATimeSentThatIsNotADate()); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + Assert.That(await IndexStateNow(), Is.EqualTo(IndexState.Normal), "RavenDB errors the whole index only above a 15 percent failure rate, so one bad document among many leaves it working and silently short"); + + var rows = await ReadAll(); + + Assert.That(rows.Select(row => row.SourceId), Is.EqualTo(new[] { unindexable.Id }), "a query would leave out the document the index could not map, where the target should record it as a skip"); + } + + [Test] + public async Task A_batch_yields_as_soon_as_it_holds_batch_size_rows_however_far_below_the_walk_limit() + { + var ids = await SeedInIdOrder(Enumerable.Repeat(FailedMessageStatus.Unresolved, 5).ToArray()); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 3); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[1], ids[3], ids[4] })); + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 2, 1 }), "a batch never holds more than the target can write in one go"); + } + } + + [Test] + public async Task A_batch_short_of_batch_size_rows_yields_once_the_walk_limit_is_reached() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Archived, FailedMessageStatus.Archived, + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Archived); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 4); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[3], ids[6] }), "a sparse stretch still saves its cursor every walk limit, so a resume walks at most that far again"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => row.SourceId)), Is.EqualTo(new[] { new[] { ids[0] }, new[] { ids[4] } })); + } + } + + [Test] + public async Task A_full_batch_yields_on_rows_held_not_on_documents_walked() + { + var ids = await SeedInIdOrder( + FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved, FailedMessageStatus.Archived, FailedMessageStatus.Unresolved); + + var batches = await StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch: 10); + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches.Select(batch => batch.Cursor), Is.EqualTo(new[] { ids[2], ids[4] }), "a batch is full when it holds batch size rows, however many documents it walked to find them"); + Assert.That(batches.Select(batch => batch.Rows.Select(row => row.SourceId)), Is.EqualTo(new[] { new[] { ids[0], ids[2] }, new[] { ids[4] } })); + } + } + + [TestCase(0)] + [TestCase(-1)] + public void Refuses_a_walk_limit_that_is_not_positive(int maxDocumentsWalkedPerBatch) + { + var refusal = Assert.ThrowsAsync<ArgumentOutOfRangeException>(() => StreamWithWalkLimit(batchSize: 2, maxDocumentsWalkedPerBatch)); + + Assert.That(refusal.ParamName, Is.EqualTo("maxDocumentsWalkedPerBatch"), "a limit of zero or less is never reached, so every batch would wait for batch size rows however far it walked"); + } + + [Test] + public async Task The_category_saves_its_cursor_at_least_every_ten_thousand_documents_read() + { + var messages = Enumerable.Range(0, 10_001) + .Select(_ => new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved)) + .OrderBy(message => FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId), StringComparer.Ordinal) + .ToList(); + + await using (var bulkInsert = DocumentStore.BulkInsert(token: TestContext.CurrentContext.CancellationToken)) + { + foreach (var message in messages) + { + await bulkInsert.StoreAsync(message, FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId)); + } + } + + await CompleteDatabaseOperation(); + + await using var source = await OpenMigrationSource(); + var batches = await CollectBatches(source, Category, batchSize: 20_000); + + Assert.That( + batches.Select(batch => (batch.Cursor, batch.Rows.Count)), + Is.EqualTo(new[] { (FailedMessageIdGenerator.MakeDocumentId(messages[9_999].UniqueMessageId), 10_000), (FailedMessageIdGenerator.MakeDocumentId(messages[10_000].UniqueMessageId), 1) }), + "the reader commits every RavenDocumentStream.MaxDocumentsWalkedPerBatch documents, which is 10,000, however large the batch size"); + } + + [Test] + public async Task A_document_stored_without_FailureDetails_comes_back_with_an_empty_failing_address() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage(numberOfAttempts: 2)); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "delete this.ProcessingAttempts[0].FailureDetails; delete this.ProcessingAttempts[1].FailureDetails;" })); + await CompleteDatabaseOperation(); + var readStartedAt = DateTime.UtcNow; + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempts.Select(attempt => attempt.FailureDetails?.AddressOfFailingEndpoint), Is.All.Empty, "a missing key reads back as the default object, not null, so the translator has to treat an empty address as missing"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.TimeOfFailure), Is.All.GreaterThanOrEqualTo(readStartedAt), "the default object stamps the time it was read, so its time of failure is invented"); + } + } + + [Test] + public async Task A_FailureDetails_stored_without_its_TimeOfFailure_comes_back_with_no_time_rather_than_the_time_it_was_read() + { + var failure = new IngestedFailure { FailingEndpointAddress = "Sales@MACHINE" }; + var stored = await SeedFailedMessage(failure.ToFailedMessage(numberOfAttempts: 2)); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "delete this.ProcessingAttempts[0].FailureDetails.TimeOfFailure; delete this.ProcessingAttempts[1].FailureDetails.TimeOfFailure;" })); + await CompleteDatabaseOperation(); + + var attempts = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts; + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempts[^1].FailureDetails.TimeOfFailure, Is.EqualTo(default(DateTime)), "the translator skips a newest attempt with no time of failure, and the read time would copy it with a made-up one"); + Assert.That(attempts[0].FailureDetails.TimeOfFailure, Is.EqualTo(default(DateTime)), "the translator leaves an older attempt with no time out of the first and last failure times"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.AddressOfFailingEndpoint), Is.All.EqualTo("Sales@MACHINE"), "the rest of FailureDetails is still read as stored"); + Assert.That(attempts.Select(attempt => attempt.FailureDetails.Exception?.Message), Is.All.EqualTo(failure.ExceptionMessage)); + } + } + + [Test] + public async Task A_FailureDetails_stored_as_null_comes_back_as_null() + { + var stored = await SeedFailedMessage(new IngestedFailure().ToFailedMessage()); + await DocumentStore.Operations.SendAsync(new PatchOperation(stored.Id, null, new PatchRequest { Script = "this.ProcessingAttempts[0].FailureDetails = null;" })); + await CompleteDatabaseOperation(); + + var attempt = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single(); + + Assert.That(attempt.FailureDetails, Is.Null, "the translator skips an attempt with no FailureDetails, so one bad document must not halt the category"); + } + + [TestCase("SendingEndpoint", NServiceBus.Headers.OriginatingHostId)] + [TestCase("ReceivingEndpoint", NServiceBus.Headers.HostId)] + public void RestoreMetadataTypes_gives_an_endpoint_the_empty_host_id_when_its_host_id_header_is_null(string key, string hostIdHeader) + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[hostIdHeader] = null; + attempt.MessageMetadata[key] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata[key], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty), "a header present with no value reads as no host id, as ingestion stores it, so the message copies"); + } + + [Test] + public void RestoreMetadataTypes_leaves_a_document_whose_attempt_list_is_null_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + } + + [Test] + public async Task Reads_a_document_whose_attempt_list_is_null_with_no_StatusChangedAt() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.Null, "the translator skips a message with no attempts, so the source passes it through rather than halting the category"); + } + + [Test] + public async Task Hands_over_the_metadata_ingestion_reads_as_the_types_it_reads() + { + var failure = new IngestedFailure { IsSystemMessage = true }; + await SeedFailedMessage(failure.ToFailedMessage()); + await CompleteDatabaseOperation(); + + var metadata = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single().MessageMetadata; + + using (Assert.EnterMultipleScope()) + { + Assert.That(metadata["TimeSent"], Is.TypeOf<DateTime>().And.EqualTo(failure.TimeSent)); + Assert.That(((DateTime)metadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(metadata["SendingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(failure.SendingEndpoint.HostId)); + Assert.That(metadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(failure.ReceivingEndpoint.HostId)); + Assert.That(metadata["MessageType"], Is.TypeOf<string>().And.EqualTo(failure.MessageType)); + Assert.That(metadata["ConversationId"], Is.TypeOf<string>().And.EqualTo(failure.ConversationId)); + Assert.That(metadata["IsSystemMessage"], Is.TypeOf<bool>().And.True, "the translator reads it with 'is true', so any other type reads as false"); + } + } + + [Test] + public async Task Restores_a_TimeSent_stored_as_text_and_an_endpoint_stored_without_its_type() + { + var document = new IngestedFailure().ToFailedMessage(); + var stored = document.ProcessingAttempts.Single(); + stored.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + stored.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00.0000000Z"; + stored.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + await SeedFailedMessage(document); + await CompleteDatabaseOperation(); + + var attempt = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempt.MessageMetadata["TimeSent"], Is.TypeOf<DateTime>().And.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc))); + Assert.That(((DateTime)attempt.MessageMetadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.Name)).EqualTo("Sales"), "rebuilt from the headers, not converted from the stored dictionary"); + Assert.That(((EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]).Host, Is.EqualTo("MACHINE")); + } + } + + [Test] + public async Task Passes_a_TimeSent_that_is_not_a_date_through_as_stored_and_gives_endpoints_whose_host_id_is_not_a_GUID_the_empty_host_id() + { + var document = SenderAndReceiverHeadersFailure(); + var stored = document.ProcessingAttempts.Single(); + stored.Headers[NServiceBus.Headers.HostId] = "not-a-guid"; + stored.Headers[NServiceBus.Headers.OriginatingHostId] = "not-a-guid-either"; + stored.MessageMetadata["TimeSent"] = "garbage"; + stored.MessageMetadata["SendingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored sender" }; + stored.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored receiver" }; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var metadata = ((FailedMessage)(await ReadAll()).Single().Document).ProcessingAttempts.Single().MessageMetadata; + + using (Assert.EnterMultipleScope()) + { + Assert.That(metadata["TimeSent"], Is.EqualTo("garbage"), "the target skips the document as malformed, so one bad document does not halt the category"); + Assert.That(metadata["SendingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty)); + Assert.That(metadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(Guid.Empty)); + } + } + + [Test] + public void RestoreMetadataTypes_parses_text_as_UTC_and_rebuilds_untyped_endpoints_from_the_headers() + { + var document = new IngestedFailure().ToFailedMessage(numberOfAttempts: 2); + var sendingHostId = Guid.NewGuid(); + + foreach (var attempt in document.ProcessingAttempts) + { + attempt.Headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + attempt.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00.1234567"; + attempt.Headers[NServiceBus.Headers.OriginatingEndpoint] = "Ordering"; + attempt.Headers["NServiceBus.OriginatingMachine"] = "SENDER"; + attempt.Headers[NServiceBus.Headers.OriginatingHostId] = sendingHostId.ToString(); + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = "not an endpoint"; + } + + FailedMessageStream.RestoreMetadataTypes(document); + + var newest = document.ProcessingAttempts[^1]; + var older = document.ProcessingAttempts[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(newest.MessageMetadata["TimeSent"], Is.TypeOf<DateTime>().And.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc).AddTicks(1234567))); + Assert.That(((DateTime)newest.MessageMetadata["TimeSent"]).Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(newest.MessageMetadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.Name)).EqualTo("Sales")); + var sending = (EndpointDetails)newest.MessageMetadata["SendingEndpoint"]; + Assert.That(sending, Is.Not.Null); + Assert.That(sending.Name, Is.EqualTo("Ordering")); + Assert.That(sending.Host, Is.EqualTo("SENDER")); + Assert.That(sending.HostId, Is.EqualTo(sendingHostId)); + Assert.That(older.MessageMetadata["TimeSent"], Is.EqualTo("2026-07-22T09:59:00.1234567"), "the translator reads only the newest attempt, so an older one is left as stored"); + } + } + + [Test] + public void RestoreMetadataTypes_leaves_a_TimeSent_text_that_is_not_a_date_as_stored() + { + var document = new IngestedFailure().ToFailedMessage(); + var metadata = document.ProcessingAttempts.Single().MessageMetadata; + metadata["TimeSent"] = "garbage"; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(metadata["TimeSent"], Is.EqualTo("garbage"), "the target skips the document as malformed, so one bad document does not halt the category"); + } + + [TestCase("SendingEndpoint", NServiceBus.Headers.OriginatingHostId)] + [TestCase("ReceivingEndpoint", NServiceBus.Headers.HostId)] + public void RestoreMetadataTypes_gives_an_endpoint_the_empty_host_id_when_its_host_id_header_is_not_a_GUID(string key, string hostIdHeader) + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[hostIdHeader] = "not-a-guid"; + attempt.MessageMetadata[key] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata[key]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.HostId, Is.EqualTo(Guid.Empty), "ingestion stores an unparseable host id as the empty GUID, so the message copies rather than being skipped"); + Assert.That(endpoint.Name, Is.EqualTo(key == "SendingEndpoint" ? "Ordering" : "Sales")); + Assert.That(endpoint.Host, Is.EqualTo(key == "SendingEndpoint" ? "SENDER" : "MACHINE")); + } + } + + [Test] + public void RestoreMetadataTypes_builds_a_receiving_endpoint_from_the_failed_queue_when_the_headers_name_none() + { + var document = new IngestedFailure { QueueAddress = "Sales@MACHINE" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.Name, Is.EqualTo("Sales")); + Assert.That(endpoint.Host, Is.EqualTo("MACHINE")); + Assert.That(endpoint.HostId, Is.EqualTo(new Guid("f0c10bfe-aba7-6ceb-d333-ab938515451c")), "ingestion derives the host id from the name and host when it has none"); + } + } + + [Test] + public void RestoreMetadataTypes_gives_a_receiving_endpoint_built_from_a_failed_queue_with_no_machine_the_unknown_host() + { + var document = new IngestedFailure { QueueAddress = "Sales" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + var endpoint = (EndpointDetails)attempt.MessageMetadata["ReceivingEndpoint"]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoint.Name, Is.EqualTo("Sales")); + Assert.That(endpoint.Host, Is.EqualTo("unknown")); + Assert.That(endpoint.HostId, Is.EqualTo(new Guid("862257db-407e-ec11-92cc-05ab0c3f7325"))); + } + } + + [Test] + public void RestoreMetadataTypes_leaves_a_newest_attempt_with_no_metadata_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.MessageMetadata = null; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + Assert.That(attempt.MessageMetadata, Is.Null, "the translator skips an attempt with no metadata, so an empty stand-in would hide the fault"); + } + + [Test] + public void RestoreMetadataTypes_leaves_an_untyped_endpoint_on_an_attempt_with_no_headers_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers = null; + attempt.MessageMetadata["TimeSent"] = "2026-07-22T09:59:00Z"; + var stored = new Dictionary<string, object> { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = stored; + + FailedMessageStream.RestoreMetadataTypes(document); + + using (Assert.EnterMultipleScope()) + { + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.SameAs(stored)); + Assert.That(attempt.MessageMetadata["TimeSent"], Is.EqualTo(new DateTime(2026, 7, 22, 9, 59, 0, DateTimeKind.Utc)), "TimeSent does not come from the headers, so it is still restored"); + } + } + + [Test] + public void RestoreMetadataTypes_lets_an_unexpected_fault_through() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers = new HeadersThatCannotBeRead(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + Assert.Throws<InvalidOperationException>(() => FailedMessageStream.RestoreMetadataTypes(document), "a fault the data cannot explain is a defect that must halt the category, not be swallowed as one bad document"); + } + + [TestCase(0)] + [TestCase(1)] + public async Task Reads_a_document_with_a_null_attempt_and_resolves_StatusChangedAt_from_the_others(int nullAttempt) + { + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(numberOfAttempts: 2); + var expected = document.ProcessingAttempts[1 - nullAttempt].AttemptedAt; + document.ProcessingAttempts[nullAttempt] = null; + await SeedFailedMessage(document); + await DocumentStore.WaitForIndexingAllowingErrorsAsync(TestContext.CurrentContext.CancellationToken); + + var row = (await ReadAll()).Single(); + + Assert.That(row.Metadata[MigrationMetadataKeys.StatusChangedAt], Is.EqualTo(expected), "a null attempt is a data fault for the translator to skip, not a reason to halt the category"); + } + + [Test] + public void RestoreMetadataTypes_ignores_a_TimeSent_that_is_not_a_date_on_an_attempt_the_mapper_never_reads() + { + var document = new IngestedFailure().ToFailedMessage(numberOfAttempts: 2); + document.ProcessingAttempts[0].MessageMetadata["TimeSent"] = "garbage"; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document), "a fault in data that is never copied must not halt the category"); + } + + [Test] + public void RestoreMetadataTypes_leaves_a_document_with_no_attempts_for_the_mapper_to_skip() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document)); + } + + [Test] + public void RestoreMetadataTypes_keeps_the_receiving_host_id_header_on_the_named_header_path() + { + var hostId = Guid.NewGuid(); + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.HostId] = hostId.ToString(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(hostId)); + } + + [Test] + public void RestoreMetadataTypes_keeps_the_receiving_host_id_header_on_the_failed_queue_path() + { + var hostId = Guid.NewGuid(); + var document = new IngestedFailure { QueueAddress = "Sales@MACHINE" }.ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers.Remove(NServiceBus.Headers.ProcessingEndpoint); + attempt.Headers[NServiceBus.Headers.HostId] = hostId.ToString(); + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.HostId)).EqualTo(hostId), "a host id the headers carry is not replaced by one derived from the name and host"); + } + + [Test] + public void RestoreMetadataTypes_prefers_the_host_display_name_to_the_processing_machine() + { + var document = SenderAndReceiverHeadersFailure(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.HostDisplayName] = "DISPLAY"; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.TypeOf<EndpointDetails>().And.Property(nameof(EndpointDetails.Host)).EqualTo("DISPLAY")); + } + + [Test] + public void RestoreMetadataTypes_does_not_throw_when_every_header_it_reads_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + + foreach (var header in new[] + { + NServiceBus.Headers.OriginatingEndpoint, "NServiceBus.OriginatingMachine", NServiceBus.Headers.OriginatingHostId, NServiceBus.Headers.OriginatingAddress, + NServiceBus.Headers.ProcessingEndpoint, NServiceBus.Headers.HostDisplayName, "NServiceBus.ProcessingMachine", NServiceBus.Headers.HostId, "NServiceBus.FailedQ" + }) + { + attempt.Headers[header] = null; + } + + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + Assert.DoesNotThrow(() => FailedMessageStream.RestoreMetadataTypes(document), "with the catch gone, a throw on any header value would halt the whole category"); + } + + [Test] + public void RestoreMetadataTypes_restores_no_sending_endpoint_when_its_address_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers[NServiceBus.Headers.OriginatingAddress] = null; + attempt.MessageMetadata["SendingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["SendingEndpoint"], Is.Null, "ingestion stores no endpoint when the address it falls back to has no value"); + } + + [Test] + public void RestoreMetadataTypes_restores_no_receiving_endpoint_when_its_failed_queue_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var attempt = document.ProcessingAttempts.Single(); + attempt.Headers["NServiceBus.FailedQ"] = null; + attempt.MessageMetadata["ReceivingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + FailedMessageStream.RestoreMetadataTypes(document); + + Assert.That(attempt.MessageMetadata["ReceivingEndpoint"], Is.Null, "ingestion stores no endpoint when the failed queue it falls back to has no value"); + } + + static FailedMessage SenderAndReceiverHeadersFailure() + { + var document = new IngestedFailure().ToFailedMessage(); + var headers = document.ProcessingAttempts.Single().Headers; + headers[NServiceBus.Headers.OriginatingEndpoint] = "Ordering"; + headers["NServiceBus.OriginatingMachine"] = "SENDER"; + headers["NServiceBus.ProcessingMachine"] = "MACHINE"; + return document; + } + + // The index cannot turn this TimeSent into a date, so RavenDB fails to index the document. + static FailedMessage FailedMessageWithATimeSentThatIsNotADate() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.Unresolved); + document.ProcessingAttempts.Single().MessageMetadata["TimeSent"] = "garbage"; + return document; + } + + async Task UseCorax() + { + await DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(IndexName), TestContext.CurrentContext.CancellationToken); + await new FailedMessageViewIndex { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } }.ExecuteAsync(DocumentStore, token: TestContext.CurrentContext.CancellationToken); + + var engine = (await DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(IndexName), TestContext.CurrentContext.CancellationToken)).SearchEngineType; + Assert.That(engine, Is.EqualTo(SearchEngineType.Corax), "the test proves nothing about Corax if the index still runs on Lucene"); + } + + async Task<IndexState> IndexStateNow() => + (await DocumentStore.Maintenance.SendAsync(new GetStatisticsOperation(), TestContext.CurrentContext.CancellationToken)).Indexes.Single(index => index.Name == IndexName).State; + + async Task<List<MigrationRow>> ReadAll() + { + await using var source = await OpenMigrationSource(); + + return (await CollectBatches(source, Category)).SelectMany(batch => batch.Rows).ToList(); + } + + // RavenDB streams in id order, so statuses are handed out by sorted id rather than by seeding order. + async Task<List<string>> SeedInIdOrder(params FailedMessageStatus[] statuses) + { + var messages = statuses.Select(_ => new IngestedFailure().ToFailedMessage()) + .OrderBy(message => FailedMessageIdGenerator.MakeDocumentId(message.UniqueMessageId), StringComparer.Ordinal) + .ToList(); + var ids = new List<string>(messages.Count); + + for (var position = 0; position < messages.Count; position++) + { + messages[position].Status = statuses[position]; + ids.Add((await SeedFailedMessage(messages[position])).Id); + } + + await CompleteDatabaseOperation(); + + return ids; + } + + // Takes the walk limit as a parameter so a test can reach it without seeding the 10,000 documents of RavenDocumentStream.MaxDocumentsWalkedPerBatch. + async Task<List<MigrationBatch>> StreamWithWalkLimit(int batchSize, int maxDocumentsWalkedPerBatch) + { + var settings = (RavenPersisterSettings)PersistenceSettings; + await using var lifecycle = new RavenReadOnlySourceLifecycle(settings, new SettingsRootNamespace("ServiceControl")); + await lifecycle.Open(TestContext.CurrentContext.CancellationToken); + + var batches = new List<MigrationBatch>(); + + await foreach (var batch in RavenDocumentStream.ByPrefix( + lifecycle, Category.Id, settings.DatabaseName, $"{FailedMessageIdGenerator.CollectionName}/", null, batchSize, maxDocumentsWalkedPerBatch, + new FailedMessageStream(lifecycle).Project(MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses), TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + static readonly string IndexName = new FailedMessageViewIndex().IndexName; + + // The parsers read headers through IReadOnlyDictionary, so re-implementing its lookup makes every header read fail. + sealed class HeadersThatCannotBeRead : Dictionary<string, string>, IReadOnlyDictionary<string, string> + { + bool IReadOnlyDictionary<string, string>.TryGetValue(string key, out string value) => + throw new InvalidOperationException("The headers cannot be read."); + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs b/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs index 904bc59d15..5516e0d439 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/Infrastructure/RavenIndexAwaiter.cs @@ -20,7 +20,15 @@ public static class RavenIndexAwaiter public static Task WaitForIndexingAsync(this IDocumentStore store, CancellationToken cancellationToken = default) => store.WaitForIndexingAsync(DefaultTimeout, cancellationToken); - public static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, CancellationToken cancellationToken = default) + public static Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, CancellationToken cancellationToken = default) => + store.WaitForIndexingAsync(timeout, allowIndexErrors: false, cancellationToken); + + // For a test that stores a document an index cannot map on purpose. An index in the error state counts as done, + // because it never catches up. + public static Task WaitForIndexingAllowingErrorsAsync(this IDocumentStore store, CancellationToken cancellationToken = default) => + store.WaitForIndexingAsync(DefaultTimeout, allowIndexErrors: true, cancellationToken); + + static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpan timeout, bool allowIndexErrors, CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); IndexStats[] stats; @@ -31,14 +39,14 @@ public static async Task WaitForIndexingAsync(this IDocumentStore store, TimeSpa // An index in the error state never becomes up to date. Fail now with the errors instead of after the timeout. var erroredIndexes = stats.Where(i => i.State == IndexState.Error || i.ErrorsCount > 0).Select(i => i.Name).ToArray(); - if (erroredIndexes.Length > 0) + if (erroredIndexes.Length > 0 && !allowIndexErrors) { var errors = await store.Maintenance.SendAsync(new GetIndexErrorsOperation(erroredIndexes), cancellationToken); var details = errors.SelectMany(e => e.Errors.Select(x => $"{e.Name}: {x.Action} {x.Document} {x.Error}")); Assert.Fail($"Indexes have errors:{Environment.NewLine}{string.Join(Environment.NewLine, details)}"); } - if (stats.All(i => !i.IsStale)) + if (stats.All(i => !i.IsStale || (allowIndexErrors && i.State == IndexState.Error))) { return; } diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs index b6610b3ec8..e318f8b450 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs @@ -52,6 +52,8 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) persistence.AddInstaller(hostBuilder.Services); } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public async Task PostSetup(IHost host) { DocumentStore = await host.Services.GetRequiredService<IRavenDocumentStoreProvider>().GetDocumentStore(); diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs new file mode 100644 index 0000000000..b611b22fc6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -0,0 +1,169 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class EndpointSettingsKeyCollationTests : PersistenceTestBase +{ + readonly TargetWarnings targetWarnings = new(); + + public EndpointSettingsKeyCollationTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetWarnings); + + [Test] + public async Task A_case_merge_inside_one_batch_logs_both_names_and_the_one_kept() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/2 ").And.Contains("kept 'Sales'"), "the operator finds the dropped row by this source id"); + Assert.That(targetWarnings.Messages, Has.None.StartsWith("Merged EndpointSettings/1 "), "the kept row was not dropped"); + } + } + + [Test] + public async Task A_case_merge_with_a_row_an_earlier_batch_wrote_is_logged_and_a_row_written_again_is_not() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales")); + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + Assert.That(targetWarnings.Messages, Has.Exactly(1).Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + + [Test] + public async Task A_case_merge_keeps_the_setting_whose_name_a_known_endpoint_uses_even_when_it_comes_second() + { + await OpenACaseInsensitiveTarget("Sales"); + + await Write(("EndpointSettings/1", "sales", true), ("EndpointSettings/2", "Sales", false)); + + List<EndpointSettingsEntity> stored; + using (var scope = ServiceProvider.CreateScope()) + { + stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().EndpointSettings.AsNoTracking().ToListAsync(); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Select(setting => (setting.Name, setting.TrackInstances)), Is.EqualTo(new[] { ("Sales", false) }), "the heartbeat sync deletes a setting no known endpoint has the exact name of, so the orphan must be the one dropped"); + Assert.That(targetWarnings.Messages, Has.Exactly(1).StartsWith("Merged EndpointSettings/1 ").And.Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + } + + async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // Set on the column, because the merge follows the column's collation and the test server's default can be either. + var ignoreCase = $""" + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE Latin1_General_CI_AS NOT NULL; + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await dbContext.Database.ExecuteSqlRawAsync(ignoreCase); + } + + foreach (var name in knownEndpointNames) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + await ServiceProvider.GetRequiredService<IMigrationTarget>().Open(); + } + + Task Write(params (string SourceId, string Name)[] rows) => Write([.. rows.Select(row => (row.SourceId, row.Name, true))]); + + async Task Write(params (string SourceId, string Name, bool TrackInstances)[] rows) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = row.TrackInstances }, new Dictionary<string, object>()))], + rows[^1].SourceId); + var checkpoint = await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Read(category.Id) + ?? new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + await ServiceProvider.GetRequiredService<IMigrationTarget>().Write(category, batch, checkpoint); + } + + [Test] + public async Task The_key_columns_own_collation_decides_a_merge_when_the_database_default_disagrees() + { + bool databaseIgnoresCase; + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var database = dbContext.Database; + + databaseIgnoresCase = await database + .SqlQuery<int>($"SELECT CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 AS [Value]") + .SingleAsync() == 1; + + // Named from the model, because the table sits in the configured schema when the persister has one. + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // The column is given the opposite of the database default, because a test where the two agree cannot show which one decided. + var recollate = $""" + DECLARE @columnCollation sysname = IIF(CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 = 1, N'Latin1_General_CS_AS', N'Latin1_General_CI_AS'); + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + EXEC (N'ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE ' + @columnCollation + N' NOT NULL'); + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await database.ExecuteSqlRawAsync(recollate); + } + + foreach (var name in new[] { "Sales", "sales" }) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); + await target.Open(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [ + new MigrationRow("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }, new Dictionary<string, object>()), + new MigrationRow("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false }, new Dictionary<string, object>()) + ], + "EndpointSettings/2"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(databaseIgnoresCase ? 2 : 1), "the column's own collation decides: two keys where it respects case, one where it ignores case, whatever the database default says"); + Assert.That( + ServiceProvider.GetRequiredService<IMigrationSqlDialect>().KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)).Equals("Sales", "sales"), + Is.EqualTo(!databaseIgnoresCase), + "the dry run predicts merges with this comparer, so it follows the same column the statement does"); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs new file mode 100644 index 0000000000..c10838be08 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/LicensingThroughputKeyCollationTests.cs @@ -0,0 +1,138 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingThroughputKeyCollationTests : PersistenceTestBase +{ + [Test] + public async Task Days_of_endpoints_an_accent_insensitive_key_treats_as_one_are_added_within_a_batch_and_across_batches() + { + await OpenAnAccentInsensitiveTarget(); + await LicensingDataStore.SaveEndpoint(new Endpoint("Café", ThroughputSource.Broker) { SanitizedName = "cafe" }); + var day = new DateOnly(2026, 1, 1); + + var first = await Write( + new MigrationCheckpoint(MigrationCategoryIds.LicensingThroughput, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null), + ("Café/Broker|2026-01-01", "Café", day, 100), + ("Cafe/Broker|2026-01-01", "Cafe", day, 250)); + await Write(first.Saved, ("CAFE/Broker|2026-01-01", "CAFE", day, 50)); + + long[] stored; + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().LicensingEndpointThroughput.Select(row => row.MessageCount).ToArrayAsync(); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(1), "the column's collation makes the two names one day, which the statement writes once"); + Assert.That(first.AlreadyPresent, Is.EqualTo(1), "the day folded into the other is counted as already present and summed into the kept row"); + Assert.That(stored, Is.EqualTo(new[] { 400L }), "every count for the day is added under the key column's own collation, whichever batch it came in"); + } + } + + [Test] + public async Task Endpoints_an_accent_insensitive_key_treats_as_one_are_reported_as_a_merge() + { + await OpenAnAccentInsensitiveTarget(); + var batch = new MigrationBatch( + [ + new MigrationRow("Café/Broker", new Endpoint("Café", ThroughputSource.Broker) { SanitizedName = "cafe" }, new Dictionary<string, object>()), + new MigrationRow("Cafe/Broker", new Endpoint("Cafe", ThroughputSource.Broker) { SanitizedName = "cafe" }, new Dictionary<string, object>()) + ], + "Cafe/Broker"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await ServiceProvider.GetRequiredService<IMigrationTarget>().Write( + MigrationCategoryRegistry.Find(MigrationCategoryIds.LicensingEndpoints)!, + batch, + new MigrationCheckpoint(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "Café/Broker" }), "the copy keeps the endpoint whose source id sorts first, and the column folds the accent, so the other is a merge"); + Assert.That(merges.Single().Detail, Does.Contain("kept 'cafe', 'Broker' and dropped the row for 'café', 'Broker'")); + Assert.That((result.Copied, result.AlreadyPresent, result.Merged), Is.EqualTo((1, 1, 1)), "every row the insert drops on a first copy is a merge the log names"); + } + } + + [TestCase("Café", "Cafe")] + [TestCase("cafe", "cafe")] + [TestCase("CAFE", "cafe")] + [TestCase("cafe ", "cafe")] + [TestCase("cafe", "cafes")] + public async Task The_key_comparer_folds_exactly_what_the_columns_collation_folds(string first, string second) + { + await OpenAnAccentInsensitiveTarget(); + bool sqlServerFolds; + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + sqlServerFolds = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database + .SqlQuery<int>($"SELECT CASE WHEN {first} = {second} COLLATE Latin1_General_CI_AI THEN 1 ELSE 0 END AS [Value]") + .SingleAsync() == 1; + } + + var comparer = ServiceProvider.GetRequiredService<IMigrationSqlDialect>().KeyComparer(typeof(LicensingEndpointEntity), nameof(LicensingEndpointEntity.NormalizedName)); + + Assert.That(comparer.Equals(first, second), Is.EqualTo(sqlServerFolds), $"the merge lines and the dry run predict with this comparer, so it must agree with SQL Server on '{first}' and '{second}'"); + } + + // Set on both tables, because the writer looks up each day's endpoint by name before it writes, and the test server's default respects accents. + async Task OpenAnAccentInsensitiveTarget() + { + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var endpoints = TableOf<LicensingEndpointEntity>(dbContext); + var throughput = TableOf<LicensingEndpointThroughputEntity>(dbContext); + + var ignoreAccents = $""" + ALTER TABLE {throughput} DROP CONSTRAINT [FK_LicensingEndpointThroughput_LicensingEndpoints_NormalizedName_ThroughputSource]; + ALTER TABLE {throughput} DROP CONSTRAINT [PK_LicensingEndpointThroughput]; + ALTER TABLE {endpoints} DROP CONSTRAINT [PK_LicensingEndpoints]; + ALTER TABLE {endpoints} ALTER COLUMN [NormalizedName] nvarchar(300) COLLATE Latin1_General_CI_AI NOT NULL; + ALTER TABLE {throughput} ALTER COLUMN [NormalizedName] nvarchar(300) COLLATE Latin1_General_CI_AI NOT NULL; + ALTER TABLE {endpoints} ADD CONSTRAINT [PK_LicensingEndpoints] PRIMARY KEY ([NormalizedName], [ThroughputSource]); + ALTER TABLE {throughput} ADD CONSTRAINT [PK_LicensingEndpointThroughput] PRIMARY KEY ([NormalizedName], [ThroughputSource], [DateUtc]); + ALTER TABLE {throughput} ADD CONSTRAINT [FK_LicensingEndpointThroughput_LicensingEndpoints_NormalizedName_ThroughputSource] + FOREIGN KEY ([NormalizedName], [ThroughputSource]) REFERENCES {endpoints} ([NormalizedName], [ThroughputSource]) ON DELETE CASCADE; + """; + + await dbContext.Database.ExecuteSqlRawAsync(ignoreAccents); + } + + await ServiceProvider.GetRequiredService<IMigrationTarget>().Open(); + } + + Task<MigrationWriteResult> Write(MigrationCheckpoint checkpointToExtend, params (string SourceId, string Name, DateOnly Day, long Count)[] days) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.LicensingThroughput); + var batch = new MigrationBatch( + [.. days.Select(day => new MigrationRow(day.SourceId, new ThroughputDayRow(new EndpointIdentifier(day.Name, ThroughputSource.Broker), day.Day, day.Count), new Dictionary<string, object>()))], + days[^1].SourceId); + + return ServiceProvider.GetRequiredService<IMigrationTarget>().Write(category, batch, checkpointToExtend); + } + + // Named from the model, because the table sits in the configured schema when the persister has one. + static string TableOf<TEntity>(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity)); + return dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs new file mode 100644 index 0000000000..f1914f23b1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationGroupInsertSeekTests.cs @@ -0,0 +1,69 @@ +// ReSharper disable once CheckNamespace +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationGroupInsertSeekTests : PersistenceTestBase +{ + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task The_group_insert_seeks_existing_rows_by_key_while_the_group_table_is_small() + { + List<FailedMessage.FailureGroup> sharedGroups = + [ + .. new[] { "Exception Type and Stack Trace", "Message Type", "Endpoint Address", "Endpoint Instance", "Endpoint Name" } + .Select(type => new FailedMessage.FailureGroup { Id = Guid.NewGuid().ToString(), Title = $"{type} title", Type = type }) + ]; + var rows = Enumerable.Range(0, 74) + .Select(_ => new IngestedFailure { Groups = sharedGroups }.ToFailedMessage()) + .Select(document => new MigrationRow($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary<string, object> + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }, new MigrationBody("<order>1</order>"u8.ToArray(), "text/xml"))) + .ToList(); + + var result = await Target.Write(FailedMessagesCategory, new MigrationBatch(rows, rows[^1].SourceId), new MigrationCheckpoint(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using var scope = ServiceProvider.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(74)); + Assert.That(await GroupTableReadsInCachedPlan(db), Is.Not.Empty.And.None.Contains("Scan"), "a scan reads every row of an index and HOLDLOCK locks each one, which FORCESEEK prevents"); + } + } + + // Seek and scan operators both hold an IndexScan element, so the RelOp above it says which one it is. + static Task<List<string>> GroupTableReadsInCachedPlan(ServiceControlDbContext db) + { + var statement = $"MERGE {MigrationInsert<FailedMessageGroupEntity>.For(db).Table} "; + return db.Database.SqlQuery<string>($""" + WITH XMLNAMESPACES (DEFAULT 'http://schemas.microsoft.com/sqlserver/2004/07/showplan') + SELECT o.n.value('../../@PhysicalOp', 'nvarchar(60)') + ' ' + o.n.value('@Index', 'nvarchar(260)') AS [Value] + FROM sys.dm_exec_cached_plans AS p + CROSS APPLY sys.dm_exec_sql_text(p.plan_handle) AS t + CROSS APPLY sys.dm_exec_query_plan(p.plan_handle) AS q + CROSS APPLY q.query_plan.nodes('//RelOp/IndexScan/Object[@Table="[FailedMessageGroups]"]') AS o(n) + WHERE t.dbid = DB_ID() AND CHARINDEX({statement}, t.text) > 0 + """).ToListAsync(); + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs new file mode 100644 index 0000000000..f6135cc924 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationPlanReuseTests.cs @@ -0,0 +1,78 @@ +// ReSharper disable once CheckNamespace +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationPlanReuseTests : PersistenceTestBase +{ + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task Batches_whose_unbounded_text_sits_either_side_of_4000_characters_share_one_cached_plan() + { + var shortHeaders = new IngestedFailure(); + shortHeaders.Headers["Padding"] = new string('h', 3_000); + var longHeaders = new IngestedFailure { Groups = [new() { Id = Guid.NewGuid().ToString(), Title = new string('t', 5_000), Type = "Message Type" }] }; + longHeaders.Headers["Padding"] = new string('h', 5_000); + + await Target.Write(FailedMessagesCategory, BatchOf(shortHeaders.ToFailedMessage()), Checkpoint()); + var saved = await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var result = await Target.Write(FailedMessagesCategory, BatchOf(longHeaders.ToFailedMessage()), saved); + + using var scope = ServiceProvider.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var headersJsonLengths = await db.FailedMessages.AsNoTracking().Select(message => message.HeadersJson.Length).ToListAsync(); + var titleLengths = await db.FailedMessageGroups.AsNoTracking().Select(group => group.Title.Length).ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(headersJsonLengths, Has.Exactly(1).LessThan(4_000).And.Exactly(1).GreaterThan(4_000), "one HeadersJson must fit nvarchar(4000) and the other must not. If the long one is exactly 4000, the insert cut it short"); + Assert.That(titleLengths, Does.Contain(5_000), "the long group Title must be stored whole"); + Assert.That(await PlanUseCountsFor<FailedMessageEntity>(db), Is.EqualTo(new[] { 2 }), "short and long text must reuse one plan, because each new parameter declaration list compiles its own"); + Assert.That(await PlanUseCountsFor<FailedMessageGroupEntity>(db), Is.EqualTo(new[] { 2 }), "the group Title column is unbounded too"); + } + } + + // Scoped to this test's schema, because the suite's other fixtures share the server and its plan cache. + static Task<List<int>> PlanUseCountsFor<TEntity>(ServiceControlDbContext db) where TEntity : class + { + var statement = $"MERGE {MigrationInsert<TEntity>.For(db).Table} "; + return db.Database.SqlQuery<int>($""" + SELECT p.usecounts AS [Value] + FROM sys.dm_exec_cached_plans AS p + CROSS APPLY sys.dm_exec_sql_text(p.plan_handle) AS t + WHERE t.dbid = DB_ID() AND CHARINDEX({statement}, t.text) > 0 + """).ToListAsync(); + } + + static MigrationBatch BatchOf(FailedMessage document) + { + var row = new MigrationRow($"FailedMessages/{document.UniqueMessageId}", document, new Dictionary<string, object> + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }, new MigrationBody("<order>1</order>"u8.ToArray(), "text/xml")); + + return new MigrationBatch([row], row.SourceId); + } + + static MigrationCheckpoint Checkpoint() => + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs new file mode 100644 index 0000000000..7f024576bc --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/MigrationValueLengthTests.cs @@ -0,0 +1,70 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationValueLengthTests : PersistenceTestBase +{ + static readonly string OutsideTheBasicPlane = char.ConvertFromUtf32(0x1F600); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + [Test] + public async Task A_value_is_measured_in_utf16_units_as_nvarchar_counts_them() + { + var fits = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(225), HostId = Guid.NewGuid(), Host = "SALES01" } }; + var tooLong = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = Repeat(226), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var batch = new MigrationBatch( + [ + new MigrationRow("KnownEndpoints/1", fits, new Dictionary<string, object>()), + new MigrationRow("KnownEndpoints/2", tooLong, new Dictionary<string, object>()) + ], + "KnownEndpoints/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/2", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "225 code points are 450 UTF-16 units, which an nvarchar(450) column holds"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2" }), "226 code points are 452 UTF-16 units, which would fail the insert"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { Repeat(225) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<KnownEndpointEntity>(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 452, on document 'KnownEndpoints/2'" + }), "the length reported is the one SQL Server counts"); + Assert.That(ServiceProvider.GetRequiredService<IMigrationSqlDialect>().TextLength(Repeat(226)), Is.LessThanOrEqualTo(Repeat(226).Length), "the check skips measuring a value whose UTF-16 length fits, which is safe only while no count exceeds it"); + } + } + + [Test] + public async Task A_value_holding_a_NUL_character_copies() + { + var withNul = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales", HostId = Guid.NewGuid(), Host = "SAL\u0000ES" } }; + var batch = new MigrationBatch([new MigrationRow("KnownEndpoints/1", withNul, new Dictionary<string, object>())], "KnownEndpoints/1"); + + var result = await Target.Write(KnownEndpointsCategory, batch, new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, "KnownEndpoints/1", 0, 0, null, null, null, null, null, null)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "nvarchar stores U+0000, so a skip here would give up a row SQL Server can hold"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Single().EndpointDetails.Host, Is.EqualTo("SAL\u0000ES")); + } + } + + static string Repeat(int codePoints) => string.Concat(Enumerable.Repeat(OutsideTheBasicPlane, codePoints)); +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs index 9ded3182bc..2dab62abe4 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs @@ -50,7 +50,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton<TimeProvider>(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -58,6 +58,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService<IDatabaseMigrator>().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs new file mode 100644 index 0000000000..67d7c9c307 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +class EFMigrationCheckpointStoreTests : PersistenceTestBase +{ + IMigrationCheckpointStore Store => ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + + [Test] + public async Task Read_returns_null_for_a_category_that_has_never_run() + { + Assert.That(await Store.Read("EndpointSettings"), Is.Null); + } + + [Test] + public async Task Upsert_then_Read_round_trips_every_field() + { + var checkpoint = new MigrationCheckpoint( + "EndpointSettings", MigrationCategoryState.CompleteWithErrors, "cursor-1", 5, 2, 40, + new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable", AlreadyPresentCount: 33, + StartedWindowSeconds: (long)TimeSpan.FromDays(14).TotalSeconds); + + var saved = await Store.Upsert(checkpoint); + var stored = await Store.Read("EndpointSettings"); + + // Record equality compares SkipReasons by reference, and a round-tripped dictionary is a new instance. + using (Assert.EnterMultipleScope()) + { + Assert.That(stored! with { SkipReasons = null, Version = 0 }, Is.EqualTo(checkpoint with { SkipReasons = null })); + Assert.That(stored.SkipReasons, Is.EquivalentTo(checkpoint.SkipReasons)); + Assert.That(stored.Version, Is.EqualTo(1), "the first save leaves the row at version 1"); + // The engine saves from what Upsert hands back, so a returned version that is not the stored one + // makes the next save conflict against a row nothing else touched. + Assert.That(saved.Version, Is.EqualTo(stored.Version)); + } + } + + [Test] + public async Task A_second_Upsert_for_the_same_category_updates_rather_than_duplicates() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 0, 40, null, Now, Now, null, null)); + var saved = await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "cursor-2", 9, 1, 40, null, Now, Now, Now, null, Version: 1)); + + var all = await Store.ReadAll(); + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(all, Has.Count.EqualTo(1)); + Assert.That(saved.Version, Is.EqualTo(stored!.Version), "an update hands back the version it landed on, not the one it was saved from"); + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(9)); + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + } + } + + [Test] + public async Task A_checkpoint_saved_as_Abandoned_stores_its_counts_its_reasons_and_when_it_was_abandoned() + { + var reasons = new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 400 }; + var halted = Now.AddMinutes(-5); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Halted, "g-1", 12, 400, 412, reasons, Now, Now, halted, "body storage unavailable")); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Abandoned, "g-1", 12, 400, 412, reasons, Now, Now, Now, "body storage unavailable", Version: 1)); + + var stored = await Store.Read("GroupComments"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Abandoned)); + // State says which terminal state was reached and SettledAt says when, so abandoning a halted + // category moves the one timestamp on rather than filling a second column beside it. + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Not.EqualTo(halted)); + Assert.That(stored.SkippedCount, Is.EqualTo(400)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 400 })); + Assert.That(stored.LastError, Is.EqualTo("body storage unavailable")); + } + } + + [Test] + public async Task A_restart_after_a_halt_clears_the_fields_the_previous_save_set() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Halted, "cursor-1", 5, 2, 40, + new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable")); + + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 2, 40, + null, Now, Now, null, null, Version: 1)); + + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.SettledAt, Is.Null, "a resumed category must not look finished"); + Assert.That(stored.LastError, Is.Null); + Assert.That(stored.SkipReasons, Is.Null); + } + } + + [Test] + public void An_upsert_for_a_category_that_has_never_run_is_refused_when_it_carries_a_version() + { + var conflict = Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Store.Upsert( + new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 1, 0, 40, null, Now, Now, null, null, Version: 1))); + + Assert.That(conflict!.Message, Does.Contain("EndpointSettings")); + } + + [Test] + public async Task ReadAll_returns_every_category_that_has_run() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "e-1", 1, 0, 1, null, Now, Now, Now, null)); + await Store.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.InProgress, "k-1", 4, 0, 9, null, Now, Now, null, null)); + + var all = await Store.ReadAll(); + + Assert.That(all.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "EndpointSettings", "KnownEndpoints" })); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs new file mode 100644 index 0000000000..b23deff4cf --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomCheckRowMapperTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class CustomCheckRowMapperTests : PersistenceTestBase +{ + [Test] + public void Sets_every_mapped_property_of_a_custom_check() + { + var entity = CustomCheckRowMapper.Map(new CustomCheckDetail + { + OriginatingEndpoint = new EndpointDetails { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }, + CustomCheckId = "Sales.HealthCheck", + Category = "Ping", + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + // Passing, because Status.Fail is the enum's zero and would read as unset. + HasFailed = false, + FailureReason = "recovered after a timeout" + }); + + using var scope = ServiceProvider.CreateScope(); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs new file mode 100644 index 0000000000..4927d099d0 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/CustomChecksMigrationTargetTests.cs @@ -0,0 +1,142 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Contracts.CustomChecks; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; + +class CustomChecksMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; + + static readonly EndpointDetails Endpoint = new() { Name = "Sales", Host = "MACHINE", HostId = Guid.NewGuid() }; + + [Test] + public async Task A_check_the_target_already_holds_is_counted_as_already_present_and_keeps_its_stored_status() + { + var first = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: false))), CheckpointAfter("CustomChecks/1")); + var second = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: true))), first.Saved); + + var stored = (await CustomChecks.GetStats(new PagingInfo())).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.Zero); + Assert.That(second.AlreadyPresent, Is.EqualTo(1), "a replayed batch finds its row already there, where a plain insert would fail the batch on the duplicate key"); + Assert.That(stored.Single().Status, Is.EqualTo(Status.Pass), "the copy never replaces a row the target already holds"); + Assert.That(await Target.Count(Category), Is.EqualTo(1)); + } + } + + [TestCase(nameof(CustomCheckEntity.CustomCheckId))] + [TestCase(nameof(CustomCheckEntity.Category))] + [TestCase(nameof(CustomCheckEntity.OriginatingEndpointHost))] + public async Task A_check_with_no_value_for_a_required_column_is_skipped_as_a_required_value_missing_and_the_rest_copy(string column) + { + var missing = column switch + { + nameof(CustomCheckEntity.CustomCheckId) => Detail(failed: true, checkId: null), + nameof(CustomCheckEntity.Category) => Detail(failed: true, category: null), + _ => Detail(failed: true, endpoint: new EndpointDetails { Name = "Sales", Host = null, HostId = Guid.NewGuid() }) + }; + var batch = BatchOf(("CustomChecks/1", missing), ("CustomChecks/2", Detail(failed: false, checkId: "Sales.DiskCheck"))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a skip copies the rest of the batch, where a NOT NULL failure would fail the whole insert"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 1 })); + Assert.That(details, Is.EqualTo(new[] { $"CustomChecks.{column} is NOT NULL and the document has no value for it" })); + } + } + + [Test] + public async Task A_row_the_source_flags_as_a_storage_check_is_skipped_as_harmless_whatever_its_id() + { + var batch = new MigrationBatch( + [ + new MigrationRow("CustomChecks/1", Detail(failed: false, checkId: "Some.Other.Source.Check"), Flag(true)), + new MigrationRow("CustomChecks/2", Detail(failed: false, checkId: "Sales.DiskCheck"), new Dictionary<string, object>()) + ], "CustomChecks/2"); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new CustomChecksWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.SourceStorageCheck] = 1 })); + Assert.That(result.SkipReasons.Keys.All(reason => reason.IsBenign()), Is.True); + Assert.That(details, Is.EqualTo(new[] { "'Some.Other.Source.Check' under 'Ping' is a check the source's own storage runs about itself, which nothing reports once the instance runs on the target" })); + } + } + + [Test] + public async Task A_row_with_a_RavenDB_persister_id_and_category_but_no_flag_is_copied() + { + var result = await Target.Write(Category, BatchOf(("CustomChecks/1", Detail(failed: false, checkId: "Error Database Index Lag", category: "ServiceControl Health"))), CheckpointAfter("CustomChecks/1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "the target holds no list of ids, so only the source's flag can skip a row"); + Assert.That(result.SkipReasons, Is.Null); + } + } + + [Test] + public void A_flag_that_is_not_a_bool_fails_the_batch_naming_the_row() + { + var batch = new MigrationBatch([new MigrationRow("CustomChecks/1", Detail(failed: false), Flag("true"))], "CustomChecks/1"); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(Category, batch, CheckpointAfter("CustomChecks/1"))); + + Assert.That(exception.Message, Does.Contain("'CustomChecks/1'").And.Contain("System.String")); + } + + [Test] + public async Task A_row_whose_storage_check_flag_is_false_is_copied() + { + var batch = new MigrationBatch([new MigrationRow("CustomChecks/1", Detail(failed: false), Flag(false))], "CustomChecks/1"); + + var result = await Target.Write(Category, batch, CheckpointAfter("CustomChecks/1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons, Is.Null); + } + } + + static Dictionary<string, object> Flag(object value) => new() { [MigrationMetadataKeys.SourceStorageCheck] = value }; + + static CustomCheckDetail Detail(bool failed, string checkId = "Sales.HealthCheck", string category = "Ping", EndpointDetails endpoint = null) => new() + { + OriginatingEndpoint = endpoint ?? Endpoint, + CustomCheckId = checkId, + Category = category, + ReportedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + HasFailed = failed + }; + + static MigrationBatch BatchOf(params (string Id, CustomCheckDetail Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.CustomChecks, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..a6e01f9fc4 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs @@ -0,0 +1,171 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class EndpointSettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + IMigrationCheckpointStore CheckpointStore => ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.EndpointSettings); + + [Test] + public async Task Every_setting_in_a_batch_arrives_with_its_track_instances_value() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false }), + ("EndpointSettings/3", new EndpointSettings { Name = "Shipping", TrackInstances = true })), + CheckpointAfter("EndpointSettings/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(3), "Copied is what the insert added, not the size of the batch"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + Assert.That(await TrackInstancesFor("Billing"), Is.False); + Assert.That(await TrackInstancesFor("Shipping"), Is.True); + } + } + + [Test] + public async Task A_name_already_in_the_target_is_left_alone_and_counted_as_already_present() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf(("EndpointSettings/7", new EndpointSettings { Name = "Sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/7")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a row the target already holds lost nothing, so it is not a skip"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + } + } + + [Test] + public async Task A_setting_whose_endpoint_is_not_known_is_copied_and_left_to_the_heartbeat_sync() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = string.Empty, TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Retired", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero, "whether a setting outlives its endpoint is the heartbeat sync's rule, not the migration's"); + Assert.That(await TrackInstancesFor(string.Empty), Is.True); + Assert.That(await TrackInstancesFor("Retired"), Is.False); + } + } + + [Test] + public async Task A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var batch = BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = new string('s', 450), TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = new string('n', 451), TrackInstances = true }), + ("EndpointSettings/3", new EndpointSettings { Name = "Billing", TrackInstances = false })); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new EndpointSettingsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(EndpointSettingsCategory, batch, CheckpointAfter("EndpointSettings/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "a name exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "EndpointSettings/2" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 1 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(await TrackInstancesFor(new string('s', 450)), Is.True); + Assert.That(await TrackInstancesFor("Billing"), Is.False); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<EndpointSettingsEntity>(ServiceProvider, nameof(EndpointSettingsEntity.Name))} takes at most 450 characters and the value holds 451, on document 'EndpointSettings/2'" + })); + } + } + + [Test] + public async Task A_batch_that_copies_nothing_still_saves_the_cursor() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Billing", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = false }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + var stored = await CheckpointStore.Read(MigrationCategoryIds.EndpointSettings); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(stored.Cursor, Is.EqualTo("EndpointSettings/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + await Target.Write(EndpointSettingsCategory, BatchOf(("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true })), CheckpointAfter("EndpointSettings/1")); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.EndpointSettings.AsNoTracking().SingleAsync()); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); + + async Task<bool> TrackInstancesFor(string name) => + (await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync()).Single(settings => settings.Name == name).TrackInstances; + + [Test] + public async Task Two_names_in_one_batch_differing_only_in_case_are_each_copied_or_already_present() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2), "SQL Server's default collation makes these one key and PostgreSQL makes them two; either way neither row may throw or go uncounted"); + Assert.That(result.Skipped, Is.Zero, "a merge onto a row the batch writes loses nothing the target lacks, and a skip would count toward the halt threshold"); + Assert.That(await Target.Count(EndpointSettingsCategory), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); + Assert.That(await TrackInstancesFor("Sales"), Is.True, "the first row in document-id order wins where the key makes the two one"); + } + } + +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs new file mode 100644 index 0000000000..eb0bcbfab4 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportRowMapperTests.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class FailedErrorImportRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property_of_a_failed_error_import() + { + var import = new FailedErrorImport + { + Id = "native-1", + Message = new FailedTransportMessage { Id = "native-1", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-1" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }; + var failedAt = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + // Inline leaves the flag false and external leaves the column empty, so each call covers the one the other cannot. + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, FailedErrorImportRowMapper.Map(import, failedAt, bodyStoredExternally: false), nameof(FailedErrorImportEntity.BodyStoredExternally)); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, FailedErrorImportRowMapper.Map(import, failedAt, bodyStoredExternally: true), nameof(FailedErrorImportEntity.Body)); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs new file mode 100644 index 0000000000..06024a2453 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedErrorImportsMigrationTargetTests.cs @@ -0,0 +1,133 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class FailedErrorImportsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.FailedErrorImports)!; + + // Not a whole microsecond, because PostgreSQL keeps only microseconds and a re-read must still match what it stored. + static readonly DateTime FirstFailure = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc).AddTicks(1234567); + + [Test] + public async Task A_second_import_of_the_same_message_in_a_batch_is_reported_as_a_merge_naming_the_document_kept() + { + var batch = BatchOf(Row("FailedErrorImports/a", FirstFailure), Row("FailedErrorImports/b", FirstFailure.AddMinutes(5))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/b")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "FailedErrorImports/b" }), "6.18 and earlier stored each failed import attempt under its own document, and both derive one key"); + Assert.That(merges.Single().Detail, Does.Contain("kept the row from 'FailedErrorImports/a'"), "the target logs this line, so it has to name the document it kept"); + Assert.That((result.Copied, result.AlreadyPresent), Is.EqualTo((1, 1))); + } + } + + [Test] + public async Task An_import_of_a_message_an_earlier_batch_copied_from_another_document_is_reported_as_a_merge() + { + var first = await Target.Write(Category, BatchOf(Row("FailedErrorImports/a", FirstFailure)), CheckpointAfter("FailedErrorImports/a")); + var later = BatchOf(Row("FailedErrorImports/b", FirstFailure.AddMinutes(5))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), later); + var result = await Target.Write(Category, later, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "FailedErrorImports/b" }), "random document ids spread the imports of one message over different batches"); + Assert.That(merges.Single().Detail, Does.Contain("already holds an import of the same message")); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_document_an_earlier_pass_copied_is_already_present_and_not_a_merge() + { + var batch = BatchOf(Row("FailedErrorImports/a", FirstFailure)); + var first = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/a")); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, Writer(), batch); + var again = await Target.Write(Category, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges, Is.Empty, "--migration-retry re-reads the same documents, and finding its own rows again is not a merge"); + Assert.That(again.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task An_import_with_no_exception_or_no_body_is_reported_as_a_change_once() + { + var batch = BatchOf( + ImportRow("FailedErrorImports/no-exception", "native-1", exceptionInfo: null, [1]), + ImportRow("FailedErrorImports/no-body", "native-2", "Deserialization failed", body: null), + ImportRow("FailedErrorImports/neither", "native-3", exceptionInfo: null, body: null), + ImportRow("FailedErrorImports/complete", "native-4", "Deserialization failed", [1]), + ImportRow("FailedErrorImports/no-exception-again", "native-1", exceptionInfo: null, [1])); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var first = await Target.Write(Category, batch, CheckpointAfter("FailedErrorImports/no-exception-again")); + var again = await Target.Write(Category, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ("FailedErrorImports/no-exception", "Stored an empty exception, because the document has none."), + ("FailedErrorImports/no-body", "Stored an empty body, because the document's message has none."), + ("FailedErrorImports/neither", "Stored an empty exception and an empty body, because the document has neither.") + }), "a complete import changes nothing, and the second import of one message is a merge, not a change"); + Assert.That((first.Changed, again.Changed), Is.EqualTo((3, 0)), "a re-read finds the rows an earlier pass copied and changes nothing"); + } + } + + static MigrationRow ImportRow(string documentId, string messageId, string exceptionInfo, byte[] body) => new( + documentId, + new FailedErrorImport + { + Id = documentId, + Message = new FailedTransportMessage { Id = messageId, Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = messageId }, Body = body }, + ExceptionInfo = exceptionInfo + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = FirstFailure }, + new MigrationBody(body ?? [], "application/octet-stream")); + + FailedErrorImportsWriter Writer() => new( + ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), + ServiceProvider.GetRequiredService<IBodyStoragePersistence>(), + ServiceProvider.GetRequiredService<EFPersisterSettings>()); + + static MigrationRow Row(string documentId, DateTime failedAt) => new( + documentId, + new FailedErrorImport + { + Id = documentId, + Message = new FailedTransportMessage { Id = "native-8", Headers = new Dictionary<string, string> { ["NServiceBus.MessageId"] = "native-8" }, Body = [1, 2, 3] }, + ExceptionInfo = "Deserialization failed" + }, + new Dictionary<string, object> { [MigrationMetadataKeys.FailedAt] = failedAt }, + new MigrationBody(new byte[] { 1, 2, 3 }, "application/octet-stream")); + + static MigrationBatch BatchOf(params MigrationRow[] rows) => new([.. rows], rows[^1].SourceId); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs new file mode 100644 index 0000000000..9686b9a4f1 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageDocumentTranslatorTests.cs @@ -0,0 +1,570 @@ +#nullable enable +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[FixtureLifeCycle(LifeCycle.InstancePerTestCase)] +class FailedMessageDocumentTranslatorTests +{ + static readonly DateTime StatusChangedAtFromTheSource = new(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc); + static readonly DateTime LastModifiedFromTheSource = new(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc); + + [Test] + public void Carries_the_unique_message_id_across_unchanged() + { + var uniqueMessageId = Guid.NewGuid(); + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(); + document.UniqueMessageId = uniqueMessageId.ToString(); + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.UniqueMessageId, Is.EqualTo(uniqueMessageId)); + } + + [Test] + public void Skips_a_unique_message_id_that_is_not_a_guid() + { + var document = new IngestedFailure().ToFailedMessage(); + document.UniqueMessageId = "not-a-guid"; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.UnparseableUniqueMessageId)); + Assert.That(skip.Detail, Does.Contain("not-a-guid")); + } + } + + [Test] + public void Skips_a_document_with_no_processing_attempts() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts)); + Assert.That(skip.Detail, Does.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Skips_a_document_with_no_processing_attempts_that_the_source_sent_with_no_StatusChangedAt() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts.Clear(); + var complete = Row(document); + var row = complete with { Metadata = new Dictionary<string, object?>(complete.Metadata) { [MigrationMetadataKeys.StatusChangedAt] = null } }; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts), "the source resolves StatusChangedAt from the newest attempt, so an attemptless document always arrives without one"); + } + + [Test] + public void Skips_a_document_whose_newest_attempt_has_no_failing_endpoint_address() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].FailureDetails.AddressOfFailingEndpoint = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "FailingEndpointAddress is NOT NULL, so the insert would fail and halt the category"); + Assert.That(skip.Detail, Does.Contain(nameof(FailedMessageEntity.FailingEndpointAddress)).And.Contain(document.UniqueMessageId), "the log has to name the column and the message an operator would look up"); + } + } + + [TestCase(nameof(FailedMessage.ProcessingAttempt.FailureDetails))] + [TestCase(nameof(FailedMessage.ProcessingAttempt.MessageMetadata))] + [TestCase(nameof(FailedMessage.ProcessingAttempt.Headers))] + public void Skips_a_newest_attempt_with_no_value_the_row_is_built_from(string missing) + { + var document = new IngestedFailure().ToFailedMessage(); + var newest = document.ProcessingAttempts[^1]; + switch (missing) + { + case nameof(newest.FailureDetails): + newest.FailureDetails = null!; + break; + case nameof(newest.MessageMetadata): + newest.MessageMetadata = null!; + break; + default: + newest.Headers = null!; + break; + } + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + Assert.That(skip.Detail, Does.Contain(missing).And.Contain(document.UniqueMessageId), "the log has to name what is missing and the message an operator would look up"); + } + } + + [TestCase("TimeSent", "garbage", "garbage")] + [TestCase("MessageType", 43L, "43")] + [TestCase("ConversationId", 44L, "44")] + [TestCase("IsSystemMessage", "yes", "yes")] + public void Skips_a_value_present_with_the_wrong_type_and_names_the_key_the_type_found_and_the_value(string key, object malformed, string shown) + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata[key] = malformed; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.MalformedValue), "copying it as null or false would lose a value the source holds without a skip to show for it"); + Assert.That(skip.Detail, Does.Contain(key).And.Contain(malformed.GetType().FullName).And.Contain(document.UniqueMessageId)); + Assert.That(skip.Detail, Does.Contain($"'{shown}'"), "the operator should not have to open RavenDB to see the value"); + } + } + + [Test] + public void Shows_only_the_first_100_characters_of_a_malformed_value() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = new string('a', 100) + new string('b', 400); + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Detail, Does.Contain($"'{new string('a', 100)}...'").And.Not.Contain(new string('a', 100) + "b"), "a stored value can be any length, and the detail is one log line"); + } + + [Test] + public void Skips_a_newest_attempt_whose_FailureDetails_key_was_missing_from_the_document() + { + var document = new IngestedFailure().ToFailedMessage(); + // What RavenDB hands back for an attempt stored without the key. + document.ProcessingAttempts[^1].FailureDetails = new FailedMessage.ProcessingAttempt().FailureDetails; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "copying it would store an empty failing address to retry to and a time of failure that is the time it was read"); + Assert.That(skip.Detail, Does.Contain(nameof(FailedMessageEntity.FailingEndpointAddress)).And.Contain(document.UniqueMessageId)); + } + } + + [TestCase("TimeOfFailure")] + [TestCase("AttemptedAt")] + public void Skips_a_newest_attempt_with_no_time_it_failed_or_was_attempted(string missing) + { + var document = new IngestedFailure().ToFailedMessage(); + var newest = document.ProcessingAttempts[^1]; + if (missing == "TimeOfFailure") + { + newest.FailureDetails.TimeOfFailure = default; + } + else + { + newest.AttemptedAt = default; + } + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing), "the year 1 is not a time the message failed"); + Assert.That(skip.Detail, Does.Contain(missing).And.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Leaves_an_older_attempt_whose_FailureDetails_key_was_missing_out_of_the_first_and_last_failure_times() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails = new FailedMessage.ProcessingAttempt().FailureDetails; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[1].FailureDetails.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure), "the default object's time of failure is the time it was read, which would become the last failure"); + } + } + + [Test] + public void Leaves_an_older_attempt_with_no_time_of_failure_out_of_the_first_failure_time() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails.TimeOfFailure = default; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[1].FailureDetails.TimeOfFailure), "the year 1 is not a time the message failed"); + } + + [Test] + public void Skips_a_document_whose_processing_attempts_list_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.NoProcessingAttempts)); + } + + [Test] + public void Translates_a_null_failure_groups_list_to_no_groups() + { + var document = new IngestedFailure().ToFailedMessage(); + document.FailureGroups = null!; + + var translation = FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That(((FailedMessageDocumentTranslation.Translated)translation).Groups, Is.Empty); + } + + [Test] + public void Drops_a_failure_group_entry_that_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + document.FailureGroups = [null!, new() { Id = "group-1", Title = "Title 1", Type = "Type 1" }]; + + var translation = FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That(((FailedMessageDocumentTranslation.Translated)translation).Groups.Select(group => group.GroupId), Is.EqualTo(new[] { "group-1" })); + } + + [TestCase("SendingEndpoint", "not an endpoint")] + [TestCase("ReceivingEndpoint", 42)] + public void Refuses_an_endpoint_that_is_not_an_EndpointDetails_rather_than_copying_it_as_null(string key, object wrongType) + { + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata[key] = wrongType; + + Assert.Throws<InvalidCastException>(() => FailedMessageDocumentTranslator.Translate(Row(document)), "the source restores every endpoint before the translator sees it, so a wrong type is a defect that must halt the copy, not a row with empty endpoint columns"); + } + + [TestCase(true)] + [TestCase(false)] + public void Translates_a_TimeSent_or_endpoint_that_is_missing_or_null_to_null(bool keyPresent) + { + var document = new IngestedFailure().ToFailedMessage(); + var metadata = document.ProcessingAttempts[^1].MessageMetadata; + foreach (var key in new[] { "TimeSent", "SendingEndpoint", "ReceivingEndpoint" }) + { + if (keyPresent) + { + metadata[key] = null!; + } + else + { + metadata.Remove(key); + } + } + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.TimeSent, Is.Null); + Assert.That(new[] { row.SendingEndpointName, row.SendingEndpointHost, row.ReceivingEndpointName, row.ReceivingEndpointHost }, Is.All.Null, "RavenDB's own read path shows a missing endpoint as null"); + Assert.That(new[] { row.SendingEndpointHostId, row.ReceivingEndpointHostId }, Is.All.Null); + } + } + + [Test] + public void Skips_a_document_whose_newest_attempt_is_null() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[2] = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + Assert.That(skip.Detail, Does.Contain(document.UniqueMessageId)); + } + } + + [Test] + public void Takes_the_earliest_failure_from_the_attempts_that_have_failure_details() + { + var document = ThreeAttempts(); + var earliestWithDetails = document.ProcessingAttempts[1].FailureDetails.TimeOfFailure; + document.ProcessingAttempts[0].FailureDetails = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(earliestWithDetails), "an older attempt with no failure details has no time to offer, and must not halt the category"); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + } + } + + [Test] + public void Ignores_an_earlier_attempt_that_is_null() + { + var document = ThreeAttempts(); + var earliestRemaining = document.ProcessingAttempts[1].FailureDetails.TimeOfFailure; + document.ProcessingAttempts[0] = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(3), "RavenDB's read path counts every stored attempt, null ones included"); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(earliestRemaining)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + } + } + + [Test] + public void Translates_when_only_an_earlier_attempt_has_a_value_of_the_wrong_type() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].MessageMetadata["TimeSent"] = "garbage"; + document.ProcessingAttempts[0].MessageMetadata["ReceivingEndpoint"] = "not an endpoint"; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address"), "only the newest attempt's metadata becomes the row"); + } + + [Test] + public void Keeps_the_last_450_characters_of_a_message_type_longer_than_the_column() + { + var document = new IngestedFailure().ToFailedMessage(); + var longType = new string('a', 100) + new string('b', 450); + document.ProcessingAttempts[^1].MessageMetadata["MessageType"] = longType; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.MessageType, Is.EqualTo(new string('b', 450))); + } + + [Test] + public void Takes_the_newest_attempt_for_the_row_and_the_earliest_and_latest_failure_across_all() + { + var document = ThreeAttempts(); + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NumberOfProcessingAttempts, Is.EqualTo(3)); + Assert.That(row.FirstTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[0].FailureDetails.TimeOfFailure)); + Assert.That(row.LastTimeOfFailure, Is.EqualTo(document.ProcessingAttempts[2].FailureDetails.TimeOfFailure)); + Assert.That(row.LastAttemptedAt, Is.EqualTo(document.ProcessingAttempts[2].AttemptedAt)); + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address")); + Assert.That(row.ExceptionMessage, Is.EqualTo("newest-exception")); + } + } + + [Test] + public void Skips_when_only_the_newest_of_three_attempts_has_no_failing_endpoint_address() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[2].FailureDetails.AddressOfFailingEndpoint = null!; + + var skip = SkipOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(skip.Reason, Is.EqualTo(MigrationSkipReason.RequiredValueMissing)); + } + + [Test] + public void Translates_when_only_an_earlier_attempt_has_no_failing_endpoint_address() + { + var document = ThreeAttempts(); + document.ProcessingAttempts[0].FailureDetails.AddressOfFailingEndpoint = null!; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + Assert.That(row.FailingEndpointAddress, Is.EqualTo("newest-address")); + } + + [Test] + public void Translates_each_group_with_an_id_to_a_row_keyed_by_the_message_and_drops_a_group_with_no_id() + { + var uniqueMessageId = Guid.NewGuid(); + var document = new IngestedFailure().ToFailedMessage(); + document.UniqueMessageId = uniqueMessageId.ToString(); + document.FailureGroups = + [ + new() { Id = "group-1", Title = "Title 1", Type = "Type 1" }, + new() { Id = "group-2", Title = null, Type = null }, + new() { Id = null!, Title = "No id", Type = "Type" } + ]; + + var translation = (FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(Row(document)); + + Assert.That( + translation.Groups.Select(group => (group.FailedMessageUniqueId, group.GroupId, group.Title, group.Type)), + Is.EqualTo(new[] + { + (uniqueMessageId, "group-1", "Title 1", "Type 1"), + (uniqueMessageId, "group-2", string.Empty, string.Empty) + })); + } + + [TestCase(FailedMessageStatus.Unresolved)] + [TestCase(FailedMessageStatus.Resolved)] + [TestCase(FailedMessageStatus.RetryIssued)] + [TestCase(FailedMessageStatus.Archived)] + public void Preserves_the_stored_status(FailedMessageStatus status) + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(status)))); + + Assert.That(row.Status, Is.EqualTo(status)); + } + + [Test] + public void Stores_the_StatusChangedAt_the_source_resolved() + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)))); + + Assert.That(row.StatusChangedAt, Is.EqualTo(StatusChangedAtFromTheSource)); + } + + [Test] + public void Stores_the_LastModified_the_source_carried() + { + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(new IngestedFailure().ToFailedMessage(FailedMessageStatus.Archived)))); + + Assert.That(row.LastModified, Is.EqualTo(LastModifiedFromTheSource), "RavenDB's @last-modified, not the copy time"); + } + + [TestCase(MigrationMetadataKeys.StatusChangedAt)] + [TestCase(MigrationMetadataKeys.LastModified)] + public void Refuses_a_row_the_source_sent_without_a_resolved_value(string missingKey) + { + var complete = Row(new IngestedFailure().ToFailedMessage()); + var row = complete with { Metadata = complete.Metadata.Where(entry => entry.Key != missingKey).ToDictionary(entry => entry.Key, entry => entry.Value) }; + + var ex = Assert.Throws<InvalidOperationException>(() => FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(ex.Message, Does.Contain(missingKey).And.Contain(row.SourceId)); + } + + [TestCase(MigrationMetadataKeys.StatusChangedAt)] + [TestCase(MigrationMetadataKeys.LastModified)] + public void Refuses_a_resolved_value_of_the_wrong_type_and_names_the_type_it_found(string key) + { + var complete = Row(new IngestedFailure().ToFailedMessage()); + var row = complete with { Metadata = new Dictionary<string, object?>(complete.Metadata) { [key] = "2026-09-01T08:00:00Z" } }; + + var ex = Assert.Throws<InvalidOperationException>(() => FailedMessageDocumentTranslator.Translate(row)); + + Assert.That(ex.Message, Does.Contain(typeof(string).FullName).And.Contain(key).And.Contain(row.SourceId), "a value that is present but mistyped must not read as missing"); + } + + [TestCase(DateTimeKind.Unspecified)] + [TestCase(DateTimeKind.Local)] + public void Writes_every_date_as_UTC_whatever_kind_it_arrived_with(DateTimeKind kind) + { + var document = ThreeAttempts(); + foreach (var attempt in document.ProcessingAttempts) + { + attempt.FailureDetails.TimeOfFailure = DateTime.SpecifyKind(attempt.FailureDetails.TimeOfFailure, kind); + attempt.AttemptedAt = DateTime.SpecifyKind(attempt.AttemptedAt, kind); + } + + var newest = document.ProcessingAttempts[^1]; + var timeSent = new DateTime(2026, 7, 22, 9, 59, 0, kind); + newest.MessageMetadata["TimeSent"] = timeSent; + var statusChangedAt = DateTime.SpecifyKind(StatusChangedAtFromTheSource, kind); + var lastModified = DateTime.SpecifyKind(LastModifiedFromTheSource, kind); + var complete = Row(document); + var source = complete with + { + Metadata = new Dictionary<string, object?> + { + [MigrationMetadataKeys.StatusChangedAt] = statusChangedAt, + [MigrationMetadataKeys.LastModified] = lastModified + } + }; + + var row = RowOf(FailedMessageDocumentTranslator.Translate(source)); + + DateTime[] written = [row.StatusChangedAt, row.LastModified, row.FirstTimeOfFailure, row.LastTimeOfFailure, row.LastAttemptedAt, row.TimeSent!.Value]; + DateTime[] expected = [statusChangedAt, lastModified, document.ProcessingAttempts[0].FailureDetails.TimeOfFailure, newest.FailureDetails.TimeOfFailure, newest.AttemptedAt, timeSent]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(written.Select(value => value.Kind), Is.All.EqualTo(DateTimeKind.Utc), "Npgsql refuses to write a DateTime of any other kind to a timestamp with time zone column"); + Assert.That(written, Is.EqualTo(expected.Select(value => kind == DateTimeKind.Local ? value.ToUniversalTime() : DateTime.SpecifyKind(value, DateTimeKind.Utc))), "a value with no kind is already UTC and must not be shifted by the local offset"); + } + } + + [Test] + public void Stores_a_body_externally_when_the_content_type_header_is_null() + { + var document = new IngestedFailure().ToFailedMessage(); + var headers = document.ProcessingAttempts[^1].Headers; + headers[NServiceBus.Headers.ContentType] = null!; + var row = RowOf(FailedMessageDocumentTranslator.Translate(Row(document))); + + var storeExternally = FailedMessageDocumentTranslator.SetBody(row, headers, new MigrationBody("hello"u8.ToArray(), "text/plain"), BodyStorageSettings.DefaultMaxBodySizeToStore); + + using (Assert.EnterMultipleScope()) + { + Assert.That(storeExternally, Is.True, "a null content type says nothing about the body, so it is treated like a missing one, which is binary"); + Assert.That(row.BodyStoredExternally, Is.True); + Assert.That(row.BodyText, Is.Null); + } + } + + static FailedMessageEntity RowOf(FailedMessageDocumentTranslation translation) + { + Assert.That(translation, Is.TypeOf<FailedMessageDocumentTranslation.Translated>()); + return ((FailedMessageDocumentTranslation.Translated)translation).Row; + } + + static FailedMessageDocumentTranslation.Skipped SkipOf(FailedMessageDocumentTranslation translation) + { + Assert.That(translation, Is.TypeOf<FailedMessageDocumentTranslation.Skipped>()); + return (FailedMessageDocumentTranslation.Skipped)translation; + } + + static FailedMessage ThreeAttempts() + { + var failure = new IngestedFailure(); + var document = failure.ToFailedMessage(numberOfAttempts: 3); + for (var i = 0; i < 3; i++) + { + var attempt = document.ProcessingAttempts[i]; + attempt.FailureDetails.TimeOfFailure = new DateTime(2026, 7, 22, 10 + i, 0, 0, DateTimeKind.Utc); + attempt.AttemptedAt = attempt.FailureDetails.TimeOfFailure; + } + + var newest = document.ProcessingAttempts[2]; + newest.FailureDetails.AddressOfFailingEndpoint = "newest-address"; + newest.FailureDetails.Exception!.Message = "newest-exception"; + return document; + } + + // Every row carries the two dates the source resolves, as a real source's rows do. + static MigrationRow Row(FailedMessage document) => + new( + $"FailedMessages/{document.UniqueMessageId}", + document, + new Dictionary<string, object?> + { + [MigrationMetadataKeys.StatusChangedAt] = StatusChangedAtFromTheSource, + [MigrationMetadataKeys.LastModified] = LastModifiedFromTheSource + }); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs new file mode 100644 index 0000000000..b273ece28a --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessageMigrationCoverageTests.cs @@ -0,0 +1,85 @@ +#nullable enable +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class FailedMessageMigrationCoverageTests : PersistenceTestBase +{ + [Test] + public void Every_failed_message_column_is_set_from_a_fully_populated_document() + { + var source = FullyPopulatedRow(); + var row = ((FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(source)).Row; + // Over the cap, so it sets all four body columns: an inline prefix, the external flag, the size and the content type. + var overCapText = Encoding.UTF8.GetBytes(new string('a', BodyStorageSettings.DefaultMaxBodySizeToStore * 2)); + + FailedMessageDocumentTranslator.SetBody(row, ((FailedMessage)source.Document).ProcessingAttempts[^1].Headers, new MigrationBody(overCapText, "text/plain"), BodyStorageSettings.DefaultMaxBodySizeToStore); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), row); + } + + [Test] + public void Every_failure_group_column_is_set_from_a_fully_populated_document() + { + var translation = (FailedMessageDocumentTranslation.Translated)FailedMessageDocumentTranslator.Translate(FullyPopulatedRow()); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), translation.Groups.Single()); + } + + [Test] + public async Task Every_retry_claim_column_is_set_for_a_copied_retry_issued_message() + { + var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + var row = FullyPopulatedRow(); + var source = row with + { + Metadata = new Dictionary<string, object?>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = true }, + Body = new MigrationBody(Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml") + }; + ((FailedMessage)source.Document).Status = FailedMessageStatus.RetryIssued; + await target.Open(); + + await target.Write(category, new MigrationBatch([source], source.SourceId), new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null)); + + using var scope = ServiceProvider.CreateScope(); + var claim = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().FailedMessageRetries.AsNoTracking().SingleAsync(); + + // The migrated claim's batch is never staged, so its attempt count stays 0 and is never read. + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(Model(), claim, nameof(FailedMessageRetryEntity.StageAttempts)); + } + + static MigrationRow FullyPopulatedRow() + { + var failure = new IngestedFailure { IsSystemMessage = true, FailingEndpointAddress = "Sales@MACHINE" }; + + return new MigrationRow( + $"FailedMessages/{failure.UniqueMessageIdString}", + failure.ToFailedMessage(), + new Dictionary<string, object?> + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }); + } + + IModel Model() + { + using var scope = ServiceProvider.CreateScope(); + return scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs new file mode 100644 index 0000000000..814f5565a3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/FailedMessagesMigrationTargetTests.cs @@ -0,0 +1,711 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class FailedMessagesMigrationTargetTests : PersistenceTestBase +{ + readonly TargetWarnings targetWarnings = new(); + + public FailedMessagesMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetWarnings); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + IMigrationCheckpointStore CheckpointStore => ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + + IBodyStoragePersistence BodyStore => ServiceProvider.GetRequiredService<IBodyStoragePersistence>(); + + static readonly MigrationBody SmallTextBody = new(Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml"); + + static readonly MigrationCategory FailedMessagesCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + [Test] + public async Task A_copied_message_arrives_with_its_failure_group_rows() + { + var failure = new IngestedFailure(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(failure.ToFailedMessage()), Checkpoint()); + + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().Where(group => group.FailedMessageUniqueId == failure.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(groups.Select(group => (group.GroupId, group.Title, group.Type)), Is.EqualTo(failure.Groups.Select(group => (group.Id, group.Title, group.Type)))); + } + } + + [Test] + public async Task A_message_already_in_the_target_keeps_its_row_and_gets_no_group_rows() + { + var failure = new IngestedFailure(); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = failure.UniqueMessageId, Status = FailedMessageStatus.Unresolved, ExceptionMessage = "Already in the target", HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(failure.ToFailedMessage()), Checkpoint()); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == failure.UniqueMessageId)); + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().CountAsync(group => group.FailedMessageUniqueId == failure.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "nothing was lost, so a skip would push a healthy category toward the halt threshold"); + Assert.That(row.ExceptionMessage, Is.EqualTo("Already in the target"), "insert-if-absent never updates"); + Assert.That(groups, Is.Zero, "the copy wrote the source's groups onto a message it did not insert"); + } + } + + [Test] + public async Task A_copied_retry_issued_message_gets_a_retry_claim_and_an_unresolved_one_does_not() + { + var unresolved = new IngestedFailure(); + var retryIssued = new IngestedFailure(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(unresolved.ToFailedMessage(), retryIssued.ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => new { claim.UniqueMessageId, claim.RetryBatchId, claim.StageAttempts }).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + Assert.That(claims, Is.EqualTo(new[] { new { retryIssued.UniqueMessageId, RetryBatchId = FailedMessagesWriterBase.MigratedRetryClaimBatchId, StageAttempts = 0 } }), + "without a claim a retry by id sends a retry-issued message again, and with one an unresolved message could not be retried by id"); + } + } + + [Test] + public async Task A_retry_issued_message_already_in_the_target_gets_no_retry_claim() + { + var present = new IngestedFailure(); + var absent = new IngestedFailure(); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = present.UniqueMessageId, Status = FailedMessageStatus.RetryIssued, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage(FailedMessageStatus.RetryIssued), absent.ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + + var claimed = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(claimed, Is.EqualTo(new[] { absent.UniqueMessageId }), "a message the insert turned away keeps what the target had, so it gets no claim from the copy"); + } + } + + [Test] + public async Task Two_documents_sharing_a_unique_message_id_get_a_retry_claim_only_when_the_kept_one_is_retry_issued() + { + var unresolvedKept = new IngestedFailure(); + var retryIssuedKept = new IngestedFailure(); + var retryIssuedTurnedAway = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + retryIssuedTurnedAway.UniqueMessageId = unresolvedKept.UniqueMessageIdString; + var unresolvedTurnedAway = new IngestedFailure().ToFailedMessage(); + unresolvedTurnedAway.UniqueMessageId = retryIssuedKept.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + unresolvedKept.ToFailedMessage(), retryIssuedTurnedAway, + retryIssuedKept.ToFailedMessage(FailedMessageStatus.RetryIssued), unresolvedTurnedAway), Checkpoint()); + + var claimed = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(claimed, Is.EqualTo(new[] { retryIssuedKept.UniqueMessageId }), "the claim follows the status of the row the insert kept, not of the document it turned away"); + } + } + + [Test] + public async Task Copied_counts_the_messages_inserted_not_the_messages_handed_in() + { + var present = new IngestedFailure(); + var absent = new IngestedFailure(); + await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage()), Checkpoint()); + var saved = await CheckpointStore.Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(present.ToFailedMessage(), absent.ToFailedMessage()), saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + } + } + + [Test] + public async Task Two_documents_sharing_a_unique_message_id_store_only_the_first_documents_groups() + { + var first = new IngestedFailure { Groups = [new() { Id = "group-of-the-first", Title = "First", Type = "Message Type" }] }; + var second = new IngestedFailure { Groups = [new() { Id = "group-of-the-second", Title = "Second", Type = "Message Type" }] }; + var secondDocument = second.ToFailedMessage(); + secondDocument.UniqueMessageId = first.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf(first.ToFailedMessage(), secondDocument), Checkpoint()); + + var groupIds = await Query(db => db.FailedMessageGroups.AsNoTracking().Where(group => group.FailedMessageUniqueId == first.UniqueMessageId).Select(group => group.GroupId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(groupIds, Is.EqualTo(new[] { "group-of-the-first" }), "the declined document's groups landed on the message the insert kept"); + } + } + + [Test] + public async Task A_document_the_mapper_skips_is_counted_under_its_reason_and_never_written() + { + var good = new IngestedFailure(); + var unparseable = new IngestedFailure().ToFailedMessage(); + unparseable.UniqueMessageId = "not-a-guid"; + var attemptless = new IngestedFailure().ToFailedMessage(); + attemptless.ProcessingAttempts.Clear(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(good.ToFailedMessage(), unparseable, attemptless), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(2)); + Assert.That(result.AlreadyPresent, Is.Zero); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.UnparseableUniqueMessageId] = 1, + [MigrationSkipReason.NoProcessingAttempts] = 1 + })); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(1)); + } + } + + [Test] + public async Task A_document_with_a_value_of_the_wrong_type_or_no_failure_details_is_skipped_and_the_rest_copy() + { + var good = new IngestedFailure(); + var badTimeSent = new IngestedFailure().ToFailedMessage(); + badTimeSent.ProcessingAttempts[^1].MessageMetadata["TimeSent"] = "garbage"; + var noFailureDetails = new IngestedFailure().ToFailedMessage(); + noFailureDetails.ProcessingAttempts[^1].FailureDetails = null; + + var result = await Target.Write(FailedMessagesCategory, BatchOf(badTimeSent, good.ToFailedMessage(), noFailureDetails), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EquivalentTo(new[] { $"FailedMessages/{badTimeSent.UniqueMessageId}", $"FailedMessages/{noFailureDetails.UniqueMessageId}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.MalformedValue] = 1, + [MigrationSkipReason.RequiredValueMissing] = 1 + }), "one malformed document no longer halts the category"); + Assert.That(await Query(db => db.FailedMessages.AsNoTracking().Select(message => message.UniqueMessageId).ToListAsync()), Is.EqualTo(new[] { good.UniqueMessageId })); + } + } + + [Test] + public async Task A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy() + { + var fitsExactly = new IngestedFailure + { + ConversationId = new string('c', 450), + Groups = [new() { Id = new string('g', 64), Title = "OrderPlaced", Type = new string('t', 255) }] + }; + var longConversation = new IngestedFailure { ConversationId = new string('c', 451) }; + var longGroupType = new IngestedFailure { Groups = [new() { Id = Guid.NewGuid().ToString(), Title = "OrderPlaced", Type = new string('t', 256) }] }; + var longMessageType = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(fitsExactly.ToFailedMessage(), longConversation.ToFailedMessage(), longGroupType.ToFailedMessage(), longMessageType.ToFailedMessage()); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new FailedMessagesWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), BodyStore, ServiceProvider.GetRequiredService<EFPersisterSettings>()), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + + var copiedIds = await Query(db => db.FailedMessages.AsNoTracking().Select(message => message.UniqueMessageId).ToListAsync()); + var groupOwners = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => group.FailedMessageUniqueId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "values exactly as long as their columns fit, and MessageType keeps its last 450 characters as ingestion does, so neither is skipped for length"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{longConversation.UniqueMessageIdString}", $"FailedMessages/{longGroupType.UniqueMessageIdString}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(copiedIds, Is.EquivalentTo(new[] { fitsExactly.UniqueMessageId, longMessageType.UniqueMessageId })); + Assert.That(groupOwners, Is.EquivalentTo(new[] { fitsExactly.UniqueMessageId, longMessageType.UniqueMessageId }), "a message whose group row cannot be stored is skipped whole, not copied without its group"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<FailedMessageEntity>(ServiceProvider, nameof(FailedMessageEntity.ConversationId))} takes at most 450 characters and the value holds 451, on document 'FailedMessages/{longConversation.UniqueMessageIdString}'", + $"{PreparedSkips.TableAndColumnOf<FailedMessageGroupEntity>(ServiceProvider, nameof(FailedMessageGroupEntity.Type))} takes at most 255 characters and the value holds 256, on document 'FailedMessages/{longGroupType.UniqueMessageIdString}'" + })); + } + } + + [Test] + public async Task A_change_is_not_logged_when_the_batch_does_not_commit() + { + var longType = new IngestedFailure { MessageType = new string('m', 600) }; + await Target.Write(FailedMessagesCategory, BatchOf(new IngestedFailure().ToFailedMessage()), Checkpoint()); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.Write(FailedMessagesCategory, BatchOf(longType.ToFailedMessage()), Checkpoint()), + "the checkpoint it extends is older than the stored one, so the save in the batch's transaction is refused"); + Assert.That(targetWarnings.Messages, Has.None.StartsWith("Changed "), "a change line written before the commit would name a row SQL never stored"); + } + + [Test] + public async Task A_second_document_with_the_same_message_id_reports_no_change_of_its_own() + { + var first = new IngestedFailure(); + var second = new IngestedFailure { MessageType = new string('m', 600) }.ToFailedMessage(); + second.UniqueMessageId = first.UniqueMessageIdString; + var batch = BatchOf(first.ToFailedMessage(), second); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + + Assert.That(changes, Is.Empty, "the insert keeps the first document, so the second's cut message type never reaches SQL"); + } + + [Test] + public async Task A_message_type_longer_than_its_column_is_copied_with_a_change_naming_its_length() + { + var longType = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(longType.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] { ($"FailedMessages/{longType.UniqueMessageIdString}", "Cut the message type from 600 characters to its last 450, as ingestion does.") })); + Assert.That(result.Changed, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_failure_group_with_no_id_or_no_title_is_reported_as_a_change_naming_the_group() + { + var failure = new IngestedFailure + { + Groups = + [ + new() { Id = "group-kept", Title = null, Type = "Exception Type" }, + new() { Id = null!, Title = "Has no id", Type = "Message Type" }, + null! + ] + }; + var batch = BatchOf(failure.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var groups = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => new { group.GroupId, group.Title }).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ($"FailedMessages/{failure.UniqueMessageIdString}", + "Dropped the failure groups with no id, because SQL keys a group by its id: the group titled 'Has no id', an empty entry. Stored an empty title or type for the failure groups that had none: 'group-kept'.") + })); + Assert.That(result.Changed, Is.EqualTo(1)); + Assert.That(groups.Select(group => (group.GroupId, group.Title)), Is.EqualTo(new[] { ("group-kept", "") }), "the message is missing from the group with no id, which the line has to say"); + } + } + + [Test] + public async Task Two_failure_groups_of_one_message_with_one_key_are_reported_as_a_change_naming_both() + { + var failure = new IngestedFailure + { + Groups = + [ + new() { Id = "group-a", Title = "First", Type = "Exception Type" }, + new() { Id = "group-a", Title = "Second", Type = "Exception Type" } + ] + }; + var batch = BatchOf(failure.ToFailedMessage()); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, Writer(), batch); + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var titles = await Query(db => db.FailedMessageGroups.AsNoTracking().Select(group => group.Title).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ($"FailedMessages/{failure.UniqueMessageIdString}", "Kept the failure group 'group-a' titled 'First' and dropped 'group-a' titled 'Second', because SQL treats their ids as one key.") + })); + Assert.That(result.Changed, Is.EqualTo(1)); + Assert.That(titles, Is.EqualTo(new[] { "First" }), "the line names the group the insert kept"); + } + } + + [Test] + public async Task A_retry_issued_message_gets_a_retry_claim_only_when_the_source_still_held_one() + { + var released = new IngestedFailure(); + var held = new IngestedFailure(); + var batch = BatchOf( + WithClaimHeld(RowOf(released.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), true)); + + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "the test proves nothing unless both messages reached the target"); + Assert.That(claims, Is.EqualTo(new[] { held.UniqueMessageId }), "a claim the source had released would block a retry by id that RavenDB would have sent"); + Assert.That(result.Changed, Is.Zero, "the claims match the source, so nothing changed on the way in"); + } + } + + [Test] + public async Task A_retry_by_id_after_the_copy_stages_the_message_whose_claim_the_source_had_released() + { + var released = new IngestedFailure(); + var held = new IngestedFailure(); + await Target.Write(FailedMessagesCategory, BatchOf( + WithClaimHeld(RowOf(released.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), true)), Checkpoint()); + + string[] ids = [released.UniqueMessageIdString, held.UniqueMessageIdString]; + var batchId = await RetryBatchStore.CreateBatch("migration-test-session", "retry-by-id", RetryType.MultipleMessages, ids, null, DateTime.UtcNow); + await RetryBatchStore.AssignMessagesToBatch(batchId, ids); + + var staged = await RetryStagingStore.GetMessagesToStage(batchId); + + Assert.That(staged.Select(message => message.UniqueMessageId), Is.EqualTo(new[] { released.UniqueMessageIdString }), + "RavenDB would send the released message and hold back the claimed one, so after the move a retry by id must do the same"); + } + + [Test] + public void A_retry_issued_message_without_its_claim_flag_fails_the_write_naming_the_row() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(WithClaimHeld(RowOf(document, SmallTextBody), null)), Checkpoint())); + + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationMetadataKeys.RetryClaimHeld), + "the source resolves the flag for every retry-issued message, so a missing one is a defect that must not pass as a held claim"); + } + + [Test] + public async Task A_retry_issued_message_whose_claim_flag_is_not_a_bool_fails_the_write_and_stores_nothing() + { + var document = new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued); + var row = WithClaimHeld(RowOf(document, SmallTextBody), null); + row = row with { Metadata = new Dictionary<string, object>(row.Metadata) { [MigrationMetadataKeys.RetryClaimHeld] = "true" } }; + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(row), Checkpoint())); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().CountAsync()); + var messages = await Query(db => db.FailedMessages.AsNoTracking().CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationMetadataKeys.RetryClaimHeld).And.Contain(typeof(string).FullName)); + Assert.That((claims, messages), Is.EqualTo((0, 0)), "a flag the writer cannot read must stop the batch before anything is stored, not be read as either answer"); + } + } + + [Test] + public async Task Two_retry_issued_documents_sharing_a_unique_message_id_take_the_claim_flag_of_the_first() + { + var first = new IngestedFailure(); + var second = first.ToFailedMessage(FailedMessageStatus.RetryIssued); + var batch = BatchOf( + WithClaimHeld(RowOf(first.ToFailedMessage(FailedMessageStatus.RetryIssued), SmallTextBody), false), + WithClaimHeld(RowOf(second, SmallTextBody) with { SourceId = "FailedMessages/second" }, true)); + + var result = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().CountAsync()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "the test proves nothing unless the insert kept one of the two"); + Assert.That(claims, Is.Zero, "the insert keeps the first document, whose claim the source had released, so the second's held flag must not add one"); + } + } + + [Test] + public async Task Copying_a_batch_with_a_held_claim_again_leaves_one_claim() + { + var held = new IngestedFailure(); + var batch = BatchOf(held.ToFailedMessage(FailedMessageStatus.RetryIssued)); + + var first = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + await Target.Write(FailedMessagesCategory, batch, first.Saved); + var claims = await Query(db => db.FailedMessageRetries.AsNoTracking().Select(claim => claim.UniqueMessageId).ToListAsync()); + + Assert.That(claims, Is.EqualTo(new[] { held.UniqueMessageId }), "a retry re-reads rows an earlier pass copied, and must not add a second claim or fail on the first"); + } + + [Test] + public async Task A_changed_message_the_target_already_holds_is_not_reported_again() + { + var changed = new IngestedFailure { MessageType = new string('m', 600) }; + var batch = BatchOf(changed.ToFailedMessage()); + + var first = await Target.Write(FailedMessagesCategory, batch, Checkpoint()); + var again = await Target.Write(FailedMessagesCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That((first.Changed, again.Changed), Is.EqualTo((1, 0)), "a retry re-reads rows an earlier pass copied, and this write changed nothing"); + Assert.That(again.AlreadyPresent, Is.EqualTo(1)); + Assert.That(targetWarnings.Messages.Count(message => message.StartsWith("Changed ")), Is.EqualTo(1)); + } + } + + [Test] + public async Task Count_takes_in_only_unresolved_and_retry_issued_messages() + { + await Target.Write(FailedMessagesCategory, BatchOf(new IngestedFailure().ToFailedMessage(), new IngestedFailure().ToFailedMessage(FailedMessageStatus.RetryIssued)), Checkpoint()); + await Query(db => + { + db.FailedMessages.AddRange( + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.Archived, HeadersJson = "{}", FailingEndpointAddress = "error" }, + new FailedMessageEntity { UniqueMessageId = Guid.NewGuid(), Status = FailedMessageStatus.Resolved, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await Target.Count(FailedMessagesCategory), Is.EqualTo(2), "the archived and resolved rows belong to another category"); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(4), "the filter is doing the work, not an empty table"); + } + } + + // PostgreSQL takes a fixed number of rows per statement whatever the entity, so only SQL Server can catch the wrong entity here. + [Test] + public async Task A_batch_is_one_statement_of_failed_message_rows() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var expected = await Query(db => Task.FromResult(dialect.RowsPerStatement(MigrationInsert<FailedMessageEntity>.For(db).Columns.Count))); + + Assert.That(await Target.BatchSizeFor(FailedMessagesCategory), Is.EqualTo(expected)); + } + + [Test] + public async Task A_write_larger_than_one_statement_copies_every_message_and_its_groups() + { + var rowsPerStatement = await Target.BatchSizeFor(FailedMessagesCategory); + var documents = Enumerable.Range(0, (rowsPerStatement * 2) + 1).Select(_ => new IngestedFailure().ToFailedMessage()).ToArray(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(documents), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(documents.Length)); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.EqualTo(documents.Length)); + Assert.That(await Query(db => db.FailedMessageGroups.CountAsync()), Is.EqualTo(documents.Length)); + } + } + + [Test] + public async Task A_message_already_in_the_target_has_no_body_written_and_one_it_lacks_does() + { + var present = new IngestedFailure { ContentType = "application/octet-stream" }; + var absent = new IngestedFailure { ContentType = "application/octet-stream" }; + var binaryBody = new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"); + await Query(db => + { + db.FailedMessages.Add(new FailedMessageEntity { UniqueMessageId = present.UniqueMessageId, Status = FailedMessageStatus.Unresolved, HeadersJson = "{}", FailingEndpointAddress = "error" }); + return db.SaveChangesAsync(); + }); + + var result = await Target.Write(FailedMessagesCategory, BatchOf(RowOf(present.ToFailedMessage(), binaryBody), RowOf(absent.ToFailedMessage(), binaryBody)), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(await BodyStored(present.UniqueMessageIdString), Is.False, "the insert declines a message the target holds, so a body written for it is a wasted write"); + Assert.That(await BodyStored(absent.UniqueMessageIdString), Is.True, "the test proves nothing if no body is ever written"); + } + } + + [Test] + public async Task Only_the_first_document_sharing_a_unique_message_id_has_its_body_stored() + { + var first = new IngestedFailure(); + var second = new IngestedFailure { ContentType = "application/octet-stream" }; + var secondDocument = second.ToFailedMessage(); + secondDocument.UniqueMessageId = first.UniqueMessageIdString; + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + RowOf(first.ToFailedMessage(), SmallTextBody), + RowOf(secondDocument, new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"))), Checkpoint()); + + var row = await Query(db => db.FailedMessages.AsNoTracking().SingleAsync(message => message.UniqueMessageId == first.UniqueMessageId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(row.BodyStoredExternally, Is.False, "the insert keeps the first document, whose body is inline"); + Assert.That(await BodyStored(first.UniqueMessageIdString), Is.False, "the declined document's body was stored under the id of the row the insert kept"); + } + } + + [Test] + public async Task A_body_content_type_longer_than_its_column_skips_the_message_and_the_rest_copy() + { + var good = new IngestedFailure(); + var longContentType = new IngestedFailure().ToFailedMessage(); + + var result = await Target.Write(FailedMessagesCategory, BatchOf( + RowOf(good.ToFailedMessage(), SmallTextBody), + RowOf(longContentType, SmallTextBody with { ContentType = $"text/{new string('x', 446)}" })), Checkpoint()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"FailedMessages/{longContentType.UniqueMessageId}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 1 }), "an over-long content type would fail the whole insert and halt the category"); + } + } + + [Test] + public async Task A_body_that_cannot_be_written_fails_the_write_before_any_row_or_checkpoint_commits() + { + var target = new EFCoreMigrationTarget( + ServiceProvider.GetRequiredService<IServiceScopeFactory>(), + ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), + new UnwritableBodyStore(), + ServiceProvider.GetRequiredService<EFPersisterSettings>(), + ServiceProvider.GetRequiredService<ILogger<EFCoreMigrationTarget>>()); + var inline = new IngestedFailure(); + var binary = new IngestedFailure { ContentType = "application/octet-stream" }; + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => target.Write(FailedMessagesCategory, BatchOf( + RowOf(inline.ToFailedMessage(), SmallTextBody), + RowOf(binary.ToFailedMessage(), new MigrationBody(new byte[] { 0x08, 0x00, 0x96, 0x01 }, "application/octet-stream"))), Checkpoint())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{binary.UniqueMessageId}'").And.Contain($"'{binary.UniqueMessageId}'").And.Contain(nameof(IOException)), + "a storage SDK's own error does not name the body, so without the row and body id the operator cannot tell which message halted the category"); + Assert.That(exception.InnerException, Is.TypeOf<IOException>().And.Message.EqualTo(UnwritableBodyStore.Failure)); + Assert.That(await Query(db => db.FailedMessages.CountAsync()), Is.Zero, "a row committed without its body would name a body that was never stored"); + Assert.That(await CheckpointStore.Read(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), Is.Null, "a checkpoint past the batch would stop a retry from reading it again"); + } + } + + [Test] + public void A_document_that_cannot_be_prepared_fails_the_write_naming_the_row_and_category() + { + // The source rebuilds SendingEndpoint as EndpointDetails, so a value it left as stored is an engine defect the translator's cast rejects. + var document = new IngestedFailure().ToFailedMessage(); + document.ProcessingAttempts[^1].MessageMetadata["SendingEndpoint"] = new Dictionary<string, object> { ["Name"] = "Stored" }; + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(document), Checkpoint())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages).And.Contain(nameof(InvalidCastException)), + "the halt's LastError is built from this message, and without the row the operator cannot tell which document to look at"); + Assert.That(exception.InnerException, Is.TypeOf<InvalidCastException>()); + } + } + + [Test] + public void A_row_that_reaches_the_writer_without_a_body_fails_the_write_naming_the_row_and_category() + { + var document = new IngestedFailure().ToFailedMessage(); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(() => Target.Write(FailedMessagesCategory, BatchOf(RowOf(document, body: null)), Checkpoint())); + + Assert.That(exception.Message, Does.Contain($"'FailedMessages/{document.UniqueMessageId}'").And.Contain(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)); + } + + // Disposes the stream, because the file store holds the body file open until it is. + async Task<bool> BodyStored(string bodyId) + { + var stored = await BodyStore.ReadBody(bodyId); + if (stored is null) + { + return false; + } + + await stored.Stream.DisposeAsync(); + return true; + } + + async Task<T> Query<T>(Func<ServiceControlDbContext, Task<T>> query) + { + using var scope = ServiceProvider.CreateScope(); + return await query(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>()); + } + + static MigrationBatch BatchOf(params FailedMessage[] documents) => BatchOf([.. documents.Select(document => RowOf(document, SmallTextBody))]); + + static MigrationBatch BatchOf(params MigrationRow[] rows) => new(rows, rows[^1].SourceId); + + // Every row carries the values the source resolves, as a real source's rows do, and a retry-issued one finds its claim still held. + static MigrationRow RowOf(FailedMessage document, MigrationBody body) + { + var metadata = new Dictionary<string, object> + { + [MigrationMetadataKeys.StatusChangedAt] = new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), + [MigrationMetadataKeys.LastModified] = new DateTime(2026, 9, 2, 8, 0, 0, DateTimeKind.Utc) + }; + + if (document.Status == FailedMessageStatus.RetryIssued) + { + metadata[MigrationMetadataKeys.RetryClaimHeld] = true; + } + + return new($"FailedMessages/{document.UniqueMessageId}", document, metadata, body); + } + + // A null removes the flag, as a source that never resolved it would send the row. + static MigrationRow WithClaimHeld(MigrationRow row, bool? held) + { + var metadata = new Dictionary<string, object>(row.Metadata); + metadata.Remove(MigrationMetadataKeys.RetryClaimHeld); + + if (held is { } value) + { + metadata[MigrationMetadataKeys.RetryClaimHeld] = value; + } + + return row with { Metadata = metadata }; + } + + sealed class UnwritableBodyStore : IBodyStoragePersistence + { + public const string Failure = "The body store is unreachable"; + + public Task WriteBody(string bodyId, ReadOnlyMemory<byte> body, string contentType, CancellationToken cancellationToken = default) => Task.FromException(new IOException(Failure)); + + public Task<MessageBodyFileResult> ReadBody(string bodyId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + + public Task DeleteBodyIfExists(string bodyId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + } + + FailedMessagesWriter Writer() => new(ServiceProvider.GetRequiredService<IMigrationSqlDialect>(), BodyStore, ServiceProvider.GetRequiredService<EFPersisterSettings>()); + + static MigrationCheckpoint Checkpoint() => + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs new file mode 100644 index 0000000000..c4b017a7af --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentRowMapperTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class GroupCommentRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + var entity = GroupCommentRowMapper.Map(new GroupComment { Id = "group-1", Comment = "Waiting on the payments team" }); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs new file mode 100644 index 0000000000..9f50b13692 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/GroupCommentsMigrationTargetTests.cs @@ -0,0 +1,115 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; + +class GroupCommentsMigrationTargetTests : ErrorIngestionTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory GroupCommentsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.GroupComments)!; + + [Test] + public async Task A_comment_whose_group_id_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fits = new string('g', 64); + var tooLong = new string('g', 65); + var batch = BatchOf((fits, "Waiting on the payments team"), (tooLong, "Raised with the shipping team")); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new GroupCommentsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter(tooLong)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a group id exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { $"GroupComment/{tooLong}" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 1 }), "an id too long for its column would fail the whole insert and stop the category"); + Assert.That(await Target.Count(GroupCommentsCategory), Is.EqualTo(1), "--migration-verify reads this count"); + Assert.That(details.Single(), Does.EndWith($"takes at most 64 characters and the value holds 65, on document 'GroupComment/{tooLong}'")); + } + } + + [Test] + public async Task A_comment_an_earlier_pass_copied_keeps_its_text_and_is_counted_as_already_present() + { + var failure = new IngestedFailure(); + var group = failure.Groups.Single(); + await Ingest(failure); + + var first = await Target.Write(GroupCommentsCategory, BatchOf((group.Id, "Waiting on the payments team")), CheckpointAfter(group.Id)); + var again = await Target.Write(GroupCommentsCategory, BatchOf((group.Id, "Edited in RavenDB since")), first.Saved); + var view = (await GroupsStore.GetUnresolvedGroupsByClassifier(group.Type, null, new PagingInfo())).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(again.Copied, Is.Zero); + Assert.That(again.AlreadyPresent, Is.EqualTo(1), "a retry re-reads the category from the start and finds what the first pass copied"); + Assert.That((view.Id, view.Comment), Is.EqualTo((group.Id, "Waiting on the payments team")), "the copy only inserts what is missing and never changes a row"); + } + } + + [Test] + public async Task A_comment_whose_group_id_the_key_treats_as_an_earlier_ones_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf(("group-1", "Waiting on the payments team"), ("group-1 ", "Raised with the shipping team"), ("group-1", "Edited in RavenDB since")); + var keyIgnoresTrailingSpaces = dialect.KeyComparer(typeof(GroupCommentEntity), nameof(GroupCommentEntity.GroupId)).Equals("group-1", "group-1 "); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new GroupCommentsWriter(dialect), batch); + var result = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter("group-1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpaces ? new[] { "GroupComment/group-1 ", "GroupComment/group-1" } : ["GroupComment/group-1"]), + "an id that differs only in trailing spaces is dropped only where the key column ignores them, an exact repeat on every provider, and the log has to say so"); + Assert.That(merges.Select(merge => merge.Detail), Has.All.Contains("kept 'group-1'"), "the target logs this line, so it has to name the group id it kept"); + Assert.That((result.Copied, result.AlreadyPresent), Is.EqualTo(keyIgnoresTrailingSpaces ? (1, 2) : (2, 1)), + "on an empty target every row the insert drops is a merge, so the merges must account for every row counted as already present"); + } + } + + [Test] + public async Task A_comment_on_a_group_with_no_copied_message_is_reported_as_a_change_the_sweep_will_delete() + { + var failure = new IngestedFailure(); + var group = failure.Groups.Single(); + await Ingest(failure); + var longComment = new string('c', 150); + var batch = BatchOf((group.Id, "Waiting on the payments team"), ("group-of-archived-messages", longComment)); + + var changes = await PreparedSkips.ChangesFrom(ServiceProvider, new GroupCommentsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var first = await Target.Write(GroupCommentsCategory, batch, CheckpointAfter("group-of-archived-messages")); + var again = await Target.Write(GroupCommentsCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(changes, Is.EqualTo(new[] + { + ("GroupComment/group-of-archived-messages", + "The group 'group-of-archived-messages' has no copied failed message, so the retention sweep removes this comment shortly after ServiceControl opens. " + + "Its messages are archived or resolved, are gone from the source, or were skipped by the copy. " + + $"The comment was: '{longComment}'") + }), "a comment on a group with a copied message stays, so only the other is named, with the whole comment because the sweep deletes it"); + Assert.That((first.Copied, first.Changed), Is.EqualTo((2, 1))); + Assert.That(again.Changed, Is.Zero, "a re-read finds the comment already present, and this write copied nothing"); + } + } + + static MigrationBatch BatchOf(params (string GroupId, string Comment)[] comments) => + new([.. comments.Select(comment => new MigrationRow($"GroupComment/{comment.GroupId}", new GroupComment { Id = comment.GroupId, Comment = comment.Comment }, new Dictionary<string, object>()))], $"GroupComment/{comments[^1].GroupId}"); + + static MigrationCheckpoint CheckpointAfter(string groupId) => + new(MigrationCategoryIds.GroupComments, MigrationCategoryState.InProgress, $"GroupComment/{groupId}", 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs new file mode 100644 index 0000000000..f63dbdfdd0 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs @@ -0,0 +1,227 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class KnownEndpointsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + [Test] + public async Task A_known_endpoint_is_written_with_its_monitored_flag() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + var stored = (await MonitoringDataStore.GetAllKnownEndpoints()).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(stored.Monitored, Is.True); + Assert.That(stored.EndpointDetails.Name, Is.EqualTo("Sales.Orders")); + } + } + + [Test] + public async Task A_known_endpoint_already_in_the_target_keeps_its_flag_and_is_counted_as_already_present() + { + var details = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }; + await MonitoringDataStore.CreateIfNotExists(details); + + var endpoint = new KnownEndpoint { EndpointDetails = details, HostDisplayName = "SALES01", Monitored = true }; + var sourceId = $"KnownEndpoints/{details.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Single().Monitored, Is.False, "insert-if-absent never updates, so the flag the target already had stays"); + } + } + + [Test] + public async Task A_known_endpoint_with_no_name_or_no_host_is_skipped_as_required_value_missing() + { + var named = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" } }; + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", named), ("KnownEndpoints/2", nameless), ("KnownEndpoints/3", hostless)), CheckpointAfter("KnownEndpoints/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EquivalentTo(new[] { "KnownEndpoints/2", "KnownEndpoints/3" })); + Assert.That(result.SkipReasons[MigrationSkipReason.RequiredValueMissing], Is.EqualTo(2), "KnownEndpoints.Name and Host are NOT NULL, and a throw here would halt a required category"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { "Sales.Orders" })); + } + } + + [Test] + public async Task A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fitsExactly = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = new string('n', 450), HostId = Guid.NewGuid(), Host = new string('h', 450) } }; + var longName = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = new string('n', 451), HostId = Guid.NewGuid(), Host = "SALES02" } }; + var longHost = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = new string('h', 452) } }; + var batch = BatchOf(("KnownEndpoints/1", fitsExactly), ("KnownEndpoints/2", longName), ("KnownEndpoints/3", longHost)); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new KnownEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(KnownEndpointsCategory, batch, CheckpointAfter("KnownEndpoints/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "KnownEndpoints/2", "KnownEndpoints/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { new string('n', 450) })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<KnownEndpointEntity>(ServiceProvider, nameof(KnownEndpointEntity.Name))} takes at most 450 characters and the value holds 451, on document 'KnownEndpoints/2'", + $"{PreparedSkips.TableAndColumnOf<KnownEndpointEntity>(ServiceProvider, nameof(KnownEndpointEntity.Host))} takes at most 450 characters and the value holds 452, on document 'KnownEndpoints/3'" + })); + } + } + + [Test] + public async Task A_batch_of_nothing_but_skips_saves_the_cursor_and_writes_no_rows() + { + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", nameless), ("KnownEndpoints/2", hostless)), CheckpointAfter("KnownEndpoints/2")); + + var stored = await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Read(MigrationCategoryIds.KnownEndpoints); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero, "no key was looked up, so no row may be counted as one the target already held"); + Assert.That(result.Skipped, Is.EqualTo(2)); + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty); + Assert.That(stored.Cursor, Is.EqualTo("KnownEndpoints/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public void A_batch_reporting_more_copied_and_skipped_rows_than_it_held_is_refused() + { + // The guard only counts rows, so what the documents hold cannot change its answer. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + + var exception = Assert.Throws<InvalidOperationException>(() => EFCoreMigrationTarget.AlreadyPresentIn(KnownEndpointsCategory, batch, copied: 2, skipped: 1)); + + Assert.That(exception.Message, Does.Contain("copied 2").And.Contain("skipped 1").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + [Test] + public async Task A_batch_whose_checkpoint_save_fails_leaves_no_rows_behind() + { + // The rows save first and the checkpoint second, so this is the only order in which the two can part + // company: a stale version fails the checkpoint after the endpoint row is already in the transaction. + var store = ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + await store.Upsert(CheckpointAfter("KnownEndpoints/0")); + + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(async () => + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty, "the rows and the checkpoint commit together, so a failed checkpoint takes the rows with it"); + Assert.That((await store.Read(MigrationCategoryIds.KnownEndpoints)).Cursor, Is.EqualTo("KnownEndpoints/0"), "the stored cursor must still describe the rows the target actually holds"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.KnownEndpoints.AsNoTracking().SingleAsync()); + } + + + [Test] + public async Task A_batch_spanning_several_statements_counts_what_each_statement_inserted() + { + var rows = Enumerable.Range(0, 500) + .Select(index => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = $"Endpoint{index}", HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = index % 2 == 0 }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint)) + .ToArray(); + var batch = BatchOf(rows); + + var first = await Target.Write(KnownEndpointsCategory, batch, CheckpointAfter(batch.Cursor)); + // The engine carries the committed checkpoint into the next write, and the version guard refuses anything else. + var second = await Target.Write(KnownEndpointsCategory, batch, first.Saved with { Cursor = batch.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(500)); + Assert.That(second.Copied, Is.Zero, "every key is present on the second write, so no statement may report a row it did not insert"); + Assert.That(second.AlreadyPresent, Is.EqualTo(500)); + } + } + + + [Test] + public void A_batch_whose_writer_neither_prepared_nor_skipped_a_row_is_refused() + { + // The already-present count is a subtraction, so it only catches a writer that over-reports. A row dropped in silence under-reports, and without this guard it is counted as a row the target already held. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + var prepared = new PreparedBatch((_, _) => Task.FromResult(1), PreparedRowCount: 1, Skips: [], Merges: []); + + var exception = Assert.Throws<InvalidOperationException>(() => EFCoreMigrationTarget.AccountForEveryRow(KnownEndpointsCategory, batch, prepared)); + + Assert.That(exception.Message, Does.Contain("prepared 1").And.Contain("skipped 0").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs new file mode 100644 index 0000000000..7b0661a1b2 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingEndpointsMigrationTargetTests.cs @@ -0,0 +1,147 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingEndpointsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory LicensingEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + + [Test] + public async Task Count_is_the_licensing_endpoint_rows_the_target_holds() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Audit) { SanitizedName = "sales.orders" }); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 2), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Audit, new DateOnly(2026, 1, 1), 5); + + Assert.That(await Target.Count(LicensingEndpointsCategory), Is.EqualTo(2), "one row per name and source, and not the three throughput days that hang off them"); + } + + [Test] + public async Task A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var fitsExactly = new Endpoint(new string('n', 300), ThroughputSource.Broker) { SanitizedName = new string('s', 450) }; + var longName = new Endpoint(new string('n', 301), ThroughputSource.Audit) { SanitizedName = "sales.orders" }; + var longSanitizedName = new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = new string('s', 451) }; + var batch = BatchOf(("Endpoint/1", fitsExactly), ("Endpoint/2", longName), ("Endpoint/3", longSanitizedName)); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(LicensingEndpointsCategory, batch, CheckpointAfter("Endpoint/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "Endpoint/2", "Endpoint/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await LicensingDataStore.GetAllEndpoints(includePlatformEndpoints: true)).Select(endpoint => endpoint.Id.Name), Is.EqualTo(new[] { new string('n', 300) })); + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details[0], Does.EndWith("takes at most 300 characters and the value holds 301, on document 'Endpoint/2'")); + Assert.That(details[1], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Endpoint/3'")); + } + } + + [Test] + public async Task Two_names_differing_only_in_a_trailing_space_in_one_batch_are_reported_as_one_merge_where_the_key_folds_them() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var keyIgnoresTrailingSpace = KeyIgnoresTrailingSpace(dialect); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { "Sales.Orders/Broker" } : Array.Empty<string>()), + "the row whose id sorts first is kept, and the other is reported only where the key folds a trailing space"); + + if (keyIgnoresTrailingSpace) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders ', 'Broker' and dropped the row for 'sales.orders', 'Broker'"), "the line names both endpoints and the one kept"); + } + } + } + + [Test] + public async Task A_name_differing_only_in_a_trailing_space_from_a_stored_endpoint_is_reported_as_a_merge_where_the_key_folds_them() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var first = await Target.Write(LicensingEndpointsCategory, BatchOf(("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })), CheckpointAfter("Sales.Orders/Broker")); + var second = BatchOf(("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var keyIgnoresTrailingSpace = KeyIgnoresTrailingSpace(dialect); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), second); + var result = await Target.Write(LicensingEndpointsCategory, second, first.Saved with { Cursor = second.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { "Sales.Orders /Broker" } : Array.Empty<string>())); + + if (keyIgnoresTrailingSpace) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders', 'Broker' and dropped the row for 'sales.orders ', 'Broker'"), "the stored endpoint is the one kept"); + } + + Assert.That(result.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the folded endpoint is counted as already present"); + } + } + + [Test] + public async Task An_exact_replay_of_a_stored_endpoint_is_already_present_and_not_a_merge() + { + var batch = BatchOf(("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + var first = await Target.Write(LicensingEndpointsCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var again = await Target.Write(LicensingEndpointsCategory, batch, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges, Is.Empty, "a retry re-reads the rows an earlier pass copied, and the stored row is that same endpoint"); + Assert.That((again.Copied, again.AlreadyPresent, again.Merged), Is.EqualTo((0, 1, 0))); + } + } + + [Test] + public async Task Two_names_differing_only_in_case_are_reported_as_one_merge_on_every_provider() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("SALES.ORDERS/Broker", new Endpoint("SALES.ORDERS", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new LicensingEndpointsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "Sales.Orders/Broker" }), "the product lower-cases the name, so both providers store one row"); + Assert.That(merges[0].Detail, Does.Contain("kept 'sales.orders', 'Broker' and dropped the row for 'sales.orders', 'Broker'")); + } + } + + static bool KeyIgnoresTrailingSpace(IMigrationSqlDialect dialect) => + dialect.KeyComparer(typeof(LicensingEndpointEntity), nameof(LicensingEndpointEntity.NormalizedName)).Equals("a", "a "); + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs new file mode 100644 index 0000000000..b91e3c4778 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingMigrationMapperCoverageTests.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class LicensingMigrationMapperCoverageTests : PersistenceTestBase +{ + [Test] + public async Task The_endpoint_mapper_sets_every_mapped_property() + { + var endpoint = new Endpoint("Sales.Orders", ThroughputSource.Audit) + { + SanitizedName = "Sales.Orders", + EndpointIndicators = [EndpointIndicator.PlatformEndpoint.ToString()], + UserIndicator = "Sales team", + Scope = "Sales", + CreatedDate = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc) + }; + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(await Model(), LicensingEndpointRowMapper.Map(endpoint)); + } + + [Test] + public async Task The_throughput_mapper_sets_every_mapped_property() + { + var day = new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Audit), new DateOnly(2026, 1, 1), 100); + + var row = LicensingThroughputRowMapper.Map(day); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(await Model(), row); + using (Assert.EnterMultipleScope()) + { + Assert.That(row.NormalizedName, Is.EqualTo("sales.orders"), "the key must be the name the endpoint row is stored under, or the foreign key refuses the day"); + Assert.That(row.ThroughputSource, Is.EqualTo(ThroughputSource.Audit)); + Assert.That(row.DateUtc, Is.EqualTo(new DateOnly(2026, 1, 1))); + Assert.That(row.MessageCount, Is.EqualTo(100)); + } + } + + async Task<IModel> Model() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + return scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs new file mode 100644 index 0000000000..56edd2b44f --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/LicensingThroughputMigrationTargetTests.cs @@ -0,0 +1,297 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class LicensingThroughputMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory LicensingThroughputCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingThroughput); + + [Test] + public async Task Count_is_the_daily_throughput_rows_the_target_holds() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + await LicensingDataStore.SaveEndpoint(new Endpoint("Billing", ThroughputSource.Broker) { SanitizedName = "billing" }); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + await LicensingDataStore.RecordEndpointThroughput("Sales.Orders", ThroughputSource.Broker, new DateOnly(2026, 1, 2), 5); + await LicensingDataStore.RecordEndpointThroughput("Billing", ThroughputSource.Broker, new DateOnly(2026, 1, 1), 5); + + Assert.That(await Target.Count(LicensingThroughputCategory), Is.EqualTo(3), "one row per endpoint and day, and not the two endpoint rows they hang off"); + } + + [Test] + public async Task A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100)), + ("long/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier(new string('n', 301), ThroughputSource.Broker), day, 250))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new LicensingThroughputWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("long/Broker|2026-01-01")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "long/Broker|2026-01-01" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 1 }), "the target cannot hold an endpoint with this name, so the day must be skipped here rather than fail the whole batch at the endpoint check"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<LicensingEndpointThroughputEntity>(ServiceProvider, nameof(LicensingEndpointThroughputEntity.NormalizedName))} takes at most 300 characters and the value holds 301, on document 'long/Broker|2026-01-01'" + })); + } + } + + [Test] + public async Task A_day_repeated_in_one_batch_keeps_its_last_row_and_counts_the_other_as_already_present() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01|b", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250)), + ("Sales.Orders/Broker|2026-01-01|a", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100))); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker|2026-01-01|b")); + await using var scope = ServiceProvider.CreateAsyncScope(); + var stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().LicensingEndpointThroughput.Select(row => row.MessageCount).SingleAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the repeat is the same day of the same endpoint, so it is dropped before the statement and counted"); + Assert.That(stored, Is.EqualTo(250), "the row whose id sorts last by ordinal order sets the day rather than adding to it, whatever order the batch arrives in"); + } + } + + [Test] + public async Task Days_of_two_endpoints_the_key_treats_as_one_are_added_when_they_arrive_in_one_batch() + { + var endpoints = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + var endpointBatch = BatchOf( + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100)), + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))); + + var first = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var keyIgnoresTrailingSpace = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database.ProviderName!.Contains("SqlServer"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 2)); + Assert.That(first.AlreadyPresent, Is.EqualTo(keyIgnoresTrailingSpace ? 1 : 0), "the folded endpoint's days are summed into the kept row, and the endpoint writer reports the fold"); + Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 350L } : new[] { 100L, 250L }), "SQL Server's key makes the two one day, which holds both counts, as the product's own throughput write would"); + } + } + + [Test] + public async Task Days_of_two_endpoints_differing_only_in_case_are_added_when_they_arrive_in_one_batch() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + var batch = BatchOf( + ("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100)), + ("SALES.ORDERS/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("SALES.ORDERS", ThroughputSource.Broker), day, 250))); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("SALES.ORDERS/Broker")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.AlreadyPresent, Is.EqualTo(1), "the folded endpoint's days are summed into the kept row, and the endpoint writer reports the fold"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 350L }), "two source endpoints store the same lower-cased name, so their days are added, not one kept as a repeat of the other"); + } + } + + [Test] + public async Task A_later_batch_adds_its_count_to_the_day_an_earlier_batch_stored_for_a_name_differing_only_in_case() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var day = new DateOnly(2026, 1, 1); + + var first = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 100))), CheckpointAfter("Sales.Orders/Broker")); + var second = await Target.Write(LicensingThroughputCategory, BatchOf(("SALES.ORDERS/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("SALES.ORDERS", ThroughputSource.Broker), day, 250))), first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.EqualTo(1), "an update is a row the statement wrote"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 350L }), "both endpoints store the same lower-cased name, so the second batch adds its count to the day rather than replacing it"); + } + } + + [Test] + public async Task A_batch_written_again_from_the_checkpoint_it_already_extended_is_refused_and_adds_nothing() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var checkpointToExtend = CheckpointAfter("Sales.Orders/Broker"); + + await Target.Write(LicensingThroughputCategory, batch, checkpointToExtend); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.Write(LicensingThroughputCategory, batch, checkpointToExtend)); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the add and the checkpoint commit together, so a replay of a committed batch rolls back instead of counting the day twice"); + } + + [Test] + public async Task A_retry_deletes_the_copied_days_so_copying_again_gives_the_same_totals() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var first = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + var reset = await Target.ResetForRetry(ResetForRetry(first.Saved)); + var afterReset = await StoredCounts(); + await Target.Write(LicensingThroughputCategory, batch, reset); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterReset, Is.Empty, "the reset deletes the days the first copy added, because copying them again adds each count a second time"); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "copying again after the retry gives the totals one copy gave, not double"); + } + } + + [Test] + public async Task A_stale_retry_reset_deletes_no_copied_day() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var stale = (await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker"))).Saved; + await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Upsert(stale); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => Target.ResetForRetry(ResetForRetry(stale))); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the delete shares the reset's transaction, so a refused reset leaves every copied day"); + } + + [Test] + public async Task A_retry_reset_after_ServiceControl_opened_on_the_target_is_refused_and_deletes_nothing() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var batch = BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), new DateOnly(2026, 1, 1), 100))); + var copied = (await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker"))).Saved; + await ServiceProvider.GetRequiredService<IMigrationTargetReadiness>().RecordHostOpened(); + + var refused = Assert.ThrowsAsync<InvalidOperationException>(() => Target.ResetForRetry(ResetForRetry(copied))); + var stored = await ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Read(MigrationCategoryIds.LicensingThroughput); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refused.Message, Does.Contain(MigrationCategoryIds.LicensingThroughput).And.Contain("only possible before ServiceControl first opens on this database")); + Assert.That(await StoredCounts(), Is.EqualTo(new[] { 100L }), "the days may now include counts the running instance collected, so deleting them would lose live throughput"); + Assert.That((stored!.Version, stored.State, stored.Cursor, stored.CopiedCount), Is.EqualTo((copied.Version, copied.State, copied.Cursor, copied.CopiedCount)), "a refused reset saves nothing"); + } + } + + [Test] + public async Task A_retry_reset_that_deletes_nothing_still_succeeds_after_ServiceControl_opened_on_the_target() + { + var checkpoints = ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + var stopped = await checkpoints.Upsert(new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.InProgress, "FailedMessages/1", 1, 0, null, null, null, null, null, "RavenDB is unreachable")); + await ServiceProvider.GetRequiredService<IMigrationTargetReadiness>().RecordHostOpened(); + + var reset = await Target.ResetForRetry(ResetForRetry(stopped)); + + Assert.That((reset.State, reset.Cursor, reset.LastError), Is.EqualTo((MigrationCategoryState.NotStarted, (string)null, (string)null)), "an optional category copies after ServiceControl opens, and its reset deletes nothing, so it is still allowed"); + } + + [Test] + public async Task One_endpoint_with_more_days_than_one_statement_takes_stores_every_day_with_its_own_count() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + int dayCount; + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + dayCount = dialect.RowsPerStatementFor<LicensingEndpointThroughputEntity>(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>()) + 1; + } + + var firstDay = new DateOnly(2026, 1, 1); + var days = Enumerable.Range(0, dayCount).Select(offset => (Date: firstDay.AddDays(offset), Count: 1000L + offset)).ToArray(); + var batch = BatchOf([.. days.Select(day => ($"Sales.Orders/Broker|{day.Date:yyyy-MM-dd}", (object)new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day.Date, day.Count)))]); + + var result = await Target.Write(LicensingThroughputCategory, batch, CheckpointAfter("Sales.Orders/Broker")); + + await using var readScope = ServiceProvider.CreateAsyncScope(); + var stored = await readScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().LicensingEndpointThroughput + .OrderBy(row => row.DateUtc) + .Select(row => new { row.DateUtc, row.MessageCount }) + .ToListAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(dayCount), "the batch runs as two statements, and Copied adds up what both wrote"); + Assert.That(result.AlreadyPresent, Is.Zero, "a day the second statement wrote must not be counted as already present"); + Assert.That(stored.Select(row => (row.DateUtc, row.MessageCount)), Is.EqualTo(days), "every day is stored with its own count, including the day past the first statement"); + } + } + + [Test] + public async Task A_later_batch_adds_to_a_day_stored_under_a_name_that_differs_only_by_a_trailing_space() + { + var endpoints = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.LicensingEndpoints); + var endpointBatch = BatchOf( + ("Sales.Orders /Broker", new Endpoint("Sales.Orders ", ThroughputSource.Broker) { SanitizedName = "sales.orders" }), + ("Sales.Orders/Broker", new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" })); + await Target.Write(endpoints, endpointBatch, new MigrationCheckpoint(endpoints.Id, MigrationCategoryState.InProgress, endpointBatch.Cursor, 0, 0, null, null, null, null, null, null)); + var day = new DateOnly(2026, 1, 1); + + var first = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders /Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders ", ThroughputSource.Broker), day, 100))), CheckpointAfter("Sales.Orders /Broker")); + var second = await Target.Write(LicensingThroughputCategory, BatchOf(("Sales.Orders/Broker|2026-01-01", new ThroughputDayRow(new EndpointIdentifier("Sales.Orders", ThroughputSource.Broker), day, 250))), first.Saved); + + await using var scope = ServiceProvider.CreateAsyncScope(); + var keyIgnoresTrailingSpace = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database.ProviderName!.Contains("SqlServer"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(second.Copied, Is.EqualTo(1), "the second batch's day is matched back to the row that produced it, not refused because the stored name keeps its trailing space"); + Assert.That(await StoredCounts(), Is.EqualTo(keyIgnoresTrailingSpace ? new[] { 350L } : new[] { 100L, 250L }), "SQL Server's key makes the two names one day, which the later count adds to; PostgreSQL's keeps two"); + } + } + + async Task<long[]> StoredCounts() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + return await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().LicensingEndpointThroughput.Select(row => row.MessageCount).OrderBy(count => count).ToArrayAsync(); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + // The reset --migration-retry makes. + static MigrationCheckpoint ResetForRetry(MigrationCheckpoint checkpoint) => checkpoint with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0, + SkipReasons = null, + SettledAt = null, + LastError = null + }; + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs new file mode 100644 index 0000000000..a6e99bb5bb --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectRowMapperTests.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + var entity = MessageRedirectRowMapper.Map(new MessageRedirect + { + FromPhysicalAddress = "Sales@OLDHOST", + ToPhysicalAddress = "Sales@NEWHOST", + LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) + }); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, entity); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs new file mode 100644 index 0000000000..467cedecc8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MessageRedirectsMigrationTargetTests.cs @@ -0,0 +1,75 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.MessageRedirects; + +class MessageRedirectsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory MessageRedirectsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + + [Test] + public async Task A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var batch = BatchOf( + ("Redirect/1", Redirect(new string('f', 450), new string('t', 450))), + ("Redirect/2", Redirect(new string('f', 451), "Sales@NEWHOST")), + ("Redirect/3", Redirect("Billing@OLDHOST", new string('t', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new MessageRedirectsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(MessageRedirectsCategory, batch, CheckpointAfter("Redirect/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "Redirect/2", "Redirect/3" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 2 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That((await MessageRedirectsDataStore.GetRedirects()).Select(redirect => redirect.FromPhysicalAddress), Is.EqualTo(new[] { new string('f', 450) })); + Assert.That(details, Has.Count.EqualTo(2)); + Assert.That(details[0], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/2'")); + Assert.That(details[1], Does.EndWith("takes at most 450 characters and the value holds 451, on document 'Redirect/3'")); + } + } + + [Test] + public async Task A_redirect_whose_address_the_key_treats_as_an_earlier_ones_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("Redirect/1", Redirect("Sales@HOST", "Sales@NEWHOST")), + ("Redirect/2", Redirect("Sales@HOST", "Sales@OTHERHOST")), + ("Redirect/3", Redirect("sales@host", "Sales@NEWHOST"))); + var keyIgnoresCase = dialect.KeyComparer(typeof(MessageRedirectEntity), nameof(MessageRedirectEntity.FromPhysicalAddress)).Equals("Sales@HOST", "sales@host"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new MessageRedirectsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(keyIgnoresCase ? new[] { "Redirect/2", "Redirect/3" } : new[] { "Redirect/2" }), + "a second copy of an address is dropped on every database, and a case-only variant only where the key column ignores case"); + Assert.That(merges[0].Detail, Does.Contain("kept 'Sales@HOST'"), "the target logs this line, so it has to name the address it kept"); + } + } + + static MessageRedirect Redirect(string from, string to) => + new() { FromPhysicalAddress = from, ToPhysicalAddress = to, LastModified = new DateTime(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc) }; + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs new file mode 100644 index 0000000000..7fa6186c1e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; + +class MigrationCategoryWritersTests : PersistenceTestBase +{ + [Test] + public void Every_writer_claims_a_category_the_registry_knows() + { + var unknown = ServiceProvider.GetRequiredService<IMigrationTarget>().SupportedCategoryIds + .Where(id => MigrationCategoryRegistry.Find(id) is null) + .ToArray(); + + Assert.That(unknown, Is.Empty, "a writer for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_writer_derives_from_the_typed_base() + { + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(writers, Is.Not.Empty, "the test proves nothing if it finds no writer"); + Assert.That(writers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryWriter<>))).Select(type => type.Name), Is.Empty, + "a writer that implements the interface directly skips the named cast and can declare a DocumentType it does not take"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); + + [Test] + public async Task Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type() + { + var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); + await target.Open(); + + using (Assert.EnterMultipleScope()) + { + foreach (var (categoryId, document) in target.SupportedCategoryIds.SelectMany(id => new (string, object)[] { (id, new object()), (id, null) })) + { + var category = MigrationCategoryRegistry.Find(categoryId)!; + var batch = new MigrationBatch([new MigrationRow("Wrong/1", document, new Dictionary<string, object>())], "Wrong/1"); + var checkpoint = new MigrationCheckpoint(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + var exception = Assert.ThrowsAsync<InvalidCastException>(() => target.Write(category, batch, checkpoint)); + + Assert.That(exception!.Message, Does.Contain(categoryId).And.Contain("Wrong/1").And.Contain(target.DocumentTypes[categoryId].FullName), "an engine halt quotes this message, so it has to say which pair disagrees"); + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs new file mode 100644 index 0000000000..d3ede8f8ec --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections; +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using NUnit.Framework; + +static class MigrationEntityCoverage +{ + public static void AssertEveryMappedPropertyIsSet<TEntity>(IModel model, TEntity entity, params string[] legitimatelyDefault) + { + var entityType = model.FindEntityType(typeof(TEntity)) + ?? throw new ArgumentException($"{typeof(TEntity).Name} is not an entity in the EF Core model.", nameof(entity)); + + // Reflection rather than a new() constraint, which an entity with required members cannot satisfy. + var defaults = Activator.CreateInstance(typeof(TEntity)); + + // Every property in this model that is not ValueGenerated.Never is an identity column the database assigns. + var unset = entityType.GetProperties() + .Where(property => property.ValueGenerated == ValueGenerated.Never && !legitimatelyDefault.Contains(property.Name)) + .Where(property => SameValue(property.GetGetter().GetClrValue(entity), property.GetGetter().GetClrValue(defaults))) + .Select(property => property.Name) + .ToArray(); + + Assert.That(unset, Is.Empty, $"{typeof(TEntity).Name} has mapped properties equal to a default-constructed {typeof(TEntity).Name}. Set each from the source document, feed the test a document whose values all differ from those defaults, or name the property in legitimatelyDefault if RavenDB genuinely holds that value."); + } + + // Collections compare by item, and an empty one is unset whether or not the entity initialises it. + static bool SameValue(object actual, object defaultValue) => + actual is IEnumerable items and not string + ? !items.Cast<object>().Any() || (defaultValue is IEnumerable defaultItems && items.Cast<object>().SequenceEqual(defaultItems.Cast<object>())) + : Equals(actual, defaultValue); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs new file mode 100644 index 0000000000..d5d0fedf42 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DbContexts; + +class MigrationEntityCoverageClaimTests : PersistenceTestBase +{ + [Test] + public void Every_property_the_database_fills_in_is_a_key_it_assigns_on_insert() + { + using var scope = ServiceProvider.CreateScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + var databaseFilled = model.GetEntityTypes() + .SelectMany(entityType => entityType.GetProperties()) + .Where(property => property.ValueGenerated != ValueGenerated.Never) + .ToArray(); + + var notKeysAssignedOnInsert = databaseFilled + .Where(property => !property.IsPrimaryKey() || property.ValueGenerated != ValueGenerated.OnAdd) + .Select(property => $"{property.DeclaringType.ClrType.Name}.{property.Name} is {property.ValueGenerated}") + .ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(databaseFilled, Is.Not.Empty, "the model no longer has a single property the database fills in, so this test is watching nothing. Either the identity keys have gone, or the model was never built."); + Assert.That(notKeysAssignedOnInsert, Is.Empty, "MigrationEntityCoverage checks only the properties that are ValueGenerated.Never, on the claim that every other one is a key the database assigns on insert. These properties break that claim, so the coverage check silently stops watching them and a migration can copy a row with them left unset."); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs new file mode 100644 index 0000000000..629d6d1684 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationInsertDateKeyTests.cs @@ -0,0 +1,36 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationInsertDateKeyTests : PersistenceTestBase +{ + [Test] + public async Task A_row_whose_key_holds_a_date_is_matched_back_to_the_row_that_produced_it() + { + await LicensingDataStore.SaveEndpoint(new Endpoint("Sales.Orders", ThroughputSource.Broker) { SanitizedName = "sales.orders" }); + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var day = new LicensingEndpointThroughputEntity { NormalizedName = "sales.orders", ThroughputSource = ThroughputSource.Broker, DateUtc = new DateOnly(2026, 1, 1), MessageCount = 100 }; + + await using var scope = ServiceProvider.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + await dialect.Open(dbContext); + + var inserted = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + var rows = await dialect.InsertMissing(dbContext, [day]); + await transaction.CommitAsync(); + return rows; + }); + + Assert.That(inserted, Is.EqualTo(new[] { day }), "a date key comes back from the database as DateTime unless it is read as the type its mapping stores, and the exact lookup then refuses the row"); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs new file mode 100644 index 0000000000..3e942eace6 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -0,0 +1,130 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationTargetCoverageTests : PersistenceTestBase +{ + [Test] + public async Task Every_category_the_target_supports_has_a_batch_size_and_a_count() + { + var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); + + Assert.That(target.SupportedCategoryIds, Is.SupersetOf(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages }), "the test proves nothing if the target supports no category"); + + foreach (var id in target.SupportedCategoryIds) + { + var category = MigrationCategoryRegistry.Find(id)!; + + Assert.That(await target.BatchSizeFor(category, CancellationToken.None), Is.Positive, $"{id}: every supported category has a batch size"); + Assert.That(await target.Count(category, CancellationToken.None), Is.Zero, id); + } + } + + [Test] + public void Every_category_the_target_supports_has_a_test_for_a_value_longer_than_its_column() + { + var target = ServiceProvider.GetRequiredService<IMigrationTarget>(); + + Assert.That(target.SupportedCategoryIds, Is.EquivalentTo(ValueTooLongTests.Keys.Concat(NoValueCanBeTooLong)), + "a writer that does not run the length check sends an over-long value to the insert, which fails the whole statement and halts the category; give the new category a ValueTooLong test like those listed here, then add it to this list"); + } + + [Test] + public async Task The_settings_value_column_has_no_maximum_length_so_no_setting_can_be_too_long() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + Assert.That(model.FindEntityType(typeof(SettingEntity))!.FindProperty(nameof(SettingEntity.Value))!.GetMaxLength(), Is.Null, "NoValueCanBeTooLong exempts the settings categories because of this; give them a value-too-long test if the column gains a limit"); + } + + [TestCase(typeof(CustomCheckEntity))] + [TestCase(typeof(FailedErrorImportEntity))] + public async Task No_text_column_of_the_entity_has_a_maximum_length_so_no_value_can_be_too_long(Type entity) + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var entityType = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model.FindEntityType(entity)!; + + Assert.That( + entityType.GetProperties().Where(property => property.ClrType == typeof(string) && property.GetMaxLength() is not null).Select(property => $"{entityType.GetTableName()}.{property.GetColumnName()}"), + Is.Empty, + "NoValueCanBeTooLong exempts this entity's category because of this; give it a value-too-long test if a column gains a limit"); + } + + [Test] + public async Task Every_writer_whose_rows_a_re_read_cannot_recognise_deletes_them_for_a_retry_and_no_other_writer_does() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)); + + Assert.That(writers, Is.EquivalentTo(EntitiesEachWriterWrites.Keys), "list the entities a new writer writes here, so this test can tell whether a retry must delete its rows"); + + using (Assert.EnterMultipleScope()) + { + foreach (var (writer, entities) in EntitiesEachWriterWrites) + { + var storeKeyed = entities.Any(entity => model.FindEntityType(entity)!.FindPrimaryKey()!.Properties.Any(property => property.ValueGenerated != ValueGenerated.Never)); + + Assert.That(typeof(IStoreKeyedCategoryWriter).IsAssignableFrom(writer), Is.EqualTo(storeKeyed || WritersThatAddToRowsTheyCopied.Contains(writer)), + $"{writer.Name}: a retry keeps the rows of a writer that is not an IStoreKeyedCategoryWriter, so a re-read adds rows with store keys or added counts a second time, and a writer whose rows a re-read finds already present must not delete"); + } + } + } + + // None of these copies a source value into a text column with a maximum length, which the two no-maximum-length tests above pin. + static readonly string[] NoValueCanBeTooLong = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate, MigrationCategoryIds.CustomChecks, MigrationCategoryIds.FailedErrorImports]; + + // Uses nameof, so renaming or deleting one of these tests fails the build instead of leaving this list out of date. + static readonly Dictionary<string, string> ValueTooLongTests = new() + { + [MigrationCategoryIds.KnownEndpoints] = nameof(KnownEndpointsMigrationTargetTests.A_known_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.EndpointSettings] = nameof(EndpointSettingsMigrationTargetTests.A_setting_whose_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages] = nameof(FailedMessagesMigrationTargetTests.A_message_or_group_value_longer_than_its_column_skips_the_message_whole_and_the_rest_copy), + [MigrationCategoryIds.LicensingEndpoints] = nameof(LicensingEndpointsMigrationTargetTests.A_licensing_endpoint_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.LicensingThroughput] = nameof(LicensingThroughputMigrationTargetTests.A_day_whose_endpoint_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.MessageRedirects] = nameof(MessageRedirectsMigrationTargetTests.A_redirect_with_an_address_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.Subscriptions] = nameof(SubscriptionsMigrationTargetTests.A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy), + [MigrationCategoryIds.RetryOperations] = nameof(RetryOperationsMigrationTargetTests.A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.PendingIntegrationEvents] = nameof(PendingIntegrationEventsMigrationTargetTests.A_request_whose_context_type_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy), + [MigrationCategoryIds.GroupComments] = nameof(GroupCommentsMigrationTargetTests.A_comment_whose_group_id_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy) + }; + + // Their keys come from the source, but a re-read adds its counts to the stored ones, so a re-read cannot find them already present. + static readonly Type[] WritersThatAddToRowsTheyCopied = [typeof(LicensingThroughputWriter)]; + + // The writers do not say which entities they write, so this list does. + static readonly Dictionary<Type, Type[]> EntitiesEachWriterWrites = new() + { + [typeof(KnownEndpointsWriter)] = [typeof(KnownEndpointEntity)], + [typeof(EndpointSettingsWriter)] = [typeof(EndpointSettingsEntity)], + [typeof(FailedMessagesWriter)] = [typeof(FailedMessageEntity), typeof(FailedMessageGroupEntity)], + [typeof(LicensingEndpointsWriter)] = [typeof(LicensingEndpointEntity)], + [typeof(LicensingThroughputWriter)] = [typeof(LicensingEndpointThroughputEntity)], + [typeof(LicensingReportMasksWriter)] = [typeof(SettingEntity)], + [typeof(LicensedEndpointDetailsWriter)] = [typeof(SettingEntity)], + [typeof(MessageRedirectsWriter)] = [typeof(MessageRedirectEntity)], + [typeof(SubscriptionsWriter)] = [typeof(SubscriptionEntity)], + [typeof(NotificationSettingsWriter)] = [typeof(SettingEntity)], + [typeof(TrialEndDateWriter)] = [typeof(SettingEntity)], + [typeof(RetryOperationsWriter)] = [typeof(HistoricRetryOperationEntity), typeof(UnacknowledgedRetryOperationEntity)], + [typeof(PendingIntegrationEventsWriter)] = [typeof(ExternalIntegrationDispatchRequestEntity)], + [typeof(CustomChecksWriter)] = [typeof(CustomCheckEntity)], + [typeof(FailedErrorImportsWriter)] = [typeof(FailedErrorImportEntity)], + [typeof(GroupCommentsWriter)] = [typeof(GroupCommentEntity)] + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs new file mode 100644 index 0000000000..caffb35e0e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs @@ -0,0 +1,226 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationTargetReadinessTests : PersistenceTestBase +{ + IMigrationTargetReadiness Readiness => ServiceProvider.GetRequiredService<IMigrationTargetReadiness>(); + + [Test] + public void The_target_contributes_the_three_checks_only_it_can_make() => + Assert.That( + Readiness.ContributedChecks().Select(check => check.GetType()), + Is.EqualTo(new[] { typeof(SchemaIsCurrentCheck), typeof(TargetHoldsNoServiceControlDataCheck), typeof(BodyStorageIsWritableCheck) })); + + [Test] + public void Every_contributed_check_passes_against_a_migrated_database() + { + using (Assert.EnterMultipleScope()) + { + foreach (var check in Readiness.ContributedChecks()) + { + Assert.DoesNotThrowAsync(() => check.Run(), $"a healthy target must pass every contributed check, and it failed '{check.Name}'"); + } + } + } + + [Test] + public async Task A_target_holding_a_row_is_refused_while_no_checkpoint_exists() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var exception = Assert.ThrowsAsync<Exception>(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName<EndpointSettingsEntity>()).And.Contain("--setup")); + } + + // A plain start writes this row first, so a key someone later ignores fails here before it reaches a customer. + [Test] + public async Task A_settings_row_counts_as_data() + { + await ServiceProvider.GetRequiredService<ITrialLicenseDataProvider>().StoreTrialEndDate(new DateOnly(2030, 1, 1)); + + var exception = Assert.ThrowsAsync<Exception>(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName<SettingEntity>())); + } + + [Test] + public async Task A_target_with_a_required_checkpoint_row_is_not_judged() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + [Test] + public async Task A_row_under_an_id_this_build_does_not_know_counts_as_a_started_copy() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint("SomeCategoryFromANewerBuild", MigrationCategoryState.InProgress); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + // --migration-abandon writes this row before any copy has run, so on a database that already served on SQL it must not switch the check off. + [Test] + public async Task An_abandoned_optional_row_alone_does_not_skip_the_check() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.Abandoned); + + var exception = Assert.ThrowsAsync<Exception>(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName<EndpointSettingsEntity>())); + } + + [Test] + public void Every_mapped_entity_but_the_checkpoint_is_judged() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + var mapped = dbContext.Model.GetEntityTypes().Select(type => type.ClrType).Where(type => type != typeof(MigrationCheckpointEntity)); + + Assert.That(TargetHoldsNoServiceControlDataCheck.Tables.Select(entry => entry.Entity), Is.EquivalentTo(mapped)); + } + + IMigrationStartupCheck TargetCheck => Readiness.ContributedChecks().OfType<TargetHoldsNoServiceControlDataCheck>().Single(); + + Task SeedCheckpoint(string categoryId, MigrationCategoryState state) => + ServiceProvider.GetRequiredService<IMigrationCheckpointStore>().Upsert(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + + string TableName<T>() + { + using var scope = ServiceProvider.CreateScope(); + + return scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model.FindEntityType(typeof(T))!.GetTableName()!; + } + + [Test] + public async Task The_schema_check_refuses_a_database_whose_migrations_have_not_been_applied() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService<IHistoryRepository>(); + + foreach (var row in await history.GetAppliedMigrationsAsync()) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + var exception = Assert.ThrowsAsync<Exception>(() => Readiness.ContributedChecks().OfType<SchemaIsCurrentCheck>().Single().Run()); + + Assert.That(exception.Message, Does.Contain(dbContext.Database.GetMigrations().First()).And.Contain("--setup")); + } + + [Test] + public async Task The_body_storage_check_leaves_no_probe_body_behind() + { + var bodyStorage = ServiceProvider.GetRequiredService<IBodyStoragePersistence>(); + + await Readiness.ContributedChecks().Single(check => check.Name == "message body storage is writable").Run(); + + Assert.That(await bodyStorage.ReadBody("migration-writable-probe"), Is.Null, "a probe left in the store is a body the source never had, which a count comparison reads as an extra rather than a loss"); + } + + [Test] + public void The_body_storage_check_refuses_a_store_that_cannot_write() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + var check = new BodyStorageIsWritableCheck(new FileSystemBodyStoragePersistence( + new FileSystemBodyStorageSettings { StoragePath = Path.Combine(parentThatIsAFile, "bodies") })); + + Assert.CatchAsync<IOException>(() => check.Run()); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + // The copy runs before any hosted service starts, so a target that only works once one has started is broken exactly when the migration needs it. + [Test] + public async Task The_target_answers_from_a_container_whose_hosted_services_have_not_started() + { + var (host, context) = await BuildHostWithoutStarting(); + + try + { + await using var scope = host.Services.GetRequiredService<IServiceScopeFactory>().CreateAsyncScope(); + + Assert.That(await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().EndpointSettings.LongCountAsync(), Is.Zero); + Assert.DoesNotThrowAsync(() => host.Services.GetRequiredService<IMigrationTargetReadiness>().ContributedChecks().Single(check => check.Name == "message body storage is writable").Run()); + } + finally + { + await context.TearDown(); + host.Dispose(); + } + } + + // PersistenceTestBase already started a host in SetUp, so this builds a second one over its own test database. + static async Task<(IHost Host, PersistenceTestsContext Context)> BuildHostWithoutStarting() + { + var context = new PersistenceTestsContext(); + var hostBuilder = Host.CreateApplicationBuilder(); + + await context.Setup(hostBuilder); + var host = hostBuilder.Build(); + await context.InstallSchema(host); + + return (host, context); + } + + [Test] + public async Task The_host_opened_record_is_absent_until_written_and_stays_after_a_second_write() + { + var readiness = ServiceProvider.GetRequiredService<IMigrationTargetReadiness>(); + + Assert.That(await readiness.HasHostOpened(), Is.False, "a target nothing has opened on must not claim otherwise"); + + await readiness.RecordHostOpened(); + var firstOpened = Now; + + AdvanceClock(TimeSpan.FromDays(7)); + await readiness.RecordHostOpened(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await readiness.HasHostOpened(), Is.True); + // The stored instant is what tells an operator the clean abort is over, so a later start must not move it forward. + Assert.That(await ReadHostOpenedAt(), Is.EqualTo(firstOpened)); + } + } + + async Task<DateTime?> ReadHostOpenedAt() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + return await dbContext.GetSetting<DateTime?>(SettingKeys.MigrationHostOpenedOnTarget); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs new file mode 100644 index 0000000000..9dfa1dd395 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventRowMapperTests.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class PendingIntegrationEventRowMapperTests : PersistenceTestBase +{ + [Test] + public void Sets_every_mapped_property_of_a_dispatch_request() + { + var entity = PendingIntegrationEventRowMapper.Map(Request("one")); + + using var scope = ServiceProvider.CreateScope(); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model, entity); + } + + [Test] + public async Task Writes_exactly_the_type_name_and_json_the_product_writes_for_the_same_context() + { + await ExternalIntegrationStore.StoreDispatchRequest([Request("one")]); + + using var scope = ServiceProvider.CreateScope(); + var stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().ExternalIntegrationDispatchRequests.AsNoTracking().SingleAsync(); + var mapped = PendingIntegrationEventRowMapper.Map(Request("one")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(mapped.DispatchContextTypeName, Is.EqualTo(stored.DispatchContextTypeName)); + Assert.That(mapped.DispatchContextJson, Is.EqualTo(stored.DispatchContextJson)); + } + } + + static ExternalIntegrationDispatchRequest Request(string value) => new() { DispatchContext = new TestDispatchContext { Value = value } }; + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs new file mode 100644 index 0000000000..f3ca3ea132 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PendingIntegrationEventsMigrationTargetTests.cs @@ -0,0 +1,120 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.ExternalIntegrations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Implementation; + +class PendingIntegrationEventsMigrationTargetTests : PersistenceTestBase +{ + readonly TargetWarnings targetLog = new(); + + public PendingIntegrationEventsMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetLog); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + ExternalIntegrationRequestsDataStore Dispatcher => ServiceProvider.GetServices<IHostedService>().OfType<ExternalIntegrationRequestsDataStore>().Single(); + + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.PendingIntegrationEvents)!; + + [Test] + public async Task A_copied_request_is_dispatched_by_the_product_as_its_concrete_type_and_then_removed() + { + var dispatched = Array.Empty<object>(); + ExternalIntegrationStore.Subscribe((contexts, _) => + { + dispatched = contexts; + return Task.CompletedTask; + }); + + var result = await Write(("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "copied" } })); + + await Dispatcher.DispatchNow(TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(((TestDispatchContext)dispatched.Single()).Value, Is.EqualTo("copied")); + Assert.That(await Rows(), Is.Zero, "the dispatcher deletes what it delivered, which is delivery rather than loss"); + } + } + + [Test] + public async Task A_request_with_no_context_is_skipped_as_a_required_value_missing() + { + var result = await Write( + ("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = null! }), + ("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "kept" } })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkipReasons[MigrationSkipReason.RequiredValueMissing], Is.EqualTo(1), "both columns are NOT NULL, and a throw would stop the category at this batch"); + Assert.That(await Target.Count(Category), Is.EqualTo(1), "Count is what --migration-verify reads, so it must see the row the copy wrote"); + } + } + + [Test] + public async Task A_request_whose_context_type_name_is_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + // The stored type name ends with the assembly name, so a long assembly name makes it longer than the column. + var assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(new string('a', 460)), AssemblyBuilderAccess.Run); + var longNamedContext = Activator.CreateInstance(assembly.DefineDynamicModule("Contexts").DefineType("Context", TypeAttributes.Public).CreateType())!; + + var result = await Write( + ("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = longNamedContext }), + ("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "kept" } })); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "ExternalIntegrationDispatchRequests/1" })); + Assert.That(result.SkipReasons[MigrationSkipReason.ValueTooLong], Is.EqualTo(1), "one value longer than its column fails the whole statement, so it is skipped instead"); + Assert.That(await Target.Count(Category), Is.EqualTo(1)); + } + } + + [Test] + public async Task The_first_batch_that_copies_events_says_once_that_the_database_gives_them_new_ids() + { + var first = await Write(("ExternalIntegrationDispatchRequests/1", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "first" } })); + await Target.Write( + Category, + new MigrationBatch([new MigrationRow("ExternalIntegrationDispatchRequests/2", new ExternalIntegrationDispatchRequest { DispatchContext = new TestDispatchContext { Value = "second" } }, new Dictionary<string, object>())], "ExternalIntegrationDispatchRequests/2"), + first.Saved); + + Assert.That(targetLog.Information, Has.Exactly(1).EqualTo($"Category {MigrationCategoryIds.PendingIntegrationEvents}: the database gives the copied pending integration events new ids. Nothing in SQL refers to the ids they had in the source."), + "one line per category says the ids changed, rather than one per row or per batch"); + } + + Task<MigrationWriteResult> Write(params (string Id, ExternalIntegrationDispatchRequest Request)[] rows) => + Target.Write( + Category, + new MigrationBatch([.. rows.Select(row => new MigrationRow(row.Id, row.Request, new Dictionary<string, object>()))], rows[^1].Id), + new MigrationCheckpoint(Category.Id, MigrationCategoryState.InProgress, rows[^1].Id, 0, 0, null, null, null, null, null, null)); + + async Task<int> Rows() + { + using var scope = ServiceProvider.CreateScope(); + return await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().ExternalIntegrationDispatchRequests.CountAsync(); + } + + public class TestDispatchContext + { + public string Value { get; set; } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs new file mode 100644 index 0000000000..98e6b8c93f --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/PreparedSkips.cs @@ -0,0 +1,58 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +static class PreparedSkips +{ + /// <summary> + /// Prepares a batch through one writer without writing anything, so a test can read the detail line the target logs for each skip. + /// </summary> + public static async Task<IReadOnlyList<string>> DetailsFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(), batch); + + return [.. prepared.Skips.Select(skip => skip.Detail)]; + } + + /// <summary> + /// Prepares a batch without writing it, and returns the rows the writer reports as merges with their log detail. + /// </summary> + public static async Task<IReadOnlyList<(string SourceId, string Detail)>> MergesFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(), batch); + + return prepared.Merges; + } + + /// <summary> + /// Prepares a batch without writing it, and returns the rows the writer reports as changed on the way in with their log detail. + /// </summary> + public static async Task<IReadOnlyList<(string SourceId, string Detail)>> ChangesFrom(IServiceProvider services, IMigrationCategoryWriter writer, MigrationBatch batch) + { + using var scope = services.CreateScope(); + var prepared = await writer.Prepare(scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(), batch); + + return prepared.Changes; + } + + /// <summary> + /// The table and column a property is stored in, as this provider names them. + /// </summary> + public static string TableAndColumnOf<TEntity>(IServiceProvider services, string property) + { + using var scope = services.CreateScope(); + var entityType = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model.FindEntityType(typeof(TEntity))!; + + return $"{entityType.GetTableName()}.{entityType.FindProperty(property)!.GetColumnName()}"; + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs new file mode 100644 index 0000000000..6763d6d562 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationRowMapperTests.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Recoverability; + +class RetryOperationRowMapperTests : PersistenceTestBase +{ + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task Sets_every_mapped_property_of_a_historic_and_an_unacknowledged_operation() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + using (Assert.EnterMultipleScope()) + { + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new HistoricRetryOperation + { + RequestId = "group-1", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + Failed = true, + NumberOfMessagesProcessed = 5 + })); + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, RetryOperationRowMapper.Map(new UnacknowledgedRetryOperation + { + RequestId = "group-2", + RetryType = RetryType.FailureGroup, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = "Shipping-Failures", + Failed = true, + NumberOfMessagesProcessed = 2 + })); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs new file mode 100644 index 0000000000..ded7804dac --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/RetryOperationsMigrationTargetTests.cs @@ -0,0 +1,285 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Recoverability; + +class RetryOperationsMigrationTargetTests : PersistenceTestBase +{ + readonly TargetWarnings targetLog = new(); + + public RetryOperationsMigrationTargetTests() => RegisterServices = services => services.AddSingleton<ILoggerProvider>(targetLog); + + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory RetryOperationsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.RetryOperations)!; + + static readonly DateTime CompletedAt = new(2026, 3, 1, 12, 0, 0, DateTimeKind.Utc); + + [Test] + public async Task A_retry_operation_with_a_value_longer_than_its_column_is_skipped_as_value_too_long_and_the_rest_copy() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var fourHundredCodePoints = char.ConvertFromUtf32(0x1F600) + new string('r', 399); + var countsCodePoints = dialect.TextLength(fourHundredCodePoints) < fourHundredCodePoints.Length; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(new string('r', 400)), null)), + ("historic/2", new RetryOperationRow(Historic(new string('r', 401)), null)), + ("historic/3", new RetryOperationRow(Historic(fourHundredCodePoints), null)), + ("pending/1", new RetryOperationRow(null, Pending(new string('p', 401), "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", new string('c', 451)))), + ("pending/3", new RetryOperationRow(null, Pending(new string('p', 400), new string('c', 450))))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/3")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + var historicRequestId = PreparedSkips.TableAndColumnOf<HistoricRetryOperationEntity>(ServiceProvider, nameof(HistoricRetryOperationEntity.RequestId)); + string[] expectedDetails = + [ + $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/2'", + .. countsCodePoints ? Array.Empty<string>() : new[] { $"{historicRequestId} takes at most 400 characters and the value holds 401, on document 'historic/3'" }, + $"{PreparedSkips.TableAndColumnOf<UnacknowledgedRetryOperationEntity>(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.RequestId))} takes at most 400 characters and the value holds 401, on document 'pending/1'", + $"{PreparedSkips.TableAndColumnOf<UnacknowledgedRetryOperationEntity>(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Classifier))} takes at most 450 characters and the value holds 451, on document 'pending/2'" + ]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(fourHundredCodePoints, Has.Length.EqualTo(401), "the emoji request id is one UTF-16 unit over the column, and 400 code points"); + Assert.That(result.Copied, Is.EqualTo(countsCodePoints ? 3 : 2), "a value exactly as long as its column fits, in either table"); + Assert.That(result.SkippedIds, Is.EqualTo(countsCodePoints ? new[] { "historic/2", "pending/1", "pending/2" } : new[] { "historic/2", "historic/3", "pending/1", "pending/2" }), + "400 code points fit a column that counts code points, and a skip there would lose a retry operation the database could hold"); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = countsCodePoints ? 3 : 4 }), "an over-long value would fail the whole insert and halt the category"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(countsCodePoints ? new[] { new string('r', 400), fourHundredCodePoints } : new[] { new string('r', 400) })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.Classifier)), Is.EqualTo(new[] { (new string('p', 400), new string('c', 450)) })); + Assert.That(details, Is.EqualTo(expectedDetails)); + } + } + + [Test] + public async Task A_retry_operation_holding_a_character_the_database_refuses_is_skipped_as_malformed_value_where_the_database_refuses_it() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var refused = dialect.RejectedCharacterIn("\0") is not null; + var historic = Historic("group-1"); + historic.Originator = "Sales\0Orders"; + var pending = Pending("group-2", "Shipping-Failures"); + pending.Originator = "Ship\0ping"; + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("historic/2", new RetryOperationRow(Historic("group-3"), null)), + ("pending/1", new RetryOperationRow(null, pending)), + ("pending/2", new RetryOperationRow(null, Pending("group-4", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.SkippedIds, Is.EqualTo(refused ? new[] { "historic/1", "pending/1" } : Array.Empty<string>()), "only a database that refuses the character fails the statement over it"); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-3" } : new[] { "group-1", "group-3" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EquivalentTo(refused ? new[] { "group-4" } : new[] { "group-2", "group-4" })); + + if (refused) + { + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 2 })); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<HistoricRetryOperationEntity>(ServiceProvider, nameof(HistoricRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'historic/1'", + $"{PreparedSkips.TableAndColumnOf<UnacknowledgedRetryOperationEntity>(ServiceProvider, nameof(UnacknowledgedRetryOperationEntity.Originator))} cannot hold the character U+0000, which the value contains, on document 'pending/1'" + })); + } + } + } + + [Test] + public async Task A_retry_operation_with_no_request_id_is_skipped_naming_the_table_and_the_rest_copy() + { + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic(null!), null)), + ("historic/2", new RetryOperationRow(Historic("group-1"), null)), + ("pending/1", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new RetryOperationsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/2")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "HistoricRetryOperations.RequestId is NOT NULL and the document has no value for it", + "UnacknowledgedRetryOperations.RequestId is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the table the operation was bound for"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "historic/1", "pending/1" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-1" })); + Assert.That(stored.UnacknowledgedOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_second_pending_operation_with_the_same_key_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(Historic("group-2"), null)), + ("pending/skipped", new RetryOperationRow(null, Pending(null!, "Shipping-Failures"))), + ("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/2", new RetryOperationRow(null, Pending("group-2", "Billing-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress)))); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2", "pending/1" } : new[] { "pending/1" }), + "the last pending operation with one key is kept and the earlier ones are dropped on every database, a case-only request id counts as the same key only where the column ignores case, one request retried as another type never merges, and a historic or skipped row never merges"); + Assert.That(merges[0].Detail, Does.Contain(requestIdIgnoresCase ? "kept 'GROUP-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'" : "kept 'group-2', 'FailureGroup' and dropped the row for 'group-2', 'FailureGroup'"), + "the target logs this line, so it has to name the key it kept and the key it dropped without swapping them"); + } + } + + [Test] + public async Task A_merged_pending_operation_names_the_source_row_kept_and_the_row_dropped() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("unacknowledged|group-2|FailureGroup|older", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("unacknowledged|group-2|FailureGroup|newer", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(new[] { "unacknowledged|group-2|FailureGroup|older" }), "the latest retry wins"); + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 'unacknowledged|group-2|FailureGroup|newer' and dropped the row from 'unacknowledged|group-2|FailureGroup|older'."), + "both keys print the same, so only the source rows, whose ids carry the completion time, tell the operator which run survived"); + } + } + + [Test] + public async Task A_pending_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var requestIdIgnoresCase = dialect.KeyComparer(typeof(UnacknowledgedRetryOperationEntity), nameof(UnacknowledgedRetryOperationEntity.RequestId)).Equals("group-2", "GROUP-2"); + var first = await Target.Write(RetryOperationsCategory, BatchOf(("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("pending/2", new RetryOperationRow(null, Pending("GROUP-2", "Shipping-Failures"))), + ("pending/3", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures"))), + ("pending/4", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures", RetryType.ByQueueAddress))), + ("historic/1", new RetryOperationRow(Historic("group-2"), null))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new RetryOperationsWriter(dialect), second); + var result = await Target.Write(RetryOperationsCategory, second, first.Saved with { Cursor = second.Cursor }); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(requestIdIgnoresCase ? new[] { "pending/2" } : Array.Empty<string>()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another retry type never"); + + if (requestIdIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'group-2', 'FailureGroup' and dropped the row for 'GROUP-2', 'FailureGroup'")); + } + + Assert.That(result.Copied, Is.EqualTo(requestIdIgnoresCase ? 2 : 3), "the historic row and the other retry type are copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(requestIdIgnoresCase ? 2 : 1)); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.RequestId, operation.RetryType)), Is.EquivalentTo(requestIdIgnoresCase + ? new[] { ("group-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) } + : new[] { ("group-2", RetryType.FailureGroup), ("GROUP-2", RetryType.FailureGroup), ("group-2", RetryType.ByQueueAddress) })); + Assert.That(stored.HistoricOperations.Select(operation => operation.RequestId), Is.EqualTo(new[] { "group-2" })); + } + } + + [Test] + public async Task A_retry_operation_time_of_local_or_unspecified_kind_is_copied_as_the_same_instant() + { + var historic = Historic("group-1"); + historic.StartTime = CompletedAt.AddMinutes(-1).ToLocalTime(); + historic.CompletionTime = DateTime.SpecifyKind(CompletedAt, DateTimeKind.Unspecified); + var pending = Pending("group-2", "Shipping-Failures"); + pending.StartTime = DateTime.SpecifyKind(CompletedAt.AddMinutes(-1), DateTimeKind.Unspecified); + pending.CompletionTime = CompletedAt.ToLocalTime(); + pending.Last = CompletedAt.AddMinutes(1).ToLocalTime(); + var batch = BatchOf( + ("historic/1", new RetryOperationRow(historic, null)), + ("pending/1", new RetryOperationRow(null, pending))); + + var result = await Target.Write(RetryOperationsCategory, batch, CheckpointAfter("pending/1")); + var stored = (await RetryHistoryStore.GetRetryHistory()).Results!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2), "Npgsql refuses a time that is not UTC for a timestamptz column, and that would fail the whole retry history"); + Assert.That(stored.HistoricOperations.Select(operation => (operation.StartTime, operation.CompletionTime)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt) }), + "an Unspecified time is already UTC and a Local time is converted, so both keep their instant"); + Assert.That(stored.UnacknowledgedOperations.Select(operation => (operation.StartTime, operation.CompletionTime, operation.Last)), Is.EqualTo(new[] { (CompletedAt.AddMinutes(-1), CompletedAt, CompletedAt.AddMinutes(1)) })); + } + } + + [Test] + public async Task Copied_historic_operations_get_one_line_saying_the_database_gives_them_new_ids() + { + var first = await Target.Write(RetryOperationsCategory, BatchOf(("historic/1", new RetryOperationRow(Historic("group-1"), null)), ("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + await Target.Write(RetryOperationsCategory, BatchOf(("historic/2", new RetryOperationRow(Historic("group-3"), null))), first.Saved with { Cursor = "historic/2" }); + + Assert.That(targetLog.Information, Is.EqualTo(new[] { $"Category {MigrationCategoryIds.RetryOperations}: the database gives the copied historic retry operations new ids. Nothing in SQL refers to the ids they had in the source." }), + "one line per category says the ids changed, rather than one per row or per batch"); + } + + [Test] + public async Task Copied_pending_operations_alone_get_no_line_about_new_ids() + { + await Target.Write(RetryOperationsCategory, BatchOf(("pending/1", new RetryOperationRow(null, Pending("group-2", "Shipping-Failures")))), CheckpointAfter("pending/1")); + + Assert.That(targetLog.Information, Has.None.Contains("new ids"), "a pending operation keeps its key, so nothing about it is renumbered"); + } + + static HistoricRetryOperation Historic(string requestId) => new() + { + RequestId = requestId, + RetryType = RetryType.ByQueueAddress, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Originator = "Sales.Orders", + NumberOfMessagesProcessed = 1 + }; + + static UnacknowledgedRetryOperation Pending(string requestId, string classifier, RetryType retryType = RetryType.FailureGroup) => new() + { + RequestId = requestId, + RetryType = retryType, + StartTime = CompletedAt.AddMinutes(-1), + CompletionTime = CompletedAt, + Last = CompletedAt, + Originator = "Shipping", + Classifier = classifier, + NumberOfMessagesProcessed = 1 + }; + + static MigrationBatch BatchOf(params (string Id, RetryOperationRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.RetryOperations, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs new file mode 100644 index 0000000000..84b27862f8 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingRowMapperTests.cs @@ -0,0 +1,55 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SettingRowMapperTests : PersistenceTestBase +{ + static readonly List<string> ReportMasks = ["contoso", "fabrikam"]; + + static readonly LicensedEndpointDetails Details = new LicensedEndpointDetails + { + LicenseId = "abc-123", + ValidId = true, + ServiceEndDate = "2027-01-01", + Endpoints = [new LicensedEndpoint { Name = "Sales.Orders", Classification = 2, EndpointSize = "Large", Queues = [new QueueIdentity { NameHash = "n1", ScopeHash = "s1" }] }], + InfrastructureQueues = [new QueueIdentity { NameHash = "n2", ScopeHash = "s2" }], + ExcludedQueues = [new QueueIdentity { NameHash = "n3", ScopeHash = "s3" }], + Products = [new Product { ProductCode = "P1", MonthlyThroughput = 500 }] + }; + + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, SettingRowMapper.Map(SettingKeys.ReportMasks, ReportMasks)); + } + + [Test] + public async Task Licensing_settings_read_back_through_GetSetting_as_the_product_reads_them() + { + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + dbContext.Settings.AddRange(SettingRowMapper.Map(SettingKeys.ReportMasks, ReportMasks), SettingRowMapper.Map(SettingKeys.LicensedEndpointDetails, Details)); + await dbContext.SaveChangesAsync(); + } + + await using (var scope = ServiceProvider.CreateAsyncScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + Assert.That(await dbContext.GetSetting<List<string>>(SettingKeys.ReportMasks), Is.EqualTo(ReportMasks)); + Assert.That(JsonSerializer.Serialize(await dbContext.GetSetting<LicensedEndpointDetails>(SettingKeys.LicensedEndpointDetails)), Is.EqualTo(JsonSerializer.Serialize(Details))); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..98c6ec145d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SettingsMigrationTargetTests.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Particular.LicensingComponent.Contracts; +using ServiceControl.Notifications; +using ServiceControl.Persistence.DataMigration; + +class SettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly string[] SettingsCategoryIds = [MigrationCategoryIds.LicensingReportMasks, MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryIds.NotificationSettings, MigrationCategoryIds.TrialEndDate]; + + [TestCase(MigrationCategoryIds.LicensingReportMasks)] + [TestCase(MigrationCategoryIds.LicensedEndpointDetails)] + [TestCase(MigrationCategoryIds.NotificationSettings)] + [TestCase(MigrationCategoryIds.TrialEndDate)] + public async Task Count_is_the_row_under_the_categorys_own_key_alone(string categoryId) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == categoryId); + + foreach (var otherCategoryId in SettingsCategoryIds.Where(id => id != categoryId)) + { + await StoreSettingFor(otherCategoryId); + } + + Assert.That(await Target.Count(category), Is.Zero, "the other settings rows are not rows of this category"); + + await StoreSettingFor(categoryId); + + Assert.That(await Target.Count(category), Is.EqualTo(1), "counted as the whole Settings table, this would also count the other settings rows"); + } + + Task StoreSettingFor(string categoryId) => categoryId switch + { + MigrationCategoryIds.LicensingReportMasks => LicensingDataStore.SaveReportMasks(["contoso"]), + MigrationCategoryIds.LicensedEndpointDetails => LicensingDataStore.SaveLicensedEndpointDetails(new LicensedEndpointDetails { LicenseId = "abc-123" }), + MigrationCategoryIds.NotificationSettings => NotificationsStore.SaveSettings(new NotificationsSettings { Email = new EmailNotifications { SmtpServer = "smtp.contoso.com" } }), + MigrationCategoryIds.TrialEndDate => ServiceProvider.GetRequiredService<ITrialLicenseDataProvider>().StoreTrialEndDate(new DateOnly(2027, 3, 15)), + _ => throw new ArgumentOutOfRangeException(nameof(categoryId), categoryId, "This test stores no settings row for that category.") + }; +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs new file mode 100644 index 0000000000..5334eef99d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionRowMapperTests.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class SubscriptionRowMapperTests : PersistenceTestBase +{ + [Test] + public async Task Sets_every_mapped_property() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var model = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Model; + + var row = SubscriptionRowMapper.Map(new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(model, row); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs new file mode 100644 index 0000000000..535520537d --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/SubscriptionsMigrationTargetTests.cs @@ -0,0 +1,193 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration.Writers; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class SubscriptionsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService<IMigrationTarget>(); + + static readonly MigrationCategory SubscriptionsCategory = MigrationCategoryRegistry.Find(MigrationCategoryIds.Subscriptions); + + [Test] + public async Task A_subscription_with_a_value_longer_than_its_column_is_skipped_under_its_reason_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow(new string('t', 200), version, new string('a', 200), new string('e', 450))), + ("s/2", new SubscriptionRow(new string('t', 201), version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, new string('a', 201), "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", new string('e', 451)))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/4")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1), "a value exactly as long as its column fits"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/2", "s/3", "s/4" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.ValueTooLong] = 3 }), + "an over-long value would fail the whole insert and halt the category"); + Assert.That(details, Is.EqualTo(new[] + { + $"{PreparedSkips.TableAndColumnOf<SubscriptionEntity>(ServiceProvider, nameof(SubscriptionEntity.MessageType))} takes at most 200 characters and the value holds 201, on document 's/2'", + $"{PreparedSkips.TableAndColumnOf<SubscriptionEntity>(ServiceProvider, nameof(SubscriptionEntity.TransportAddress))} takes at most 200 characters and the value holds 201, on document 's/3'", + $"{PreparedSkips.TableAndColumnOf<SubscriptionEntity>(ServiceProvider, nameof(SubscriptionEntity.Endpoint))} takes at most 450 characters and the value holds 451, on document 's/4'" + })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { (new string('t', 200), new string('a', 200), new string('e', 450)) }), "the row whose values all fit is stored as it was read"); + Assert.That(await Target.Count(SubscriptionsCategory), Is.EqualTo(1), "Count is the rows the Subscriptions table holds"); + } + } + + [Test] + public async Task A_second_version_of_a_type_for_one_subscriber_is_reported_as_a_merge() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderShipped", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), batch); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2", "s/3" } : new[] { "s/2" }), + "the key has no version, so a second version is dropped on every database, a case-only address only where the column ignores case, and another type never"); + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01'"), "the target logs this line, so it has to name the key it kept"); + + if (addressIgnoresCase) + { + Assert.That(merges[1].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'"), + "the line must not swap the key kept and the key dropped"); + } + } + } + + [Test] + public async Task A_merged_subscription_names_the_version_and_endpoint_of_the_row_kept_and_the_row_dropped() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing.V2"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 's/1' (version 1.0.0, endpoint 'Billing') and dropped the row from 's/2' (version 2.0.0, endpoint 'Billing.V2')."), + "both keys print the same, so only the source rows tell the operator which version survived"); + Assert.That((result.AlreadyPresent, result.Merged), Is.EqualTo((1, 1)), "the summary counts the merge apart from the other rows already present"); + } + } + + [Test] + public async Task A_subscription_merged_on_a_re_read_names_the_row_an_earlier_pass_copied() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var kept = ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")); + var first = await Target.Write(SubscriptionsCategory, BatchOf(kept), CheckpointAfter("s/1")); + var reRead = BatchOf(kept, ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(2, 0, 0), "Billing@BILLING01", "Billing"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), reRead); + var result = await Target.Write(SubscriptionsCategory, reRead, first.Saved); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Single().Detail, Does.EndWith("It kept the row from 's/1' (version 1.0.0, endpoint 'Billing') and dropped the row from 's/2' (version 2.0.0, endpoint 'Billing')."), + "the stored row is the one 's/1' copied on the earlier pass, so the line names that source row"); + Assert.That((result.AlreadyPresent, result.Merged), Is.EqualTo((2, 1))); + } + } + + [Test] + public async Task A_key_the_target_already_holds_merges_only_a_variant_the_column_treats_as_the_same() + { + var dialect = ServiceProvider.GetRequiredService<IMigrationSqlDialect>(); + var addressIgnoresCase = dialect.KeyComparer(typeof(SubscriptionEntity), nameof(SubscriptionEntity.TransportAddress)).Equals("Billing@BILLING01", "billing@billing01"); + var first = await Target.Write(SubscriptionsCategory, BatchOf(("s/1", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing"))), CheckpointAfter("s/1")); + + // The case variant comes first, so on a case-insensitive column only the stored row can be what it merges with. + var second = BatchOf( + ("s/2", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "billing@billing01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Billing@BILLING01", "Billing")), + ("s/4", new SubscriptionRow("MyMessages.OrderPlaced", new Version(1, 0, 0), "Shipping@SHIP01", "Shipping"))); + + var merges = await PreparedSkips.MergesFrom(ServiceProvider, new SubscriptionsWriter(dialect), second); + var result = await Target.Write(SubscriptionsCategory, second, first.Saved with { Cursor = second.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(merges.Select(merge => merge.SourceId), Is.EqualTo(addressIgnoresCase ? new[] { "s/2" } : Array.Empty<string>()), + "an exact repeat of a stored key is an earlier pass's row, a case variant merges only where the column ignores case, and another address never"); + + if (addressIgnoresCase) + { + Assert.That(merges[0].Detail, Does.Contain("kept 'MyMessages.OrderPlaced', 'Billing@BILLING01' and dropped the row for 'MyMessages.OrderPlaced', 'billing@billing01'")); + Assert.That(merges[0].Detail, Does.EndWith("It kept the row the table already held and dropped the row from 's/2' (version 1.0.0, endpoint 'Billing')."), "no row of this batch claimed the stored row"); + } + + Assert.That(result.Copied, Is.EqualTo(addressIgnoresCase ? 1 : 2), "the other address is copied, and so is the case variant where the column tells it apart"); + Assert.That(result.AlreadyPresent, Is.EqualTo(addressIgnoresCase ? 2 : 1)); + Assert.That(await StoredSubscriptions(), Is.EquivalentTo(addressIgnoresCase + ? new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") } + : new[] { ("MyMessages.OrderPlaced", "Billing@BILLING01", "Billing"), ("MyMessages.OrderPlaced", "billing@billing01", "Billing"), ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + [Test] + public async Task A_subscription_missing_a_key_column_is_skipped_naming_the_column_and_the_rest_copy() + { + var version = new Version(1, 0, 0); + var batch = BatchOf( + ("s/1", new SubscriptionRow("MyMessages.OrderPlaced", version, TransportAddress: null!, "Billing")), + ("s/2", new SubscriptionRow(MessageType: null!, version, "Billing@BILLING01", "Billing")), + ("s/3", new SubscriptionRow("MyMessages.OrderPlaced", version, "Shipping@SHIP01", "Shipping"))); + + var details = await PreparedSkips.DetailsFrom(ServiceProvider, new SubscriptionsWriter(ServiceProvider.GetRequiredService<IMigrationSqlDialect>()), batch); + var result = await Target.Write(SubscriptionsCategory, batch, CheckpointAfter("s/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(details, Is.EqualTo(new[] + { + "Subscriptions.TransportAddress is NOT NULL and the document has no value for it", + "Subscriptions.MessageType is NOT NULL and the document has no value for it" + }), "the target logs this line, so it has to name the column that is missing"); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "s/1", "s/2" })); + Assert.That(result.SkipReasons, Is.EqualTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 })); + Assert.That(await StoredSubscriptions(), Is.EqualTo(new[] { ("MyMessages.OrderPlaced", "Shipping@SHIP01", "Shipping") })); + } + } + + async Task<IReadOnlyList<(string MessageType, string TransportAddress, string Endpoint)>> StoredSubscriptions() + { + await using var scope = ServiceProvider.CreateAsyncScope(); + var stored = await scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Subscriptions.AsNoTracking().ToListAsync(); + + return [.. stored.Select(subscription => (subscription.MessageType, subscription.TransportAddress, subscription.Endpoint))]; + } + + static MigrationBatch BatchOf(params (string Id, SubscriptionRow Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary<string, object>()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.Subscriptions, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs new file mode 100644 index 0000000000..e84ab5f282 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/TargetWarnings.cs @@ -0,0 +1,49 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; + +// RecordingLoggerProvider lives in ServiceControl.Infrastructure.Tests, which the persistence test projects do not reference. +/// <summary> +/// Keeps the lines the migration target logs, so a test can read what the operator sees. Messages holds the warnings and Information the information lines. +/// Register it as an ILoggerProvider through RegisterServices. +/// </summary> +sealed class TargetWarnings : ILoggerProvider +{ + readonly ConcurrentQueue<string> messages = new(); + readonly ConcurrentQueue<string> information = new(); + + public IReadOnlyList<string> Messages => [.. messages]; + + public IReadOnlyList<string> Information => [.. information]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(EFCoreMigrationTarget).FullName ? new TargetLogger(this) : NullLogger.Instance; + + public void Dispose() + { + } + + sealed class TargetLogger(TargetWarnings lines) : ILogger + { + public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel is LogLevel.Warning or LogLevel.Information; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) + { + if (logLevel == LogLevel.Warning) + { + lines.messages.Enqueue(formatter(state, exception)); + } + else if (logLevel == LogLevel.Information) + { + lines.information.Enqueue(formatter(state, exception)); + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs new file mode 100644 index 0000000000..5261d9fffd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTableTests : PersistenceTestBase +{ + [Test] + public async Task The_table_round_trips_every_column_written_directly_through_the_DbContext() + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "EndpointSettings", + State = MigrationCategoryState.InProgress, + Cursor = "cursor-1", + CopiedCount = 3, + SkippedCount = 1, + SourceTotal = 40, + SkipReasons = new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 }, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = null, + LastError = null, + AlreadyPresentCount = 36, + Version = 7 + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(3)); + Assert.That(stored.SkippedCount, Is.EqualTo(1)); + Assert.That(stored.SourceTotal, Is.EqualTo(40)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(stored.AlreadyPresentCount, Is.EqualTo(36)); + Assert.That(stored.Version, Is.EqualTo(7)); + Assert.That(stored.StartedAt, Is.EqualTo(Now)); + Assert.That(stored.LastProgressAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Null); + Assert.That(stored.LastError, Is.Null); + } + } + + [Test] + public async Task Every_timestamp_column_comes_back_as_a_Utc_DateTime() + { + Assert.That(Now.Kind, Is.EqualTo(DateTimeKind.Utc), "the engine stamps timings from TimeProvider.GetUtcNow().UtcDateTime, so the test clock has to match"); + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "KnownEndpoints", + State = MigrationCategoryState.Complete, + Cursor = null, + CopiedCount = 1, + SkippedCount = 0, + SourceTotal = 1, + SkipReasons = null, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = Now, + LastError = null + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "KnownEndpoints"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.StartedAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.LastProgressAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.SettledAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public void The_skip_reasons_JSON_is_keyed_by_reason_name_not_by_its_number() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 2 })!; + + // Enum.Parse accepts "0" as readily as "BodyUnreadable", so a round trip cannot tell the two encodings apart. + Assert.That(json, Does.Contain("BodyUnreadable")); + } + + [Test] + public void A_blank_group_comment_skip_round_trips_by_name() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BlankGroupComment] = 3 })!; + var reasons = (IReadOnlyDictionary<MigrationSkipReason, long>)converter.ConvertFromProvider(json)!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(json, Does.Contain("BlankGroupComment")); + Assert.That(reasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BlankGroupComment] = 3 })); + } + } + + [Test] + public void A_reason_name_this_build_does_not_know_is_read_as_Unknown_rather_than_throwing() + { + // The rows a newer build wrote are what a downgraded instance meets, and the read path that would + // throw is ReadAll, which is what decides whether the host may start. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var reasons = (IReadOnlyDictionary<MigrationSkipReason, long>)converter.ConvertFromProvider( + """{"BodyUnreadable":2,"SomethingFromTheFuture":3,"AlsoFromTheFuture":1}""")!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(reasons[MigrationSkipReason.BodyUnreadable], Is.EqualTo(2)); + Assert.That(reasons[MigrationSkipReason.Unknown], Is.EqualTo(4), "several unknown names collapse onto the one member, so their counts are summed rather than one of them winning"); + Assert.That(reasons, Has.Count.EqualTo(2)); + } + } + + [Test] + public void The_table_carries_exactly_the_columns_the_checkpoint_needs() + { + // The one thing the one-way door makes permanent. Adding, dropping or renaming one after this + // ships costs a migration against customer data, so it fails here rather than in a customer's + // database. Property names rather than column names, because each provider spells the columns + // its own way and the spelling is a convention the generated migrations already pin. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var entityType = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(entityType.GetProperties().Select(property => property.Name), Is.EquivalentTo(new[] + { + "CategoryId", "State", "Cursor", "CopiedCount", "SkippedCount", "SourceTotal", + "SkipReasons", "StartedAt", "LastProgressAt", "SettledAt", "LastError", "AlreadyPresentCount", "Version", "StartedWindowSeconds" + })); + + // Underscores stripped so one assertion covers MigrationCheckpoints and migration_checkpoints. + Assert.That(entityType.GetTableName()!.Replace("_", ""), Is.EqualTo("MigrationCheckpoints").IgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs new file mode 100644 index 0000000000..706f41f046 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs @@ -0,0 +1,178 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTransactionTests : PersistenceTestBase +{ + static MigrationCheckpoint Checkpoint(DateTime now, string cursor, long copied, long version = 0) => + new("EndpointSettings", MigrationCategoryState.InProgress, cursor, copied, 0, 40, null, now, now, null, null, Version: version); + + async Task SeedCheckpoint(string cursor, long copied) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + await dbContext.UpsertCheckpoint(Checkpoint(Now, cursor, copied)); + } + + [Test] + public async Task A_failure_before_commit_leaves_neither_the_row_nor_the_checkpoint() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync<InvalidOperationException>(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.False); + } + } + + [Test] + public async Task A_successful_commit_lands_the_row_and_the_checkpoint_together() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.True); + } + } + + [Test] + public async Task An_existing_checkpoint_is_advanced_by_a_commit_inside_the_callers_transaction() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task An_existing_checkpoint_keeps_its_previous_cursor_when_the_transaction_rolls_back() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync<InvalidOperationException>(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService<ServiceControlDbContext>(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_save_built_from_an_out_of_date_copy_is_refused_rather_than_silently_winning() + { + var store = ServiceProvider.GetRequiredService<IMigrationCheckpointStore>(); + await store.Upsert(Checkpoint(Now, "cursor-1", 1)); + + // Both read the row at the same version, as a background copy and a required copy on two hosts would. + var first = (await store.Read("EndpointSettings"))!; + var stale = (await store.Read("EndpointSettings"))!; + + await store.Upsert(first with { Cursor = "cursor-first", CopiedCount = 2 }); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => store.Upsert(stale with { Cursor = "cursor-stale", CopiedCount = 99 })); + + var stored = (await store.Read("EndpointSettings"))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Cursor, Is.EqualTo("cursor-first")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + Assert.That(stored.Version, Is.EqualTo(first.Version + 1)); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs index 15d9b76981..5e238f2d22 100644 --- a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs @@ -10,6 +10,13 @@ public interface IPersistenceTestsContext { Task Setup(IHostApplicationBuilder hostBuilder); + /// <summary> + /// Puts the schema in place on the built host, before it is started, the way <c>--setup</c> does in + /// production. The EF Core persisters refuse to start against a database whose schema predates the + /// build, and that check runs before any hosted service, so migrating after the start is too late. + /// </summary> + Task InstallSchema(IHost host); + Task PostSetup(IHost host); Task TearDown(); diff --git a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs index f065c190f6..dc5ae1178e 100644 --- a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs @@ -53,6 +53,7 @@ public async Task SetUp() host = hostBuilder.Build(); + await PersistenceTestsContext.InstallSchema(host); await host.StartAsync(); await PersistenceTestsContext.PostSetup(host); } diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs new file mode 100644 index 0000000000..08f6365e87 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// <summary> +/// The rule that stops a category early because it has lost too much to keep copying. +/// </summary> +public static class HaltThreshold +{ + /// <summary> + /// Returns true, which stops the category, when skippedCount passes minimumFloor and its share of totalCount passes percentThreshold percent. + /// The floor ignores a few bad rows in a small category, and the percentage ignores a small share of a large one. + /// skippedCount holds fault skips only, never a harmless skip that <see cref="MigrationSkipReasonExtensions.IsBenign" /> accepts, and + /// totalCount holds every row dealt with over the same stretch, copied, skipped and already present alike. + /// </summary> + public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) + { + if (skippedCount <= minimumFloor || totalCount == 0) + { + return false; + } + + var percent = skippedCount * 100m / totalCount; + return percent > percentThreshold; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs new file mode 100644 index 0000000000..0731525dab --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -0,0 +1,146 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +/// <summary> +/// Where one category has got to. <see cref="MigrationCategoryStateExtensions.IsFinished" /> says which of these +/// let the host open, and <see cref="MigrationCategoryStateExtensions.IsFailed" /> which wait for the operator. A +/// start picks up every category that is neither. +/// </summary> +public enum MigrationCategoryState +{ + /// <summary>No run has read a row of this category yet.</summary> + NotStarted, + + /// <summary> + /// A run is copying this category, or a run stopped without settling it. On an optional category, a LastError + /// means an exception stopped it or a start could not open the source, and the next start resumes it. + /// </summary> + InProgress, + + /// <summary> + /// The category reached the end of the source with no fault skips. Harmless ones stay counted. + /// </summary> + Complete, + + /// <summary> + /// The category reached the end of the source with fault skips, so it is Failed until the operator retries or abandons it. + /// </summary> + CompleteWithErrors, + + /// <summary> + /// The category stopped early. It waits for --migration-retry or --migration-abandon, and no start re-reads it. + /// </summary> + Halted, + + /// <summary> + /// The operator gave up on a Failed or started required category, on any optional one, or on one whose rows + /// depend on such a category. It is final: nothing copies it again, even in a later migration. + /// </summary> + Abandoned, + + /// <summary> + /// This category follows a category that is not Done or Abandoned, so it did not run. A restart clears it once that one is. + /// </summary> + Blocked +} + +/// <summary> +/// A category's saved progress: where a restart carries on from, and what the status and verify commands report. +/// Every count is the total across every run, not this run alone. +/// </summary> +/// <param name="Cursor">The point the last committed batch reached. A restart reads the source after it. Null means nothing has been read.</param> +/// <param name="SourceTotal">No longer written: the engine does not count the source, so this is null on every row it saves. The column stays only because dropping it is a schema change.</param> +/// <param name="SkipReasons">How many rows each reason skipped. The counts here add up to <paramref name="SkippedCount" />.</param> +/// <param name="SettledAt">The moment the category stopped running, whatever state it stopped in. Read it beside <paramref name="State" />, because a halt settles too.</param> +/// <param name="LastError">Why the category stopped or did not run, in the words the operator is shown: a halt, a category it must follow that is not finished, an exception that left an optional category copying, or a source a start could not open, on an optional category still copying or not started. It stays until a start runs the category again.</param> +/// <param name="AlreadyPresentCount">Rows the target already held, so they were neither copied nor skipped. They still count as accounted for.</param> +/// <param name="Version">The optimistic concurrency token, which is the guard against two writers. A store sets it on save and refuses a checkpoint carrying a value the stored row no longer holds.</param> +/// <param name="StartedWindowSeconds">The window, in whole seconds, an optional category started with. It is null until the copier writes it, and always null for a required category.</param> +public sealed record MigrationCheckpoint( + string CategoryId, + MigrationCategoryState State, + string? Cursor, + long CopiedCount, + long SkippedCount, + long? SourceTotal, + IReadOnlyDictionary<MigrationSkipReason, long>? SkipReasons, + DateTime? StartedAt, + DateTime? LastProgressAt, + DateTime? SettledAt, + string? LastError, + long AlreadyPresentCount = 0, + long Version = 0, + long? StartedWindowSeconds = null) +{ + /// <summary> + /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, + /// so the saved counts are the real ones. + /// </summary> + /// <param name="copied">Rows this batch wrote.</param> + /// <param name="skipped">Rows this batch could not write. Every one of them needs a reason.</param> + /// <param name="alreadyPresent">Rows this batch found the target already held.</param> + /// <param name="skipReasons">How many rows each reason skipped in this batch.</param> + /// <returns>A copy with this batch's counts and reasons added to the totals.</returns> + /// <exception cref="InvalidOperationException">The reasons do not add up to <paramref name="skipped" />, which would leave the verify command unable to account for a row.</exception> + public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, IReadOnlyDictionary<MigrationSkipReason, long>? skipReasons) + { + var explained = skipReasons?.Values.Sum() ?? 0; + if (explained != skipped) + { + throw new InvalidOperationException($"The target reported {skipped} skipped rows in category {CategoryId} but gave reasons for {explained}. Every skipped row needs a reason, or --migration-verify cannot account for it."); + } + + return this with + { + CopiedCount = CopiedCount + copied, + SkippedCount = SkippedCount + skipped, + AlreadyPresentCount = AlreadyPresentCount + alreadyPresent, + SkipReasons = AddSkipReasons(SkipReasons, skipReasons) + }; + } + + internal static IReadOnlyDictionary<MigrationSkipReason, long>? AddSkipReasons(IReadOnlyDictionary<MigrationSkipReason, long>? totals, IReadOnlyDictionary<MigrationSkipReason, long>? additions) + { + if (additions is not { Count: > 0 }) + { + return totals; + } + + Dictionary<MigrationSkipReason, long> sum = totals is null ? [] : new(totals); + foreach (var (reason, count) in additions) + { + sum[reason] = sum.GetValueOrDefault(reason) + count; + } + + return sum; + } +} + +/// <summary> +/// Where the checkpoints are kept. The target database holds them, so progress and the rows it describes +/// commit together. +/// </summary> +public interface IMigrationCheckpointStore +{ + /// <summary> + /// Every checkpoint the store holds. A run saves a not-started row for every category it will copy before it + /// copies any of them, so a category is absent when no run has recorded it. + /// </summary> + Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default); + + /// <summary> + /// One category's checkpoint, or null when no run has recorded it. + /// </summary> + Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default); + + /// <summary> + /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. + /// </summary> + /// <exception cref="MigrationCheckpointConflictException">The stored row has moved on, which means another writer saved it, so this save is refused.</exception> + Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs new file mode 100644 index 0000000000..d6c40216b4 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// <summary> +/// Implemented by a persister that can be read as the old database a migration copies from. Nothing here +/// writes to the source, because the old database has to stay usable if the migration is thrown away. +/// </summary> +public interface IMigrationSource : IAsyncDisposable +{ + /// <summary> + /// Connects to the source read-only. Every other member except <see cref="SupportedCategoryIds" /> throws + /// until this has run. + /// </summary> + Task Open(CancellationToken cancellationToken = default); + + /// <summary> + /// The checks this source wants run before the copy starts, in the order they must run. Call it after Open. + /// </summary> + IReadOnlyList<IMigrationStartupCheck> ContributedChecks(); + + /// <summary> + /// What the source report and the dry run print about the source. + /// </summary> + Task<MigrationSourceDescription> Describe(CancellationToken cancellationToken = default); + + /// <summary> + /// A count of everything the source holds, including data no category copies. + /// </summary> + Task<IReadOnlyList<MigrationSourceInventoryEntry>> Inventory(CancellationToken cancellationToken = default); + + /// <summary> + /// How many rows the source holds for one category, which --migration-verify reports. + /// </summary> + Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// <summary> + /// Reads a category in batches, after the checkpoint cursor, or from the start when it is null. Throws on a cursor it never issued. + /// </summary> + IAsyncEnumerable<MigrationBatch> Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default); + + /// <summary> + /// Reads one row's message body when Read did not attach it. The engine calls it only for a category that + /// carries bodies, and only for a row whose body Read left null. Returns null when the source has no body for the row. + /// </summary> + Task<MigrationBody?> ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); + + /// <summary> + /// The categories this source can read. A category outside this set is never handed to the engine. + /// Answers before Open and does not change across it. + /// </summary> + IReadOnlyCollection<string> SupportedCategoryIds { get; } + + /// <summary> + /// The type of <see cref="MigrationRow.Document" /> in every row this source reads, for each category in + /// <see cref="SupportedCategoryIds" />. The target's writer for the same category must take that type. + /// Answers before Open and does not change across it. + /// </summary> + IReadOnlyDictionary<string, Type> DocumentTypes { get; } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs new file mode 100644 index 0000000000..1f4de16640 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Configuration; + +/// <summary>Implemented by a persister that can be read as the old database a migration copies from.</summary> +public interface IMigrationSourceFactory +{ + /// <summary>Builds the source from the instance's own settings, without connecting to it.</summary> + IMigrationSource CreateSource(SettingsRootNamespace settingsRoot); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs new file mode 100644 index 0000000000..66af36395d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Threading; +using System.Threading.Tasks; + +/// <summary> +/// One check the host runs before the copy starts. The host, the source and the target each contribute their own. +/// </summary> +public interface IMigrationStartupCheck +{ + /// <summary> + /// What the check is called in the refusal, so it is phrased to finish the sentence + /// "Migration check '...' failed". + /// </summary> + string Name { get; } + + /// <summary> + /// Runs the check. Throws when it fails, which stops the host before anything is copied. + /// </summary> + Task Run(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs new file mode 100644 index 0000000000..3dd50089d9 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -0,0 +1,63 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// <summary> +/// Implemented by a persister that can be the new database a migration copies into. +/// </summary> +public interface IMigrationTarget +{ + /// <summary> + /// Makes the target ready to write. Called once, before the copy starts. + /// </summary> + Task Open(CancellationToken cancellationToken = default); + + /// <summary> + /// The most rows this target can write in one batch for the category. A bigger batch fails. + /// </summary> + Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default); + + /// <summary> + /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. + /// Build the checkpoint with <see cref="MigrationCheckpoint.Extend"/>. + /// Bodies kept outside the database are written first, so a failure can leave an unused body but never a row without one. + /// </summary> + Task<MigrationWriteResult> Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default); + + /// <summary> + /// How many rows the target holds for one category, even where categories share a table. + /// </summary> + Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// <summary> + /// Saves a reset checkpoint so its category can be copied again, and returns it with its new version. + /// Where the database makes the row keys, it also deletes the rows already copied, so none are copied twice. + /// Throws <see cref="MigrationCheckpointConflictException" /> if another writer saved the checkpoint first. + /// </summary> + Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default); + + /// <summary> + /// The categories this target can write. Known before Open and never changes. + /// </summary> + IReadOnlyCollection<string> SupportedCategoryIds { get; } + + /// <summary> + /// The <see cref="MigrationRow.Document" /> type this target expects for each supported category. + /// A row of any other type fails its batch. Known before Open and never changes. + /// </summary> + IReadOnlyDictionary<string, Type> DocumentTypes { get; } +} + +/// <summary> +/// What the target did with one batch, and the checkpoint it saved with it. Every skipped row has a reason in SkipReasons. +/// Merged counts the rows inside AlreadyPresent that the target dropped because their key matched another row's. +/// Changed counts the rows inside Copied whose stored value differs from the source's on purpose. The target logs each merged and changed row. +/// </summary> +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList<string> SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary<MigrationSkipReason, long>? SkipReasons = null, int Merged = 0, int Changed = 0); diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs new file mode 100644 index 0000000000..8cde500f8f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// <summary> +/// The target's own startup checks, plus a marker kept in the target database saying a host has already started on it. +/// </summary> +public interface IMigrationTargetReadiness +{ + /// <summary> + /// The checks this target wants run before the copy starts, in the order they must run. + /// </summary> + IReadOnlyList<IMigrationStartupCheck> ContributedChecks(); + + /// <summary> + /// Stamps the marker the first time a host opens on the target, and leaves that first stamp alone on every start after it. + /// </summary> + Task RecordHostOpened(CancellationToken cancellationToken = default); + + /// <summary> + /// Whether a host has already opened on the target, which is what makes discarding a partial copy a loss rather than a clean abort. + /// </summary> + Task<bool> HasHostOpened(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs new file mode 100644 index 0000000000..7a36324b40 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; + +/// <summary> +/// When a category is copied. Required categories are copied with ServiceControl closed, because the host must +/// not serve a half copied instance. Optional ones are copied in the background once it is open. +/// </summary> +public enum MigrationCategoryKind { Required, Optional } + +/// <summary>One kind of data, such as endpoint settings, copied as a unit and resumed from its own cursor.</summary> +/// <param name="Id">The name in <see cref="MigrationCategoryIds" />, which is also the key of its checkpoint row.</param> +/// <param name="CarriesBodies">Whether its rows have message bodies, which the engine fetches separately when the source does not attach them.</param> +/// <param name="Order">The order within one kind, counting from 1. Required and optional both start at 1, so the two lists are never sorted together.</param> +/// <param name="MustFollow">The category that has to finish first, or null when nothing has to. A category whose predecessor is unfinished is recorded as <see cref="MigrationCategoryState.Blocked" /> instead of running.</param> +public sealed record MigrationCategory( + string Id, + MigrationCategoryKind Kind, + bool CarriesBodies, + int Order, + string? MustFollow = null); + +/// <summary>A message body read from the source, as bytes plus its content type.</summary> +public sealed record MigrationBody(ReadOnlyMemory<byte> Content, string ContentType); + +/// <summary>One item read from the source.</summary> +/// <param name="SourceId">The row's identifier in the old database. It is what the engine logs when the row is skipped, and what <see cref="IMigrationSource.ReadBody" /> takes.</param> +/// <param name="Document">The row itself, as the object the source read. The target's writer for that category is what knows the type.</param> +/// <param name="Metadata">Facts about the row that the document does not carry. Empty when the source has none to add.</param> +/// <param name="Body">The message body, or null when the source did not attach one and the engine has not fetched it.</param> +public sealed record MigrationRow( + string SourceId, + object Document, + IReadOnlyDictionary<string, object?> Metadata, + MigrationBody? Body = null); + +/// <summary>Rows read from the source together, plus the cursor to resume after them.</summary> +/// <param name="Rows">The rows, which can be empty when every document in this stretch turned into no row. The cursor past them still has to be saved.</param> +/// <param name="Cursor">Where the source got to. A later read handed this back carries on after it.</param> +public sealed record MigrationBatch(IReadOnlyList<MigrationRow> Rows, string Cursor); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs new file mode 100644 index 0000000000..e8af7f8ab2 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBodyLoader.cs @@ -0,0 +1,129 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +/// <summary> +/// The rows <see cref="MigrationBodyLoader.Load" /> returns for one batch, each with its body, and the rows it left out +/// with the reason for each. +/// </summary> +sealed record MigrationBodyLoadResult(IReadOnlyList<MigrationRow> Rows, IReadOnlyList<(string SourceId, MigrationSkipReason Reason)> Skipped); + +/// <summary> +/// Attaches a message body to every row of a batch that arrived without one, reading at most +/// <see cref="MaxConcurrentFetches" /> bodies from the source at a time. +/// </summary> +sealed class MigrationBodyLoader(IMigrationSource source, TimeProvider timeProvider, TimeSpan retryBackoff, ILogger logger) +{ + /// <summary> + /// How many body reads are open against the source at once. Each body is its own round trip, so reading them one at + /// a time makes the copy as slow as the link, while opening a whole batch at once puts a load spike on a source that + /// may still be live. + /// </summary> + public const int MaxConcurrentFetches = 8; + + /// <summary> + /// Gives every row without a body the body the source holds for it. A row that already has one keeps it and + /// is never read. A read that fails is tried up to <see cref="MigrationEngine.MaxBodyReadAttempts" /> times, + /// and a row whose every attempt fails is left out as <see cref="MigrationSkipReason.BodyUnreadable" />. A row + /// the source holds no body for is read once and left out as <see cref="MigrationSkipReason.BodyMissing" />. + /// Each skip is logged as a warning. + /// </summary> + public async Task<MigrationBodyLoadResult> Load(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken = default) + { + using var inFlight = new SemaphoreSlim(MaxConcurrentFetches); + + var fetched = await Task.WhenAll(batch.Rows.Select(async row => + { + if (row.Body is not null) + { + return (row.Body, LastAttemptError: null); + } + + // The slot is held for the whole fetch, retries included, so each message uses one slot rather than one per attempt. + await inFlight.WaitAsync(cancellationToken); + + try + { + return await Fetch(category, row.SourceId, cancellationToken); + } + finally + { + inFlight.Release(); + } + })); + + var rows = new List<MigrationRow>(batch.Rows.Count); + var skipped = new List<(string SourceId, MigrationSkipReason Reason)>(); + + // Rebuilt in the batch's order, because that is the cursor's order and the fetches finish in any order. + for (var index = 0; index < batch.Rows.Count; index++) + { + var row = batch.Rows[index]; + var (body, lastAttemptError) = fetched[index]; + + if (body is not null) + { + rows.Add(row with { Body = body }); + } + else if (lastAttemptError is null) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId} as {Reason}: the source has no body for it, either because it holds the message without one, which a --migration-retry meets again, or because the message expired during the copy", + row.SourceId, category.Id, MigrationSkipReason.BodyMissing); + skipped.Add((row.SourceId, MigrationSkipReason.BodyMissing)); + } + else + { + logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId} as {Reason}: body unreadable after {MaxAttempts} attempts", row.SourceId, category.Id, MigrationSkipReason.BodyUnreadable, MigrationEngine.MaxBodyReadAttempts); + skipped.Add((row.SourceId, MigrationSkipReason.BodyUnreadable)); + } + } + + return new MigrationBodyLoadResult(rows, skipped); + } + + // A null body with no error means the source holds no body; with an error, every attempt failed. + async Task<(MigrationBody? Body, Exception? LastAttemptError)> Fetch(MigrationCategory category, string sourceId, CancellationToken cancellationToken) + { + Exception? lastAttemptError = null; + + for (var attempt = 1; attempt <= MigrationEngine.MaxBodyReadAttempts; attempt++) + { + try + { + return (await source.ReadBody(category, sourceId, cancellationToken), null); + } + // Only the run's own cancellation stops here; a client timeout raised as a cancellation is retried. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Logged here because the exception that halts the category does not say which message it was reading. + catch (Exception ex) when (IsDefect(ex)) + { + logger.LogError(ex, "Reading the body of {SourceId} in category {CategoryId} failed in a way no retry fixes, so the batch fails", sourceId, category.Id); + throw; + } + catch (Exception ex) + { + lastAttemptError = ex; + logger.LogWarning(ex, "Reading the body of {SourceId} failed on attempt {Attempt} of {MaxAttempts}", sourceId, attempt, MigrationEngine.MaxBodyReadAttempts); + + if (attempt < MigrationEngine.MaxBodyReadAttempts && retryBackoff > TimeSpan.Zero) + { + await Task.Delay(retryBackoff, timeProvider, cancellationToken); + } + } + } + + return (null, lastAttemptError); + } + + // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. + static bool IsDefect(Exception exception) => + exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs new file mode 100644 index 0000000000..6670fcfbcb --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// <summary> +/// The name of every category a migration can copy. The name is what the checkpoint row is keyed on, so renaming +/// one strands the progress already saved under the old name. <see cref="MigrationCategoryRegistry" /> says when each one is copied. +/// </summary> +public static class MigrationCategoryIds +{ + public const string KnownEndpoints = nameof(KnownEndpoints); + public const string EndpointSettings = nameof(EndpointSettings); + public const string MessageRedirects = nameof(MessageRedirects); + public const string Subscriptions = nameof(Subscriptions); + public const string NotificationSettings = nameof(NotificationSettings); + public const string TrialEndDate = nameof(TrialEndDate); + public const string RetryOperations = nameof(RetryOperations); + public const string LicensingEndpoints = nameof(LicensingEndpoints); + public const string LicensingThroughput = nameof(LicensingThroughput); + public const string LicensingReportMasks = nameof(LicensingReportMasks); + public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); + public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + public const string PendingIntegrationEvents = nameof(PendingIntegrationEvents); + + public const string EventLog = nameof(EventLog); + public const string CustomChecks = nameof(CustomChecks); + public const string FailedErrorImports = nameof(FailedErrorImports); + public const string ArchivedAndResolvedFailedMessages = nameof(ArchivedAndResolvedFailedMessages); + public const string GroupComments = nameof(GroupComments); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs new file mode 100644 index 0000000000..e3db874daf --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.MessageFailures; + +/// <summary> +/// Every category a migration can copy, and when each one is copied. This list is the whole of what a migration +/// covers: a source or a target that cannot handle a category says so through its own supported ids, and the +/// engine never invents one. +/// </summary> +public static class MigrationCategoryRegistry +{ + /// <summary> + /// The failed message statuses that are copied before the host opens. A message in one of these is waiting on + /// somebody, so an operator who cannot see it after the cutover has lost work. + /// </summary> + public static readonly IReadOnlyList<FailedMessageStatus> UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + + /// <summary> + /// The failed message statuses that are copied in the background. These are history, so the instance is usable + /// while they are still arriving. + /// </summary> + public static readonly IReadOnlyList<FailedMessageStatus> ArchivedAndResolvedStatuses = [FailedMessageStatus.Archived, FailedMessageStatus.Resolved]; + + public static readonly IReadOnlyList<MigrationCategory> All = + [ + // Required, copied with ServiceControl closed. + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 2, MustFollow: MigrationCategoryIds.KnownEndpoints), + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryKind.Required, CarriesBodies: false, Order: 3), + new(MigrationCategoryIds.Subscriptions, MigrationCategoryKind.Required, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.NotificationSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 5), + new(MigrationCategoryIds.TrialEndDate, MigrationCategoryKind.Required, CarriesBodies: false, Order: 6), + new(MigrationCategoryIds.RetryOperations, MigrationCategoryKind.Required, CarriesBodies: false, Order: 7), + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 8), + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryKind.Required, CarriesBodies: false, Order: 9, MustFollow: MigrationCategoryIds.LicensingEndpoints), + new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), + new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + new(MigrationCategoryIds.PendingIntegrationEvents, MigrationCategoryKind.Required, CarriesBodies: false, Order: 13), + new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 14), + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Required, CarriesBodies: true, Order: 15), + new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Required, CarriesBodies: false, Order: 16, MustFollow: MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), + + // Optional, copied in the background once the host is open. + new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 2), + ]; + + public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs new file mode 100644 index 0000000000..d68b8e6e61 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs @@ -0,0 +1,47 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class MigrationCategoryStateExtensions +{ + /// <summary> + /// Whether the category is done with, so a restart passes over it and it no longer holds the host back: Done, + /// or Abandoned by the operator. A Failed category (<see cref="IsFailed" />) is not finished. It waits for + /// --migration-retry or --migration-abandon, and a category that must follow it stays blocked until then. + /// </summary> + public static bool IsFinished(this MigrationCategoryState state) => + state is MigrationCategoryState.Complete + or MigrationCategoryState.Abandoned; + + /// <summary> + /// Whether the category is Failed: it stopped early, or reached the end having skipped rows to faults. It waits + /// for the operator to run --migration-retry or --migration-abandon, and no start copies it until then. + /// </summary> + public static bool IsFailed(this MigrationCategoryState state) => + state is MigrationCategoryState.Halted + or MigrationCategoryState.CompleteWithErrors; +} + +public static class MigrationSkipReasonExtensions +{ + /// <summary> + /// Whether the skip is no loss, because the running product would have dropped the row anyway, or the row + /// is a result of a check the source's own storage runs about itself, which nothing reports on the target. The halt threshold and the settle rule never count a + /// harmless skip, so no number of them can stop a category or leave it Failed. The list is fixed, and + /// <see cref="MigrationSkipReason.Unknown" /> is not on it, because a newer build's reason read back here + /// may be a real loss. + /// </summary> + public static bool IsBenign(this MigrationSkipReason reason) => + reason is MigrationSkipReason.PastRetention + or MigrationSkipReason.BlankGroupComment + or MigrationSkipReason.SourceStorageCheck; + + /// <summary> + /// Whether no retry can fix a row skipped for this reason. + /// </summary> + public static bool IsPermanent(this MigrationSkipReason reason) => + reason is MigrationSkipReason.RequiredValueMissing + or MigrationSkipReason.UnparseableUniqueMessageId + or MigrationSkipReason.NoProcessingAttempts + or MigrationSkipReason.ValueTooLong + or MigrationSkipReason.MalformedValue + or MigrationSkipReason.BodyMissing; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs new file mode 100644 index 0000000000..cc2822822f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// <summary>Thrown when a checkpoint save carries a version the stored row no longer holds, because another writer moved it on.</summary> +public sealed class MigrationCheckpointConflictException(string message, Exception? innerException = null) : Exception(message, innerException); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs new file mode 100644 index 0000000000..77c3fcbd4e --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -0,0 +1,262 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +/// <summary> +/// Copies one category at a time from the source to the target, and records where it got to after every batch. +/// The engine knows nothing about either database: what a row is, how it is written and how big a batch can be +/// all come from the source and the target. A category that fails comes back as a Failed checkpoint rather than +/// an exception, and no later run copies it until the operator retries or abandons it. An optional category an +/// exception stopped comes back still copying instead, with the error on its row. A shutdown and a checkpoint +/// conflict come out as exceptions, because neither is the category's fault, and so does a failure to save the +/// checkpoint a category stopped or settled on, because then there is no row to record it on. +/// </summary> +public sealed class MigrationEngine( + IMigrationSource source, + IMigrationTarget target, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + MigrationEngineOptions options, + ILogger<MigrationEngine> logger) +{ + /// <summary> + /// How many times a message body is read before the row is skipped as unreadable. + /// </summary> + public const int MaxBodyReadAttempts = 3; + + readonly MigrationBodyLoader bodyLoader = new(source, timeProvider, options.BodyRetryBackoff, logger); + + readonly ConcurrentDictionary<string, (long Merged, long Changed)> mergedAndChangedThisRun = new(StringComparer.Ordinal); + + /// <summary> + /// For each category this engine wrote to, how many rows the target merged into another row and how many it changed on the way in. + /// The counts are not saved, so they cover only the batches this engine wrote, and a category it resumed counts none from earlier runs. + /// </summary> + public IReadOnlyDictionary<string, (long Merged, long Changed)> MergedAndChangedThisRun => mergedAndChangedThisRun; + + /// <summary> + /// The categories of one kind to copy, in the order to copy them. Optional categories the operator did not + /// ask for are left out. + /// </summary> + public IReadOnlyList<MigrationCategory> SelectCategories(MigrationCategoryKind kind) => + MigrationCategoryRegistry.All + .Where(c => c.Kind == kind) + .Where(c => kind == MigrationCategoryKind.Required || options.SelectedOptionalCategoryIds.Contains(c.Id)) + .OrderBy(c => c.Order) + .ToArray(); + + /// <summary> + /// Copies the categories one after another and returns where each one ended, in the same order. A category + /// that halts does not stop the ones after it. Before copying anything it saves a not-started checkpoint for + /// every category that has none, so a copy stopped between two categories still lists the ones it never reached. + /// </summary> + // Runs in the order given without re-sorting: required and optional orders both start at 1, so + // sorting a mixed list would put an optional category in front of a required one. + public async Task<IReadOnlyList<MigrationCheckpoint>> RunCategories( + IReadOnlyList<MigrationCategory> categories, + CancellationToken cancellationToken = default) + { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(NotStarted(category), cancellationToken); + } + } + + var results = new List<MigrationCheckpoint>(categories.Count); + + foreach (var category in categories) + { + results.Add(await RunCategoryAsync(category, cancellationToken)); + } + + return results; + } + + /// <summary> + /// Copies one category, carrying on from its saved cursor, and returns the checkpoint it ended on. A category + /// already finished or Failed is returned untouched without reading the source. + /// </summary> + public async Task<MigrationCheckpoint> RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); + + if (checkpoint.State.IsFinished() || checkpoint.State.IsFailed()) + { + return checkpoint; + } + + if (category.MustFollow is { } mustFollowId) + { + var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); + if (predecessor?.State.IsFinished() != true) + { + var predecessorState = predecessor?.State.ToString() ?? "not started"; + var blocked = checkpoint with + { + State = MigrationCategoryState.Blocked, + LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" + }; + + logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", + category.Id, mustFollowId, predecessorState); + + return await checkpointStore.Upsert(blocked, cancellationToken); + } + } + + // A LastError on a row still copying is cleared here, because this start is trying it again. + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Blocked || checkpoint.LastError is not null) + { + checkpoint = await checkpointStore.Upsert(checkpoint with + { + State = MigrationCategoryState.InProgress, + StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, + SettledAt = null, + LastError = null + }, cancellationToken); + } + + var isFirstBatch = true; + // Per run, not the persisted totals: the skips an earlier run made stay on the row, so counting them + // again would stop a run resumed from its cursor on its first batch. + var skippedThisRun = 0L; + var processedThisRun = 0L; + var rowsReadThisRun = 0L; + // Saved after the try rather than inside it, so a failed save is not caught below as the category's own error. + MigrationCheckpoint? halt = null; + + try + { + var batchSize = await target.BatchSizeFor(category, cancellationToken); + + await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) + { + if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) + { + await Pause(options.ThrottlePause, cancellationToken); + } + isFirstBatch = false; + rowsReadThisRun += batch.Rows.Count; + + var batchToWrite = batch; + // Kept out of checkpoint until the write commits: the catch below saves checkpoint, and a + // restart re-reads the uncommitted batch and would count these skips twice. + var bodySkips = 0; + Dictionary<MigrationSkipReason, long>? bodySkipReasons = null; + if (category.CarriesBodies) + { + var loaded = await bodyLoader.Load(category, batch, cancellationToken); + batchToWrite = batch with { Rows = loaded.Rows }; + bodySkips = loaded.Skipped.Count; + bodySkipReasons = loaded.Skipped.CountBy(skip => skip.Reason).ToDictionary(count => count.Key, count => (long)count.Value); + } + + // The target adds its own outcome inside the transaction that writes the rows, + // so nothing provisional is ever stored. + var checkpointToExtend = checkpoint with + { + Cursor = batch.Cursor, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, + SkippedCount = checkpoint.SkippedCount + bodySkips, + SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkipReasons) + }; + + var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); + + // The target has committed this row, so every halt below settles from it. A halt settling from the + // older version would be refused by the store as a conflict and lose its reason. + checkpoint = result.Saved; + mergedAndChangedThisRun.AddOrUpdate(category.Id, (result.Merged, result.Changed), (_, sum) => (sum.Merged + result.Merged, sum.Changed + result.Changed)); + + // The result states the batch's outcome twice, as its own counts and as deltas on the checkpoint + // it committed. The halt threshold reads the first and status and verify read the second. + var committed = (result.Saved.CopiedCount - checkpointToExtend.CopiedCount, result.Saved.SkippedCount - checkpointToExtend.SkippedCount, result.Saved.AlreadyPresentCount - checkpointToExtend.AlreadyPresentCount); + if (committed != (result.Copied, result.Skipped, result.AlreadyPresent)) + { + var mismatch = $"The target reported copying {result.Copied}, skipping {result.Skipped} and finding {result.AlreadyPresent} already present in category {category.Id}, but the checkpoint it committed moved by {committed}."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, mismatch); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = mismatch }; + break; + } + + processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; + skippedThisRun += bodySkips + FaultSkips(result.SkipReasons); + + if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) + { + var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }; + break; + } + } + } + // A shutdown is not a halt: the last committed batch saved its counts with its own rows, + // so the checkpoint on disk is already correct and resumable. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Another writer holds this row, which no amount of halting resolves. Leave the state alone so their row stands. + catch (MigrationCheckpointConflictException) + { + throw; + } + catch (Exception ex) + { + var position = checkpoint.Cursor is null ? "at the start" : $"at cursor {checkpoint.Cursor}"; + var reason = $"{ex.GetType().Name} {position}: {ex.Message}"; + + // An optional category copies while ServiceControl serves, so an error there waits for the next start rather than for the operator. + if (category.Kind == MigrationCategoryKind.Optional) + { + logger.LogError(ex, "Optional category {CategoryId} stopped at cursor {Cursor} and stays copying until the next start", category.Id, checkpoint.Cursor); + return await checkpointStore.Upsert(checkpoint with { LastError = $"{reason.TrimEnd('.')}. It stays copying and the next start resumes it from the cursor." }, cancellationToken); + } + + logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + if (halt is not null) + { + return await Settle(halt, cancellationToken); + } + + // The one check a category runs on itself, judged over this run because the counts on a resumed row cover earlier runs. + if (rowsReadThisRun != processedThisRun) + { + var reason = $"Halted: {category.Id} read {rowsReadThisRun} rows in this run but the target accounted for {processedThisRun} of them as copied, skipped or already present."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + return await Settle(checkpoint with { State = FaultSkips(checkpoint.SkipReasons) > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); + } + + // A harmless skip loses nothing, as IsBenign says, so it neither stops a category nor leaves it Failed. + static long FaultSkips(IReadOnlyDictionary<MigrationSkipReason, long>? skipReasons) => + skipReasons is null ? 0 : skipReasons.Where(reason => !reason.Key.IsBenign()).Sum(reason => reason.Value); + + // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. + Task<MigrationCheckpoint> Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => + checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); + + // Zero means no throttling, and it must return without waiting: against a fake clock + // nobody advances, even a zero-length wait never finishes. + Task Pause(TimeSpan duration, CancellationToken cancellationToken) => + duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); + + static MigrationCheckpoint NotStarted(MigrationCategory category) => + new(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs new file mode 100644 index 0000000000..8dee699d9f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using ServiceControl.Configuration; + +/// <summary> +/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories +/// to copy and how far back. +/// </summary> +/// <param name="ThrottlePause">How long to wait between batches of an optional category, which is what keeps the background copy off the instance's back. Zero waits not at all.</param> +/// <param name="HaltThresholdPercent">The share of skipped rows, as a percentage, that stops a category.</param> +/// <param name="HaltThresholdMinimum">The number of skipped rows that has to be passed before the percentage counts, so a handful of bad rows in a small category is not a halt.</param> +/// <param name="SelectedOptionalCategoryIds">The optional categories to copy. An optional category outside this list is never copied.</param> +public sealed record MigrationEngineOptions( + TimeSpan ThrottlePause, + int HaltThresholdPercent, + int HaltThresholdMinimum, + IReadOnlyCollection<string> SelectedOptionalCategoryIds) +{ + public static readonly TimeSpan DefaultBodyRetryBackoff = TimeSpan.FromMilliseconds(200); + + /// <summary>How long to wait before trying again to read a message body that failed. Not read from settings.</summary> + public TimeSpan BodyRetryBackoff { get; init; } = DefaultBodyRetryBackoff; + + /// <summary> + /// How far back the event log copy goes, measured against when each event was raised. + /// </summary> + public TimeSpan EventLogWindow { get; init; } + + /// <summary> + /// How far back the archived and resolved failed messages copy goes, measured against when each message was + /// archived or resolved. + /// </summary> + public TimeSpan ArchivedAndResolvedFailedMessagesWindow { get; init; } + + /// <summary> + /// Reads the options from the instance's settings. Anything not set takes the default from + /// <see cref="MigrationSettings" />, except the two windows, which default to the retention periods passed in. + /// An optional category is selected when its window is greater than zero. + /// </summary> + /// <param name="settingsRootNamespace">The settings root the migration keys are read under.</param> + /// <param name="eventRetentionPeriod">The instance's event retention period, used as the event log window when none is set.</param> + /// <param name="errorRetentionPeriod">The instance's error retention period, used as the archived and resolved window when none is set.</param> + /// <exception cref="InvalidOperationException">A window setting is not a time span of zero or more, which the operator has to see before the copy starts.</exception> + public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace, TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) + { + var throttleMilliseconds = SettingsReader.Read(settingsRootNamespace, MigrationSettings.ThrottlePauseMillisecondsKey, MigrationSettings.DefaultThrottlePauseMilliseconds); + var haltPercent = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdPercentKey, MigrationSettings.DefaultHaltThresholdPercent); + var haltMinimum = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdMinimumKey, MigrationSettings.DefaultHaltThresholdMinimum); + var eventLogWindow = ReadWindow(settingsRootNamespace, MigrationSettings.EventLogWindowKey, eventRetentionPeriod); + var archivedAndResolvedWindow = ReadWindow(settingsRootNamespace, MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey, errorRetentionPeriod); + + var selectedIds = new List<string>(); + + if (eventLogWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.EventLog); + } + + if (archivedAndResolvedWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + } + + return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds) + { + EventLogWindow = eventLogWindow, + ArchivedAndResolvedFailedMessagesWindow = archivedAndResolvedWindow + }; + } + + static TimeSpan ReadWindow(SettingsRootNamespace settingsRootNamespace, string key, TimeSpan defaultWindow) + { + var value = SettingsReader.Read<string>(settingsRootNamespace, key); + + if (value is null) + { + return defaultWindow; + } + + if (!TimeSpan.TryParse(value, out var window) || window < TimeSpan.Zero) + { + throw new InvalidOperationException( + $"{key} is '{value}', which is not a time span of zero or more. Set it like a retention period, such as 7.00:00:00 for seven days, or to 0 to leave that category behind."); + } + + return window; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs new file mode 100644 index 0000000000..2ea004b4bf --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationMetadataKeys.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Keys for the extra values a source attaches to a <see cref="MigrationRow" />. The source works each value out +/// itself, so the target never needs to know how the source stores it. +/// </summary> +public static class MigrationMetadataKeys +{ + /// <summary> + /// The instant a failed message moved to its current status, as a UTC <see cref="System.DateTime" />. The retention sweep + /// ages a row out from this instant. + /// </summary> + public const string StatusChangedAt = "StatusChangedAt"; + + /// <summary> + /// The time the source last changed the row's document, as a UTC <see cref="System.DateTime" />. + /// </summary> + public const string LastModified = "LastModified"; + + /// <summary> + /// The time the source last stored a failed error import, as a UTC <see cref="System.DateTime" />. + /// RavenDB keeps no failure time, so the target stores this one as the failure time, and replay runs the oldest first by it. + /// </summary> + public const string FailedAt = "FailedAt"; + + /// <summary> + /// A <see cref="bool" /> that is true when the row is a result of a check the source's own storage runs about itself. + /// Rows that are not such a result carry no value. + /// </summary> + public const string SourceStorageCheck = "SourceStorageCheck"; + + /// <summary> + /// A <see cref="bool" /> on every retry-issued failed message. It is true when the source still holds a retry claim + /// for the message, and false when the source had already released it. Other rows carry no value. + /// </summary> + public const string RetryClaimHeld = "RetryClaimHeld"; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs new file mode 100644 index 0000000000..a5306e95e5 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// <summary> +/// The migration's setting names, relative to the instance's settings root, and their defaults. +/// <see cref="MigrationEngineOptions"/> explains the tuning ones. +/// </summary> +public static class MigrationSettings +{ + public const string ThrottlePauseMillisecondsKey = "Migration/ThrottlePauseMilliseconds"; + public const string HaltThresholdPercentKey = "Migration/HaltThresholdPercent"; + public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; + /// <summary> + /// How far back the event log copy goes, as a time span. Unset means the event retention period, and zero + /// turns the category off. + /// </summary> + public const string EventLogWindowKey = "Migration/EventLogWindow"; + /// <summary> + /// How far back the archived and resolved failed messages copy goes, as a time span. Unset means the error + /// retention period, and zero turns the category off. + /// </summary> + public const string ArchivedAndResolvedFailedMessagesWindowKey = "Migration/ArchivedAndResolvedFailedMessagesWindow"; + public const int DefaultThrottlePauseMilliseconds = 100; + public const int DefaultHaltThresholdPercent = 5; + public const int DefaultHaltThresholdMinimum = 100; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs new file mode 100644 index 0000000000..4c848b4f89 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -0,0 +1,69 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Why one row was not copied. Every skipped row carries one, so the verify command can account for the +/// difference between the two databases. <see cref="MigrationSkipReasonExtensions.IsBenign" /> says which of +/// these are no loss. +/// </summary> +public enum MigrationSkipReason +{ + /// <summary> + /// Every attempt to read the message body from the source failed, so the whole message was left out rather than + /// written without one. A retry reads it again once the cause is fixed. + /// </summary> + BodyUnreadable, + + /// <summary> + /// The row is older than the retention period, so the instance would have deleted it soon anyway. + /// </summary> + PastRetention, + + /// <summary> + /// The source row has no value for something the target column requires. + /// </summary> + RequiredValueMissing, + + /// <summary> + /// The group comment is null or blank, and the product keeps no row for a blank comment. + /// </summary> + BlankGroupComment, + + /// <summary> + /// The source marked the row as a result of a check its own storage runs about itself. + /// Nothing reports such a check once the instance runs on the target, so a copied result would never change. + /// </summary> + SourceStorageCheck, + + /// <summary> + /// The document's unique message id is not a GUID, and the row's key has to be one. + /// </summary> + UnparseableUniqueMessageId, + + /// <summary> + /// The document holds no processing attempt, and every column that describes the failure comes from one. + /// </summary> + NoProcessingAttempts, + + /// <summary> + /// A value is longer than its target column allows, so storing the row would fail the whole insert. + /// </summary> + ValueTooLong, + + /// <summary> + /// A value the target needs is present but holds the wrong type, so the row cannot be built from it. + /// </summary> + MalformedValue, + + /// <summary> + /// The source holds the message but no body for it, so the whole message was left out rather than written + /// without one. + /// </summary> + BodyMissing, + + /// <summary> + /// A reason a newer build wrote and this one cannot name, so an older host still starts instead of throwing. + /// Nothing in this build writes it. It counts as a fault, because harmless would let this build settle a + /// category Done over rows the newer build lost; a false Failed from it is cleared by one retry. + /// </summary> + Unknown +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs new file mode 100644 index 0000000000..3d5692a56d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; + +/// <summary>What the source report and dry run print about the source: its version and a list of facts.</summary> +public sealed record MigrationSourceDescription(string Version, IReadOnlyList<MigrationSourceFact> Facts); + +/// <summary>One line the source report prints, such as which server or database the source read, with the setting to change if it is wrong.</summary> +public sealed record MigrationSourceFact(string Label, string Value, string? SettingKey = null); + +/// <summary>A count the source report prints, such as the rows in one collection of one database, so an operator sees how much there is to copy.</summary> +public sealed record MigrationSourceInventoryEntry(string Scope, string Name, long Count); diff --git a/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs new file mode 100644 index 0000000000..aa5939175e --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/RetryOperationRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Recoverability; + +/// <summary> +/// One retry operation from the retry history document. Exactly one of Historic and Unacknowledged is set. +/// </summary> +public sealed record RetryOperationRow(HistoricRetryOperation? Historic, UnacknowledgedRetryOperation? Unacknowledged); diff --git a/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs new file mode 100644 index 0000000000..3441fd2fe6 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/SubscriptionRow.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// <summary> +/// One subscriber of one message type, on its way from source to target. +/// </summary> +public sealed record SubscriptionRow(string MessageType, Version Version, string TransportAddress, string? Endpoint); diff --git a/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs b/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs new file mode 100644 index 0000000000..bcbfb5431c --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/ThroughputDayRow.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using Particular.LicensingComponent.Contracts; + +/// <summary> +/// One endpoint's message count for one UTC day, as a licensing throughput row crosses from source to target. +/// Endpoint is the endpoint the count belongs to, DateUtc is the day, and MessageCount is that day's whole total. +/// </summary> +public sealed record ThroughputDayRow(EndpointIdentifier Endpoint, DateOnly DateUtc, long MessageCount); diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index 08a73eb1c3..4d87e98f76 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -8,25 +8,44 @@ namespace ServiceControl.Persistence /// </summary> public abstract class PersistenceSettings { + /// <summary> + /// Whether this host starts the database and nothing else, so an operator can repair or inspect it. + /// Only the RavenDB persister supports it. There the embedded server and RavenDB Studio start as + /// usual, and the host registers nothing that ingests or serves data. On any other persister the + /// host refuses to start in maintenance mode. + /// </summary> public bool MaintenanceMode { get; set; } - //HINT: This needs to be here so that ServerControl instance can add an instance specific metadata to tweak the DatabasePath value + + /// <summary> + /// The directory that holds the database files, or null when this host holds none. The RavenDB + /// persister fills it in from its DbPath setting, and its disk space checks measure the drive that + /// this path names. The SQL persisters leave it null, because their files live on the database server. + /// </summary> public string? DatabasePath { get; set; } /// <summary> - /// Whether this host owns the background deletion of data past its retention period. Only one - /// host in a deployment should, so error ingestion only hosts turn it off. + /// Whether this host deletes data that is past its retention period. Only one host in a deployment + /// must do this, so a host that only ingests errors turns it off. The SQL persisters start a + /// background sweeper when it is true. RavenDB expires documents on its own and does not read this. /// </summary> public bool RunRetentionSweep { get; set; } = true; + /// <summary> + /// Whether message bodies are indexed for search. The RavenDB persister decides this as it ingests a + /// message, so a change only reaches messages ingested after it. The SQL persisters always index the + /// body, so the value does not change what they do. + /// </summary> public bool EnableFullTextSearchOnBodies { get; set; } = true; /// <summary> - /// Wall-clock limit for the message view queries, see <see cref="QueryTimeLimit" />. + /// The wall clock limit for a message query, see <see cref="QueryTimeLimit" />. It is also how long + /// this instance waits for a remote instance to answer. /// </summary> public TimeSpan QueryTimeout { get; set; } = QueryTimeLimit.Default; /// <summary> - /// The setting <see cref="QueryTimeout" /> is read from, as named in the timeout error. + /// The name of the setting that <see cref="QueryTimeout" /> comes from. The timeout error names it, so + /// an operator can see which setting to change. /// </summary> public const string QueryTimeoutSettingName = "ServiceControl/" + QueryTimeLimit.SettingName; } diff --git a/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj b/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj index 131f2cfa0e..876ea8a78a 100644 --- a/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj +++ b/src/ServiceControl.Persistence/ServiceControl.Persistence.csproj @@ -6,6 +6,7 @@ </PropertyGroup> <ItemGroup> + <ProjectReference Include="..\Particular.LicensingComponent.Contracts\Particular.LicensingComponent.Contracts.csproj" /> <ProjectReference Include="..\ServiceControl.Configuration\ServiceControl.Configuration.csproj" /> <ProjectReference Include="..\ServiceControl.DomainEvents\ServiceControl.DomainEvents.csproj" /> <ProjectReference Include="..\ServiceControl.Infrastructure\ServiceControl.Infrastructure.csproj" /> diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 14a86520d3..d1b03f3fa2 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -113,6 +113,10 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi return embeddedDatabase; } + // Start only queues the server up, so connecting by the configured URL instead reaches whatever already holds the port. + public async Task<string> WaitUntilReady(CancellationToken cancellationToken = default) => + (await EmbeddedServer.Instance.GetServerUriAsync(cancellationToken)).ToString(); + void Start(ServerOptions serverOptions) { this.serverOptions = serverOptions; diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs index f9ff055506..03183720a3 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs @@ -10,8 +10,8 @@ public class EmbeddedDatabaseConfiguration(string serverUrl, string dbName, stri public bool RunInMemory { get; set; } - // Makes RavenDB throw on a query that would create an auto index, instead of creating it. Tests use it to fail - // on dynamic queries, which must go through a static index. + // Makes RavenDB throw on a query that would create an auto index, instead of creating it. Tests use it to catch + // dynamic queries, and the migration source uses it because creating an index would be a write. public bool DisableAutoIndexes { get; set; } } } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt new file mode 100644 index 0000000000..6c99c6b3e0 --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -0,0 +1,18 @@ +Required 1: KnownEndpoints +Required 2: EndpointSettings, after KnownEndpoints +Required 3: MessageRedirects +Required 4: Subscriptions +Required 5: NotificationSettings +Required 6: TrialEndDate +Required 7: RetryOperations +Required 8: LicensingEndpoints +Required 9: LicensingThroughput, after LicensingEndpoints +Required 10: LicensingReportMasks +Required 11: LicensedEndpointDetails +Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Required 13: PendingIntegrationEvents +Required 14: CustomChecks +Required 15: FailedErrorImports, with bodies +Required 16: GroupComments, after UnresolvedAndRetryIssuedFailedMessages +Optional 1: EventLog +Optional 2: ArchivedAndResolvedFailedMessages, with bodies \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt new file mode 100644 index 0000000000..d0b5240bad --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -0,0 +1,4 @@ +Configured: EventLog +Runs as: +Optional 1: EventLog +Never copied: ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs new file mode 100644 index 0000000000..9363cadd41 --- /dev/null +++ b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.UnitTests.Hosting; + +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceControl.Hosting.Commands; + +[TestFixture] +class MigrationSourceReportArgumentTests +{ + [Test] + public void The_flag_selects_the_report_command() => + Assert.That(new HostArguments(["--migration-source-report"]).Command, Is.EqualTo(typeof(MigrationSourceReportCommand))); + + [Test] + public void The_migrate_flag_selects_the_offline_command() => + Assert.That(new HostArguments(["--start-data-migration"]).Command, Is.EqualTo(typeof(MigrateCommand))); + + [Test] + public void An_ingestion_worker_cannot_run_the_offline_command() => + Assert.That(new HostArguments(["--start-data-migration", "--error-ingestion-only"]).Help, Is.True); + + [Test] + public void No_flag_still_selects_the_run_command() => + Assert.That(new HostArguments([]).Command, Is.EqualTo(typeof(RunCommand))); + + [Test] + public void The_setup_flag_is_undisturbed() => + Assert.That(new HostArguments(["--setup"]).Command, Is.EqualTo(typeof(SetupCommand))); +} diff --git a/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs b/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs new file mode 100644 index 0000000000..464ba86b26 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/AllowUnreleasedMigrationSwitchTests.cs @@ -0,0 +1,37 @@ +#if DEBUG +namespace ServiceControl.UnitTests.Migration; + +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Migration.Checks; + +[TestFixture] +class AllowUnreleasedMigrationSwitchTests +{ + [TestCase("true")] + [TestCase("TRUE")] + public void The_switch_registers_the_marker_when_the_variable_is_true(string value) + { + var services = new ServiceCollection(); + + RunCommand.AllowUnreleasedMigrationWhenAsked(services, value); + + using var provider = services.BuildServiceProvider(); + Assert.That(provider.GetService<AllowUnreleasedMigration>(), Is.Not.Null); + } + + [TestCase(null)] + [TestCase("")] + [TestCase("false")] + public void The_switch_registers_nothing_unless_the_variable_is_true(string value) + { + var services = new ServiceCollection(); + + RunCommand.AllowUnreleasedMigrationWhenAsked(services, value); + + using var provider = services.BuildServiceProvider(); + Assert.That(provider.GetService<AllowUnreleasedMigration>(), Is.Null); + } +} +#endif diff --git a/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs new file mode 100644 index 0000000000..e381874eac --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs @@ -0,0 +1,232 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// No acceptance test can reach the watchdog: the host copies on the real clock and nobody waits half an hour. +[TestFixture] +class ClosedWindowProgressTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_category_that_commits_nothing_for_the_stall_limit_is_stopped() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that committed nothing for longer than the limit was never stopped"); + await progress.DisposeAsync(); + } + + // A call that ignores its token never returns, so the row stays in progress and every later poll sees the same stall. + [Test] + public async Task A_category_already_stopped_is_reported_once_however_long_it_stays_stuck() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(logger.Entries.Count(entry => entry.Level == LogLevel.Error), Is.EqualTo(1), "the same stall was reported again on the next poll"); + } + + [Test] + public async Task A_resumed_row_carrying_the_previous_runs_stamp_is_not_a_stall() + { + // The row a killed run left behind keeps its last stamp, so an operator restarting hours later + // would otherwise have a healthy copy cancelled on the first tick, every time. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the window runs from when this category's run began, not from a stamp the previous run left"); + } + + [Test] + public async Task A_row_other_than_the_running_category_is_not_judged() + { + // Both were attempted, but only KnownEndpoints is running, and it has no row yet. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.EndpointSettings, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, [MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings]); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the running category was stopped over a row nobody is copying"); + } + + [Test] + public async Task Committing_nothing_for_exactly_the_stall_limit_is_not_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the limit is the point at which a copy has not yet stalled"); + } + + [Test] + public async Task A_category_that_has_committed_nothing_since_its_run_began_is_stopped() + { + // The engine stamps a category when it marks it running, so the window covers the first read, which is + // where a copy that never gets going actually hangs. When the category first started does not matter. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that has committed nothing since its run began was never stopped"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_row_an_older_build_left_without_a_stamp_is_still_watched() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: null, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "an unstamped row was left unwatched for ever"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_category_this_run_finished_is_not_stopped_for_having_gone_quiet() + { + // The watch moves to the next category only when that one starts, so a settled row can still be the one watched. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a category that finished was stopped"); + } + + [Test] + public async Task A_halted_category_this_run_attempted_is_not_stopped_for_having_gone_quiet() + { + // A halt settles the row and leaves its stamp behind. The gate is what refuses the host over it, not the watchdog. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a halted category was stopped"); + } + + // The watch has no total limit, so a copy that keeps committing outlives any length of run. + [Test] + public async Task A_category_still_committing_batches_is_never_stopped_however_long_it_takes() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var committed = await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + // Four times the limit, committing a batch every poll, which a total timeout would have killed long ago. + for (var elapsed = TimeSpan.Zero; elapsed < StallLimit * 4; elapsed += PollInterval) + { + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + committed = await store.Upsert(committed with { LastProgressAt = clock.GetUtcNow().UtcDateTime }); + } + + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a copy committing a batch every poll was stopped, so the watch is a deadline rather than a stall detector"); + } + + [Test] + public async Task A_poll_that_fails_leaves_the_watch_running_and_says_so() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var storeFailure = new InvalidOperationException("the checkpoint store is unreachable"); + store.FailReadAll(1, storeFailure); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a watch that stopped at the first blip never noticed the stall that followed"); + await progress.DisposeAsync(); + Assert.That(logger.Entries.Where(entry => entry.Level == LogLevel.Error).Select(entry => entry.Exception), Has.Member(storeFailure), "a watch that has gone deaf has to say so"); + } + + static async Task<MigrationStartup.ClosedWindowProgress> StartWatching(PollObservingCheckpointStore store, TimerRecordingTimeProvider clock, string runningCategoryId, CancellationTokenSource stop, string[]? attemptedCategoryIds = null, ILogger? logger = null) + { + var progress = new MigrationStartup.ClosedWindowProgress(store, clock, logger ?? new CapturingLogger(), attemptedCategoryIds ?? [runningCategoryId]); + progress.Watch(runningCategoryId, clock.GetUtcNow().UtcDateTime, stop); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watch never started its timer, so advancing the clock would tick nothing"); + + return progress; + } + + static MigrationCheckpoint InProgress(string categoryId, DateTime? lastProgressAt, DateTime? startedAt = null) => + new(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, startedAt ?? lastProgressAt, lastProgressAt, null, null); + + static MigrationCheckpoint Settled(string categoryId, MigrationCategoryState state, DateTime lastProgressAt) => + new(categoryId, state, null, 0, 0, null, null, lastProgressAt, lastProgressAt, lastProgressAt, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs new file mode 100644 index 0000000000..c8774a9e9a --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class CopyableCategoryIdsTests +{ + [Test] + public void A_category_only_one_side_supports_counts_as_not_copyable() + { + var source = new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }; + var target = new[] { MigrationCategoryIds.KnownEndpoints }; + + Assert.That(MigrationStartup.CopyableCategoryIds(source, target), Is.EquivalentTo(new[] { MigrationCategoryIds.KnownEndpoints }), + "a category with a reader and no writer would otherwise be attempted and fail partway through a customer's copy"); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs b/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs new file mode 100644 index 0000000000..d8bdb1d464 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CustomCheckStatusTests.cs @@ -0,0 +1,14 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Persistence; + +[TestFixture] +class CustomCheckStatusTests +{ + // CustomCheckDetail carries the status as a bool, so a third member would be copied as Pass. + [Test] + public void The_status_enum_still_has_exactly_the_two_values_a_bool_can_carry() => + Assert.That(Enum.GetValues<Status>(), Is.EqualTo(new[] { Status.Fail, Status.Pass })); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs new file mode 100644 index 0000000000..24aee91e82 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; + +public sealed class CapturingLogger : ILogger<MigrationEngine> +{ + public List<(LogLevel Level, string Message, Exception? Exception)> Entries { get; } = []; + + public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) => + Entries.Add((logLevel, formatter(state, exception), exception)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs new file mode 100644 index 0000000000..78edd726b6 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs @@ -0,0 +1,33 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationCheckpointStore : IMigrationCheckpointStore +{ + readonly ConcurrentDictionary<string, MigrationCheckpoint> checkpoints = new(); + + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => + Task.FromResult<IReadOnlyList<MigrationCheckpoint>>(checkpoints.Values.ToArray()); + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => + Task.FromResult(checkpoints.TryGetValue(categoryId, out var checkpoint) ? checkpoint : null); + + public Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var storedVersion = checkpoints.TryGetValue(checkpoint.CategoryId, out var stored) ? stored.Version : 0; + if (storedVersion != checkpoint.Version) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row is at version {storedVersion}."); + } + + var saved = checkpoint with { Version = checkpoint.Version + 1 }; + checkpoints[checkpoint.CategoryId] = saved; + return Task.FromResult(saved); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs new file mode 100644 index 0000000000..0d3bbfceb6 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -0,0 +1,167 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationSource : IMigrationSource +{ + readonly Dictionary<string, List<MigrationRow>> rowsByCategory = []; + readonly Dictionary<string, (int Times, Exception Failure)> bodyFailures = []; + readonly Dictionary<string, Exception[]> bodyFailuresInTurn = []; + readonly Dictionary<string, int> bodyReadAttempts = []; + readonly Dictionary<string, MigrationBody?> bodies = []; + // The loader reads several bodies at once, so the attempt counts and the in-flight count are shared between threads. + readonly Lock bodyReadGate = new(); + TaskCompletionSource? heldBodyReads; + int bodyReadsInFlight; + + public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); + + public void Seed(string categoryId, params MigrationRow[] rows) => rowsByCategory[categoryId] = [.. rows]; + + public void SetBody(string sourceId, MigrationBody? body) => bodies[sourceId] = body; + + public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); + + /// <summary> + /// Fails the body reads for this row with one exception per attempt, in order, so a test can tell one attempt's error from another's. An attempt past the end of the list reads the body normally. + /// </summary> + public void FailBodyReadsInTurn(string sourceId, params Exception[] failuresInAttemptOrder) => bodyFailuresInTurn[sourceId] = failuresInAttemptOrder; + + /// <summary> + /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + /// </summary> + public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } + + public int BodyReadAttempts(string sourceId) + { + lock (bodyReadGate) + { + return bodyReadAttempts.GetValueOrDefault(sourceId); + } + } + + /// <summary> + /// The most body reads that were open at the same moment since the source was created. + /// </summary> + public int PeakBodyReadsInFlight { get; private set; } + + /// <summary> + /// Makes every body read wait until <see cref="ReleaseBodyReads" /> is called. A waiting read already counts as an attempt and as open. + /// </summary> + public void HoldBodyReads() => heldBodyReads = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// <summary> + /// Lets every held body read carry on, and every later one run without waiting. Call <see cref="HoldBodyReads" /> first. + /// </summary> + public void ReleaseBodyReads() => heldBodyReads!.SetResult(); + + /// <summary> + /// Completes once at least this many body reads are open at the same moment. + /// </summary> + public async Task BodyReadsInFlight(int count, CancellationToken cancellationToken = default) + { + while (Volatile.Read(ref bodyReadsInFlight) < count) + { + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken); + } + } + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public IReadOnlyList<IMigrationStartupCheck> ContributedChecks() => []; + + public Task<MigrationSourceDescription> Describe(CancellationToken cancellationToken = default) => Task.FromResult(Description); + + public Task<IReadOnlyList<MigrationSourceInventoryEntry>> Inventory(CancellationToken cancellationToken = default) => + Task.FromResult<IReadOnlyList<MigrationSourceInventoryEntry>>( + [.. rowsByCategory.Select(pair => new MigrationSourceInventoryEntry("memory", pair.Key, pair.Value.Count))]); + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult<long>(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0); + + public async IAsyncEnumerable<MigrationBatch> Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var rows = rowsByCategory.GetValueOrDefault(category.Id) ?? []; + var startIndex = 0; + + if (resumeAfter is not null) + { + var cursorIndex = rows.FindIndex(r => r.SourceId == resumeAfter); + if (cursorIndex < 0) + { + throw new InvalidOperationException($"Category {category.Id} was asked to resume after {resumeAfter}, a cursor this source never issued"); + } + startIndex = cursorIndex + 1; + } + + for (var i = startIndex; i < rows.Count; i += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var slice = rows.Skip(i).Take(batchSize).ToArray(); + yield return new MigrationBatch(slice, slice[^1].SourceId); + await Task.Yield(); + } + } + + public async Task<MigrationBody?> ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + await Task.Yield(); + + int attempt; + + lock (bodyReadGate) + { + attempt = bodyReadAttempts[sourceId] = bodyReadAttempts.GetValueOrDefault(sourceId) + 1; + PeakBodyReadsInFlight = Math.Max(PeakBodyReadsInFlight, Interlocked.Increment(ref bodyReadsInFlight)); + } + + try + { + if (heldBodyReads is { } held) + { + await held.Task; + } + + if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) + { + throw failures.Failure; + } + + if (bodyFailuresInTurn.TryGetValue(sourceId, out var failuresInTurn) && attempt <= failuresInTurn.Length) + { + throw failuresInTurn[attempt - 1]; + } + + return bodies.GetValueOrDefault(sourceId); + } + finally + { + Interlocked.Decrement(ref bodyReadsInFlight); + } + } + + // Every registered category, not just the seeded ones: Seed writes rowsByCategory after construction, + // and the interface promises an answer that does not change across Open. + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs new file mode 100644 index 0000000000..23492909d4 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs @@ -0,0 +1,82 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationSourceTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task Streams_seeded_rows_in_batches_of_the_requested_size() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c"), Row("d"), Row("e")); + + var batches = new List<MigrationBatch>(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: null, batchSize: 2)) + { + batches.Add(batch); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches, Has.Count.EqualTo(3)); + Assert.That(batches[0].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b" })); + Assert.That(batches[2].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "e" })); + } + } + + [Test] + public async Task ResumeAfter_skips_everything_up_to_and_including_that_id() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c")); + + var batches = new List<MigrationBatch>(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "a", batchSize: 10)) + { + batches.Add(batch); + } + + Assert.That(batches.Single().Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "b", "c" })); + } + + [Test] + public void A_cursor_the_source_never_issued_throws_rather_than_starting_again() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b")); + + Assert.ThrowsAsync<InvalidOperationException>(async () => + { + await foreach (var _ in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "no-such-row", batchSize: 10)) + { + } + }); + } + + [Test] + public async Task ReadBody_fails_the_configured_number_of_times_then_returns_the_seeded_body() + { + var source = new InMemoryMigrationSource(); + var finalBody = new MigrationBody(new byte[] { 9 }, "text/plain"); + source.SetBody("msg-1", finalBody); + source.FailBodyReads("msg-1", times: 1, new TimeoutException("transient")); + + Assert.ThrowsAsync<TimeoutException>(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); + var secondAttempt = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(secondAttempt, Is.EqualTo(finalBody)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(2)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs new file mode 100644 index 0000000000..927b8be477 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -0,0 +1,189 @@ +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget +{ + readonly Dictionary<string, HashSet<string>> writtenKeysByCategory = []; + readonly Dictionary<string, List<MigrationRow>> writtenRowsByCategory = []; + readonly Dictionary<string, List<MigrationRow>> rowsHandedToWriteByCategory = []; + readonly HashSet<string> preExistingKeys = []; + readonly Dictionary<string, MigrationSkipReason> rejectedKeys = []; + readonly TaskCompletionSource ignoredCallReleased = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int DefaultBatchSize { get; set; } = 3; + public string NoBatchSizeFor { get; set; } + public int? FailOnCallNumber { get; set; } + + /// <summary> + /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + /// </summary> + public Exception FailWith { get; set; } + + /// <summary> + /// How many rows each write leaves out of the outcome it reports and commits, without failing, so the two still agree with each other. + /// </summary> + public int UnderReportBy { get; set; } + + /// <summary> + /// Runs at the start of every write, before any simulated failure, with the checkpoint the engine is extending, so a test can watch a stamp move between batches. + /// </summary> + public Action<MigrationCheckpoint> BeforeWrite { get; set; } + + /// <summary> + /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + /// </summary> + public (int CallNumber, CancellationTokenSource Source)? CancelOnCall { get; set; } + /// <summary> + /// Cancels the token on this call and throws instead of writing, so the stop surfaces from the write itself. + /// </summary> + public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } + /// <summary> + /// Waits on this call until the token is cancelled and then throws, as a write stuck on a database that stopped answering does. + /// </summary> + public int? HangOnCall { get; set; } + /// <summary> + /// Waits on this call until ReleaseIgnoredCall and never looks at the token, then writes without it, as a driver call stuck below the token does. + /// If the store then refuses the checkpoint save, this fake has already recorded the batch's rows, unlike the real target, which rolls both back. + /// </summary> + public int? IgnoreTokenOnCall { get; set; } + /// <summary> + /// The write that IgnoreTokenOnCall holds, or null until that call is made. It ends only after ReleaseIgnoredCall. + /// </summary> + public Task<MigrationWriteResult> IgnoredCall { get; private set; } + int callCount; + + /// <summary> + /// Lets the write that IgnoreTokenOnCall holds carry on. Calling it again does nothing. + /// </summary> + public void ReleaseIgnoredCall() => ignoredCallReleased.TrySetResult(); + + public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); + + public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; + + public IReadOnlyList<MigrationRow> WrittenRows(string categoryId) => + writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + /// <summary> + /// Every row the engine sent to a write that ran, in the order it sent them, including the rows this fake then skipped or found already present. It is the only way to see the same row sent twice, because WrittenRows de-duplicates as the real targets do. Empty for a category never written to. + /// </summary> + public IReadOnlyList<MigrationRow> RowsHandedToWrite(string categoryId) => + rowsHandedToWriteByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : Task.FromResult(DefaultBatchSize); + + public Task<MigrationWriteResult> Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) + { + var callNumber = ++callCount; + BeforeWrite?.Invoke(checkpointToExtend); + + return IgnoreTokenOnCall == callNumber + ? IgnoredCall = WriteOnceReleased(category, batch, checkpointToExtend, callNumber) + : WriteNow(category, batch, checkpointToExtend, callNumber, cancellationToken); + } + + // The commit ignores the token too, so a late commit meets the store's version check instead of the token. + async Task<MigrationWriteResult> WriteOnceReleased(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, int callNumber) + { + await ignoredCallReleased.Task; + return await WriteNow(category, batch, checkpointToExtend, callNumber, CancellationToken.None); + } + + async Task<MigrationWriteResult> WriteNow( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + int callNumber, + CancellationToken cancellationToken) + { + if (HangOnCall == callNumber) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + if (StopOnCall is { } stop && stop.CallNumber == callNumber) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (FailOnCallNumber == callNumber) + { + throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callNumber}"); + } + + if (CancelOnCall is { } cancel && cancel.CallNumber == callNumber) + { + await cancel.Source.CancelAsync(); + } + + var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; + var rows = writtenRowsByCategory.TryGetValue(category.Id, out var existingRows) ? existingRows : writtenRowsByCategory[category.Id] = []; + + // Recorded here rather than at the top of the method: a write that threw above never committed, so + // the restart that sends its batch again is doing the right thing. + var handedOver = rowsHandedToWriteByCategory.TryGetValue(category.Id, out var existingHandedOver) ? existingHandedOver : rowsHandedToWriteByCategory[category.Id] = []; + handedOver.AddRange(batch.Rows); + + var copied = 0; + var alreadyPresent = 0; + var skippedIds = new List<string>(); + var skipReasons = new Dictionary<MigrationSkipReason, long>(); + + foreach (var row in batch.Rows) + { + if (rejectedKeys.TryGetValue(row.SourceId, out var reason)) + { + skippedIds.Add(row.SourceId); + skipReasons[reason] = skipReasons.GetValueOrDefault(reason) + 1; + continue; + } + + if (preExistingKeys.Contains(row.SourceId) || !keys.Add(row.SourceId)) + { + alreadyPresent++; + continue; + } + + rows.Add(row); + copied++; + } + + copied -= UnderReportBy; + + // The real targets extend and save the checkpoint in the transaction that writes the rows, + // so this fake saves it here too. + var saved = await checkpointStore.Upsert( + checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), + cancellationToken); + + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + + /// <summary> + /// Saves the reset through the checkpoint store, so a stale Version throws as on the real target. + /// It deletes no rows, because no unit test needs that. + /// </summary> + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => + checkpointStore.Upsert(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs new file mode 100644 index 0000000000..ae52e815dc --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs @@ -0,0 +1,81 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationTargetTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + static MigrationCheckpoint EmptyCheckpoint(string categoryId) => + new(categoryId, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null); + + [Test] + public async Task Writes_new_rows_and_commits_the_extended_checkpoint_with_them() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("EndpointSettings")!; + var batch = new MigrationBatch([Row("a"), Row("b")], Cursor: "b"); + // Prior totals and the new cursor. The target adds this batch's own outcome before it saves. + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "b" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(2)); + Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 })); + Assert.That(result.Saved, Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 }), "the result carries the row as stored"); + } + } + + [Test] + public async Task The_committed_checkpoint_carries_the_real_split_rather_than_the_rows_handed_over() + { + // Three rows in, one copied: a checkpoint saying three would survive a crash as three. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + target.SeedExistingKey("already-present"); + target.RejectKey("rejected", MigrationSkipReason.BodyUnreadable); + var batch = new MigrationBatch([Row("already-present"), Row("rejected"), Row("new-row")], Cursor: "new-row"); + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "new-row" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "rejected" })); + Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(1)); + var stored = (await checkpointStore.Read(category.Id))!; + Assert.That((stored.CopiedCount, stored.SkippedCount, stored.AlreadyPresentCount), Is.EqualTo((1L, 1L, 1L)), "copied, skipped, already present as committed"); + } + } + + [Test] + public async Task Count_answers_for_one_category_rather_than_one_table() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var required = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + await target.Write(required, new MigrationBatch([Row("a"), Row("b")], "b"), EmptyCheckpoint(required.Id)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await target.Count(required), Is.EqualTo(2)); + Assert.That(await target.Count(archive), Is.Zero); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs new file mode 100644 index 0000000000..147fac3a8f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs @@ -0,0 +1,64 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +// Records when each poll read the rows, so a test can wait for a poll that saw the state it is judging. +// Only the stall watchdog calls ReadAll, so every ReadAll is one of its polls. +public sealed class PollObservingCheckpointStore(TimeProvider clock) : IMigrationCheckpointStore +{ + readonly InMemoryMigrationCheckpointStore inner = new(); + readonly SemaphoreSlim polled = new(0); + readonly ConcurrentQueue<DateTime> polledAt = new(); + Exception? readAllFailure; + int readAllFailuresLeft; + + public void FailReadAll(int times, Exception failure) + { + readAllFailuresLeft = times; + readAllFailure = failure; + } + + /// <summary> + /// Waits until a poll has read the rows at the clock's current time, and fails the test if none does within ten + /// seconds. Polls at earlier times are passed over. + /// </summary> + public async Task WaitForAPollAtTheCurrentTime() + { + var now = clock.GetUtcNow().UtcDateTime; + + while (true) + { + Assert.That(await polled.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, $"no poll read the checkpoints at {now:O}; a watch that stopped polling never reaches one"); + + if (polledAt.TryDequeue(out var at) && at == now) + { + return; + } + } + } + + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) + { + polledAt.Enqueue(clock.GetUtcNow().UtcDateTime); + polled.Release(); + + if (readAllFailuresLeft > 0) + { + readAllFailuresLeft--; + return Task.FromException<IReadOnlyList<MigrationCheckpoint>>(readAllFailure!); + } + + return inner.ReadAll(cancellationToken); + } + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => inner.Upsert(checkpoint, cancellationToken); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs new file mode 100644 index 0000000000..6178cbde97 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Time.Testing; + +// Signals each timer the engine creates, so a test only advances the clock once a pause is waiting on it. +public sealed class TimerRecordingTimeProvider : TimeProvider +{ + readonly FakeTimeProvider clock = new(); + + public SemaphoreSlim TimerCreated { get; } = new(0); + + public List<TimeSpan> DueTimes { get; } = []; + + public void Advance(TimeSpan delta) => clock.Advance(delta); + + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + + public override long GetTimestamp() => clock.GetTimestamp(); + + public override long TimestampFrequency => clock.TimestampFrequency; + + public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = clock.CreateTimer(callback, state, dueTime, period); + DueTimes.Add(dueTime); + TimerCreated.Release(); + return timer; + } +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs new file mode 100644 index 0000000000..ae309d6c8b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class HaltThresholdTests +{ + [Test] + public void Does_not_halt_a_three_row_category_with_one_bad_row() + { + // 1 of 3 is 33%, well past the 5% proportion, but 1 skip never passes the 100-row floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 1, totalCount: 3, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Halts_a_large_category_with_a_systemic_failure() + { + // 600 of 10,000 is 6%, past both the proportion and the floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 600, totalCount: 10_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void Does_not_halt_a_large_category_with_only_a_small_proportion_skipped() + { + // 10,000 of 5,000,000 is 0.2%: past the floor, nowhere near the proportion. + var exceeded = HaltThreshold.Exceeded(skippedCount: 10_000, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_at_both_boundaries_does_not_halt_because_both_must_be_exceeded() + { + // Exactly the floor, which settles it before the proportion is ever worked out. "Exceed" means strictly past, not "at". + var exceeded = HaltThreshold.Exceeded(skippedCount: 100, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_on_the_proportion_does_not_halt_once_the_floor_is_behind_it() + { + // 101 of 2,020 is exactly 5% with the floor already passed, so this is the only shape that + // reaches the proportion comparison and depends on it being strictly greater. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_020, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_hair_past_the_proportion_halts_once_the_floor_is_behind_it() + { + // 101 of 2,000 is 5.05%: the same skip count as above, one row's worth over the line. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void A_large_category_losing_a_small_share_does_not_halt_once_past_the_floor() + { + // 101 of 5,000,000 is 0.002%. The floor is already behind it, so only the proportion can stop a halt + // here, and a rule that halted on the floor alone would stop a healthy copy of a huge category. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_skip_count_with_nothing_processed_never_halts_and_never_divides_by_zero() + { + // Past the floor with a zero total, which is the only input that reaches the division guard. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 0, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void No_rows_processed_yet_never_halts() + { + Assert.That(HaltThreshold.Exceeded(skippedCount: 0, totalCount: 0, percentThreshold: 5, minimumFloor: 100), Is.False); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs new file mode 100644 index 0000000000..9e86ba2723 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationBeforeTheHostOpensTests.cs @@ -0,0 +1,72 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationBeforeTheHostOpensTests +{ + [Test] + public async Task A_persister_without_checkpoints_starts_without_a_migration_source() + { + using var services = new ServiceCollection().BuildServiceProvider(); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + [Test] + public async Task An_empty_checkpoint_table_starts_without_a_migration_source() + { + using var services = Services(new InMemoryMigrationCheckpointStore()); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Blocked)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task An_unfinished_required_copy_refuses_startup_without_resuming_it(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + var before = await store.Upsert(Checkpoint(MigrationCategoryIds.KnownEndpoints, state)); + using var services = Services(store); + + var exception = Assert.ThrowsAsync<Exception>(() => new MigrationBeforeTheHostOpens(services).StartingAsync()); + Assert.That(exception.Message, Does.Contain("ServiceControl will not start")); + Assert.That(await store.Read(before.CategoryId), Is.EqualTo(before), "startup must never run the offline copy"); + } + + [Test] + public async Task An_unrecognized_unfinished_category_keeps_the_host_closed() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("NewRequiredCategory", MigrationCategoryState.NotStarted)); + using var services = Services(store); + var exception = Assert.ThrowsAsync<Exception>(() => new MigrationBeforeTheHostOpens(services).StartingAsync()); + Assert.That(exception.Message, Does.Contain("--start-data-migration")); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task Optional_work_does_not_hold_the_host_closed(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint(MigrationCategoryIds.EventLog, state)); + using var services = Services(store); + await new MigrationBeforeTheHostOpens(services).StartingAsync(); + } + + static ServiceProvider Services(IMigrationCheckpointStore store) => + new ServiceCollection().AddSingleton(store).BuildServiceProvider(); + + static MigrationCheckpoint Checkpoint(string id, MigrationCategoryState state) => + new(id, state, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs new file mode 100644 index 0000000000..201bb885c8 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationBodyLoaderTests.cs @@ -0,0 +1,212 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationBodyLoaderTests +{ + static readonly MigrationCategory Category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + static readonly MigrationBody Body = new(Encoding.UTF8.GetBytes("<order>1</order>"), "text/xml"); + + [Test] + public async Task Attaches_the_body_to_every_row() + { + var source = SourceWithBodies("FailedMessages/one", "FailedMessages/two"); + + var result = await Load(source, "FailedMessages/one", "FailedMessages/two"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { "FailedMessages/one", "FailedMessages/two" })); + Assert.That(result.Rows.Select(row => row.Body), Is.All.SameAs(Body)); + Assert.That(result.Skipped, Is.Empty); + } + } + + [Test] + public async Task Retries_a_failing_read_three_times_then_skips_the_message_as_BodyUnreadable() + { + var source = SourceWithBodies("FailedMessages/one", "FailedMessages/two"); + source.FailBodyReads("FailedMessages/two", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + + var result = await Load(source, "FailedMessages/one", "FailedMessages/two"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/two"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(new[] { "FailedMessages/one" }), "a message with no body is a retry hazard, so it is left out rather than written without one"); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/two", MigrationSkipReason.BodyUnreadable) })); + } + } + + [Test] + public async Task Keeps_a_message_whose_read_succeeds_on_the_last_attempt() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Single().Body, Is.SameAs(Body)); + Assert.That(result.Skipped, Is.Empty); + } + } + + [Test] + public async Task Skips_a_message_the_source_has_no_body_for_as_BodyMissing_after_one_read() + { + var source = new InMemoryMigrationSource(); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(1), "a read that found nothing learns nothing by repeating"); + Assert.That(result.Rows, Is.Empty); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/one", MigrationSkipReason.BodyMissing) }), + "a missing body is not an unreadable one, and only a separate reason lets the refusal say no retry fixes it"); + } + } + + [Test] + public void Lets_a_defect_out_on_the_first_attempt_rather_than_skipping_the_message() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts, new NotSupportedException("this source cannot read bodies")); + + Assert.ThrowsAsync<NotSupportedException>(() => Load(source, "FailedMessages/one"), + "a defect fails the same way on every attempt, so retrying it and skipping the message would blame the body store for a bug"); + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(1)); + } + + [Test] + public void Logs_a_defect_with_the_message_it_was_reading_before_letting_it_out() + { + var source = SourceWithBodies("FailedMessages/one"); + var defect = new NotSupportedException("this source cannot read bodies"); + source.FailBodyReads("FailedMessages/one", 1, defect); + var logger = new CapturingLogger(); + var batch = new MigrationBatch([new MigrationRow("FailedMessages/one", new object(), new Dictionary<string, object?>())], "FailedMessages/one"); + + Assert.ThrowsAsync<NotSupportedException>(() => new MigrationBodyLoader(source, new FakeTimeProvider(), TimeSpan.Zero, logger).Load(Category, batch, TestContext.CurrentContext.CancellationToken)); + + var error = logger.Entries.Single(entry => entry.Level == LogLevel.Error); + + using (Assert.EnterMultipleScope()) + { + Assert.That(error.Message, Does.Contain("FailedMessages/one").And.Contain(Category.Id), "the exception that halts the category does not say which message it was reading, so this line is the only record of it"); + Assert.That(error.Exception, Is.SameAs(defect)); + } + } + + [Test] + public async Task Retries_a_cancellation_the_run_did_not_ask_for_and_then_skips_the_message_as_BodyUnreadable() + { + var source = SourceWithBodies("FailedMessages/one"); + source.FailBodyReads("FailedMessages/one", MigrationEngine.MaxBodyReadAttempts, new OperationCanceledException("the client timed out")); + + var result = await Load(source, "FailedMessages/one"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts), "a client timeout raised as a cancellation is a transient failure, not a shutdown"); + Assert.That(result.Skipped, Is.EqualTo(new[] { ("FailedMessages/one", MigrationSkipReason.BodyUnreadable) })); + } + } + + [Test] + public async Task Keeps_a_body_the_source_attached_and_never_reads_it_again() + { + var source = SourceWithBodies("FailedMessages/two"); + var attached = new MigrationBody(Encoding.UTF8.GetBytes("attached in Read"), "text/plain"); + + var batch = new MigrationBatch( + [ + new MigrationRow("FailedMessages/one", new object(), new Dictionary<string, object?>(), attached), + new MigrationRow("FailedMessages/two", new object(), new Dictionary<string, object?>()) + ], + "FailedMessages/two"); + + var result = await Loader(source).Load(Category, batch, TestContext.CurrentContext.CancellationToken); + + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("FailedMessages/one"), Is.Zero, "a source that attaches bodies in Read would otherwise pay a second read per message for data it already had"); + Assert.That(result.Rows[0].Body, Is.SameAs(attached)); + Assert.That(result.Rows[1].Body, Is.SameAs(Body)); + } + } + + [Test] + public async Task Fetches_at_most_eight_bodies_at_a_time_and_keeps_the_batch_order() + { + var sourceIds = Enumerable.Range(0, 40).Select(index => $"FailedMessages/{index}").ToArray(); + var source = SourceWithBodies(sourceIds); + source.HoldBodyReads(); + + var load = Load(source, sourceIds); + + try + { + using var waitLimit = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + await source.BodyReadsInFlight(MigrationBodyLoader.MaxConcurrentFetches, waitLimit.Token); + // Gives an unbounded loader time to open the rest of the forty reads, which it would do all at once. + await Task.Delay(TimeSpan.FromMilliseconds(100)); + + Assert.That(source.PeakBodyReadsInFlight, Is.EqualTo(MigrationBodyLoader.MaxConcurrentFetches), + "an unbounded loader opens all forty reads at once, which turns the copier into a load spike on a live source"); + } + finally + { + source.ReleaseBodyReads(); + } + + var result = await load.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Rows.Select(row => row.SourceId), Is.EqualTo(sourceIds), "the batch's order is the cursor's order, so it has to survive being fetched out of order"); + Assert.That(result.Skipped, Is.Empty); + Assert.That(source.PeakBodyReadsInFlight, Is.EqualTo(MigrationBodyLoader.MaxConcurrentFetches)); + } + } + + static InMemoryMigrationSource SourceWithBodies(params string[] sourceIds) + { + var source = new InMemoryMigrationSource(); + + foreach (var sourceId in sourceIds) + { + source.SetBody(sourceId, Body); + } + + return source; + } + + // Zero backoff because the FakeTimeProvider is never advanced, and a wait on it would never finish. + static MigrationBodyLoader Loader(InMemoryMigrationSource source) => + new(source, new FakeTimeProvider(), TimeSpan.Zero, NullLogger.Instance); + + static Task<MigrationBodyLoadResult> Load(InMemoryMigrationSource source, params string[] sourceIds) + { + var batch = new MigrationBatch( + [.. sourceIds.Select(sourceId => new MigrationRow(sourceId, new object(), new Dictionary<string, object?>()))], + sourceIds[^1]); + + return Loader(source).Load(Category, batch, TestContext.CurrentContext.CancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs new file mode 100644 index 0000000000..214b13a2d2 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -0,0 +1,110 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryRegistryTests +{ + [Test] + public void Contains_all_eighteen_categories_with_unique_ids() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); + } + } + + [Test] + public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var requiredIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Required) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(requiredIds, Is.EqualTo(new[] + { + "KnownEndpoints", "EndpointSettings", "MessageRedirects", "Subscriptions", + "NotificationSettings", "TrialEndDate", "RetryOperations", + "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", + "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages", + "PendingIntegrationEvents", "CustomChecks", "FailedErrorImports", "GroupComments" + })); + } + + [Test] + public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var optionalIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Optional) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(optionalIds, Is.EqualTo(new[] + { + "EventLog", "ArchivedAndResolvedFailedMessages" + })); + } + + [Test] + public void Three_categories_declare_a_MustFollow() + { + var declared = MigrationCategoryRegistry.All + .Where(c => c.MustFollow is not null) + .ToDictionary(c => c.Id, c => c.MustFollow); + + Assert.That(declared, Is.EqualTo(new Dictionary<string, string?> + { + // A cascading foreign key: throughput rows cannot exist before their endpoint row. + ["LicensingThroughput"] = "LicensingEndpoints", + // Keeps settings back while known endpoints are unfinished. The heartbeat sync is kept off + // the copied settings because both categories finish before the host opens. + ["EndpointSettings"] = "KnownEndpoints", + // A group comment whose failed messages have not arrived reads as an orphan to the + // retention sweeper, which deletes it. + ["GroupComments"] = "UnresolvedAndRetryIssuedFailedMessages" + })); + } + + [Test] + public void The_two_failed_message_categories_and_failed_imports_carry_bodies() + { + var carriesBodies = MigrationCategoryRegistry.All.Where(c => c.CarriesBodies).Select(c => c.Id).ToArray(); + + Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); + } + + [Test] + public void The_two_failed_message_categories_split_every_status_between_them() + { + var split = MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(split, Is.Unique); + Assert.That(split, Is.EquivalentTo(Enum.GetValues<FailedMessageStatus>()), + "a status neither category claims is a failed message no migration copies"); + } + } + + [Test] + public void Find_returns_null_for_an_unknown_id() + { + Assert.That(MigrationCategoryRegistry.Find("NoSuchCategory"), Is.Null); + } + + [Test] + public void Find_returns_the_category_for_a_known_id() + { + Assert.That(MigrationCategoryRegistry.Find("EventLog")?.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs new file mode 100644 index 0000000000..23c4080bfb --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryStateExtensionsTests.cs @@ -0,0 +1,47 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryStateExtensionsTests +{ + [Test] + public void Every_skip_reason_is_exactly_one_of_harmless_permanent_or_retryable() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(Kinds.Keys, Is.EquivalentTo(Enum.GetValues<MigrationSkipReason>()), "a reason on neither list is treated as a fault a retry can fix, so each new reason has to be placed here on purpose"); + + foreach (var (reason, kind) in Kinds) + { + Assert.That((reason.IsBenign(), reason.IsPermanent()), Is.EqualTo((kind == SkipKind.Harmless, kind == SkipKind.Permanent)), $"{reason} is {kind}"); + } + } + } + + enum SkipKind + { + Harmless, + Permanent, + Retryable + } + + static readonly Dictionary<MigrationSkipReason, SkipKind> Kinds = new() + { + [MigrationSkipReason.BodyUnreadable] = SkipKind.Retryable, + [MigrationSkipReason.PastRetention] = SkipKind.Harmless, + [MigrationSkipReason.RequiredValueMissing] = SkipKind.Permanent, + [MigrationSkipReason.BlankGroupComment] = SkipKind.Harmless, + [MigrationSkipReason.SourceStorageCheck] = SkipKind.Harmless, + [MigrationSkipReason.UnparseableUniqueMessageId] = SkipKind.Permanent, + [MigrationSkipReason.NoProcessingAttempts] = SkipKind.Permanent, + [MigrationSkipReason.ValueTooLong] = SkipKind.Permanent, + [MigrationSkipReason.MalformedValue] = SkipKind.Permanent, + [MigrationSkipReason.BodyMissing] = SkipKind.Permanent, + [MigrationSkipReason.Unknown] = SkipKind.Retryable + }; +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs new file mode 100644 index 0000000000..8b981b0665 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -0,0 +1,83 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationContractShapeTests +{ + [Test] + public void MigrationCategory_carries_no_fact_about_where_a_store_keeps_its_rows() + { + var properties = typeof(MigrationCategory).GetProperties().Select(p => p.Name); + + Assert.That(properties, Is.EquivalentTo(new[] { "Id", "Kind", "CarriesBodies", "Order", "MustFollow" }), + "a source database or target table belongs in that store's own adapter, where a different store pair can map it differently"); + } + + [Test] + public void Category_states_keep_the_integers_stored_checkpoints_already_hold() + { + var stored = System.Enum.GetValues<MigrationCategoryState>().ToDictionary(state => state.ToString(), state => (int)state); + + Assert.That(stored, Is.EqualTo(new Dictionary<string, int> + { + ["NotStarted"] = 0, + ["InProgress"] = 1, + ["Complete"] = 2, + ["CompleteWithErrors"] = 3, + ["Halted"] = 4, + ["Abandoned"] = 5, + ["Blocked"] = 6 + }), "checkpoints store State as an integer, so a reordered or inserted member silently changes what every saved checkpoint means"); + } + + [Test] + public void MigrationRow_body_defaults_to_null() + { + var row = new MigrationRow("id-1", new object(), new Dictionary<string, object?>()); + + Assert.That(row.Body, Is.Null); + } + + [Test] + public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); + } + + [Test] + public void MigrationCheckpoint_StartedWindowSeconds_defaults_to_null() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.StartedWindowSeconds, Is.Null); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs new file mode 100644 index 0000000000..6c67f2fb81 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.UnitTests.Migration; + +using System.Linq; +using System.Reflection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationOptionsSettingsTests +{ + + [Test] + public void Only_MigrationEngineOptions_parses_the_engine_settings_keys() + { + var parsers = typeof(Settings).Assembly.GetTypes() + .Concat(typeof(MigrationEngineOptions).Assembly.GetTypes()) + .Where(type => type.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static) + .Any(method => method.Name is "FromSettings" or "Read" or "Resolve" + && method.GetParameters() is [{ ParameterType.Name: nameof(SettingsRootNamespace) }, ..])) + .Select(type => type.Name) + .ToArray(); + + Assert.That(parsers, Is.EquivalentTo(new[] { nameof(MigrationEngineOptions) }), + "Migration/ThrottlePauseMilliseconds and its neighbours have exactly one parser. A second one drifts its defaults and its refusal message away from this one, and nothing fails until a customer types a window wrong."); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs new file mode 100644 index 0000000000..103d4a2043 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -0,0 +1,253 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineBodyRetryTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_written_with_its_body_attached() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + // Zeroed because FakeTimeProvider is never advanced here: a 200 ms Task.Delay against a clock + // nobody moves never completes, and the test hangs until the runner kills it. + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + } + } + + // Every type the engine treats as a defect. Each fails the same way on every attempt, so retrying one + // would only turn a code fault into skipped messages. + static readonly Exception[] Defects = + [ + new NotSupportedException("this source cannot read bodies"), + new NotImplementedException("not written yet"), + new InvalidOperationException("the session is closed"), + new ArgumentException("the id is not a document id"), + new NullReferenceException("no attachment"), + new InvalidCastException("not an attachment") + ]; + + [TestCaseSource(nameof(Defects))] + public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message(Exception defect) + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, defect); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain(defect.GetType().Name)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons, Is.Null); + } + } + + [Test] + public async Task A_shutdown_during_a_body_read_stops_the_run_instead_of_skipping_the_message() + { + // Retrying a shutdown to the attempt limit and then recording the message as permanently + // unreadable is the one path here that silently loses a customer's failed message. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + using var stopping = new CancellationTokenSource(); + source.StopOnBodyRead = ("msg-1", stopping); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + Assert.ThrowsAsync<OperationCanceledException>(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1), "a shutdown is not a transient body failure, so it is not retried"); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(persisted.SkippedCount, Is.Zero, "the message is still there to copy on the next run"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task The_configured_backoff_is_waited_between_body_read_attempts() + { + // Without the wait, three attempts against a body store that is briefly down all fail inside a + // millisecond and the message is skipped for an outage it would have survived. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // Every attempt on msg-1 fails, so the guard against a wait after the final one is the only thing + // keeping the third timer from being created. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var clock = new TimerRecordingTimeProvider(); + var backoff = TimeSpan.FromMilliseconds(200); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = backoff }; + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger<MigrationEngine>.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // A wait after the first two failures only. A third would never complete, because the clock is + // not advanced again and the run below would time out waiting for it. + for (var waitNumber = 1; waitNumber <= MigrationEngine.MaxBodyReadAttempts - 1; waitNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"backoff {waitNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"backoff {waitNumber} should still be pending"); + clock.Advance(backoff); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1), "msg-1 was given up on, so all three attempts ran"); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { backoff, backoff }), "no wait after the final attempt, which has nothing left to retry"); + } + } + + [Test] + public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() + { + // The first failure of a retry run is usually a transient connect error. The last one says what the + // body store was doing when the message was given up on, and it is all the operator gets. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var failures = Enumerable.Range(1, MigrationEngine.MaxBodyReadAttempts) + .Select(attempt => new TimeoutException($"body store unreachable on attempt {attempt}")) + .ToArray(); + source.FailBodyReadsInTurn("msg-1", failures); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + await engine.RunCategoryAsync(category); + + var skipped = logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(skipped.Exception, Is.SameAs(failures[^1])); + Assert.That(skipped.Message, Does.Contain(nameof(MigrationSkipReason.BodyUnreadable)), "the refusal counts skips by reason name, so the log line has to carry the same name for the operator to find them"); + } + } + + [Test] + public async Task A_body_the_source_already_attached_is_written_without_reading_it_again() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var attached = new MigrationBody(new byte[] { 7 }, "text/plain"); + source.Seed(category.Id, Row("msg-1") with { Body = attached }); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("a body read nobody needed")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(attached)); + } + } + + [Test] + public async Task Exhausted_attempts_skip_the_whole_message_and_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // msg-1 is excluded entirely: not written with a missing body, not written at all. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + } + } + + [Test] + public async Task A_body_store_outage_across_many_messages_halts_the_category() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var ids = Enumerable.Range(1, 1_000).Select(i => $"msg-{i}").ToArray(); + source.Seed(category.Id, [.. ids.Select(Row)]); + // Every body read fails every attempt: a down body store, not a per-message fluke. + foreach (var id in ids) + { + source.FailBodyReads(id, MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("exceeds the configured threshold"), + "the end-of-run accounting check halts a category whose body skips were never counted, so the state alone cannot tell the threshold fired"); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 200 }), + "the first batch's 100 skips only reach the 100-row floor, so the second batch's are what pass it"); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs new file mode 100644 index 0000000000..ec774e7a09 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Particular.Approvals; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCategorySelectionTests +{ + static MigrationEngine BuildEngine(IReadOnlyCollection<string> selectedOptionalIds) + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, selectedOptionalIds); + return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + } + + static string Describe(MigrationCategory category) => + $"{category.Kind} {category.Order}: {category.Id}" + + (category.CarriesBodies ? ", with bodies" : string.Empty) + + (category.MustFollow is null ? string.Empty : $", after {category.MustFollow}"); + + [Test] + public void Every_category_runs_in_a_fixed_order() + { + var everyOptionalId = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional) + .Select(category => category.Id) + .ToArray(); + var engine = BuildEngine(everyOptionalId); + + var runOrder = engine.SelectCategories(MigrationCategoryKind.Required) + .Concat(engine.SelectCategories(MigrationCategoryKind.Optional)) + .Select(Describe); + + Approver.Verify(string.Join(Environment.NewLine, runOrder)); + } + + [Test] + public void Only_configured_optional_categories_are_selected() + { + string[] configured = [MigrationCategoryIds.EventLog]; + var engine = BuildEngine(configured); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + var left = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional && !selected.Contains(category)) + .Select(category => category.Id); + + Approver.Verify(string.Join(Environment.NewLine, + [ + $"Configured: {string.Join(", ", configured)}", + "Runs as:", + .. selected.Select(Describe), + $"Never copied: {string.Join(", ", left)}" + ])); + } + + [Test] + public async Task A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() + { + // Dropping a category from the configuration must not restart, reset or delete what it already + // copied: those rows are in the target, and a row wound back to the start copies every one again. + var stillConfigured = MigrationCategoryIds.EventLog; + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var source = new InMemoryMigrationSource(); + source.Seed(stillConfigured, new MigrationRow("check-1", new object(), new Dictionary<string, object>())); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, [stillConfigured]); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var previousRun = new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.InProgress, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + await checkpointStore.Upsert(previousRun); + + var results = await engine.RunCategories(engine.SelectCategories(MigrationCategoryKind.Optional)); + + var deselected = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { stillConfigured }), "the run touched only the configured category"); + Assert.That(deselected, Is.EqualTo(previousRun with { Version = 1 }), "the row is still at the version the seeding save left it"); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs new file mode 100644 index 0000000000..01dd62c96f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs @@ -0,0 +1,46 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCollisionTests +{ + // A body already attached, so the engine reads none and every row reaches the target. + static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary<string, object?>(), new MigrationBody(new byte[] { 1 }, "application/octet-stream")); + + [Test] + public async Task Rows_already_present_in_the_target_are_counted_as_already_present_across_batches_not_skipped_or_overwritten() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1", "from-source"), Row("msg-2", "from-source"), Row("msg-3", "from-source"), Row("msg-4", "from-source")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // msg-1 and msg-4 already exist in the target: they failed again after cutover, and real ingestion wrote them. + target.SeedExistingKey("msg-1"); + target.SeedExistingKey("msg-4"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + // Asserted through the fake's recorded rows, never by the engine inspecting a document. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2", "msg-3" })); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs new file mode 100644 index 0000000000..484a171e1a --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -0,0 +1,291 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCopyTests +{ + static MigrationRow Row(string id) => new(id, new { Name = id }, new Dictionary<string, object?>()); + + [Test] + public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skipped() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.Cursor, Is.EqualTo("c")); + Assert.That(checkpoint.StartedAt, Is.Not.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(3)); + } + } + + [Test] + public async Task A_first_run_leaves_the_source_total_null_and_every_batch_moves_LastProgressAt() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var startedAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var betweenBatches = TimeSpan.FromMinutes(1); + var clock = new FakeTimeProvider(startedAt); + var stamps = new List<DateTime?>(); + // The engine stamps the checkpoint before the target sees it, so moving the clock in here is what + // makes a stamp that never moves show up as two identical entries. + target.BeforeWrite = extending => + { + stamps.Add(extending.LastProgressAt); + clock.Advance(betweenBatches); + }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SourceTotal, Is.Null, "counting the source is a full pass over it, which on a big category could trip the stall watchdog before a row is copied"); + Assert.That(stamps, Is.EqualTo(new DateTime?[] { startedAt.UtcDateTime, (startedAt + betweenBatches).UtcDateTime }), "the stall watchdog stops a copy whose LastProgressAt stops moving, so every batch has to move it"); + Assert.That(checkpoint.LastProgressAt, Is.EqualTo((startedAt + betweenBatches).UtcDateTime), "the saved row carries the last batch's stamp"); + } + } + + [Test] + public async Task A_category_that_read_fewer_rows_than_the_source_holds_still_settles_complete() + { + // The row says the source held 10 and the source yields 2. The engine judges only its own accounting, + // so a read that ends early is not something it can see, and only --migration-verify shows it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("d"), Row("e")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, 10, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task An_optional_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + [Test] + public async Task A_required_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + // The target commits a checkpoint and a result that agree with each other, so only the rows read can show the one it lost. + static Task<MigrationCheckpoint> RunWithOneRowUnaccountedFor(MigrationCategory category) + { + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 4, UnderReportBy = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + return engine.RunCategoryAsync(category); + } + + [Test] + public async Task A_category_with_no_rows_finishes_Complete_without_a_cursor() + { + // The ordinary state of several required categories on a small instance: nothing to copy is a + // finished category, not a category that never ran. + var category = MigrationCategoryRegistry.Find("MessageRedirects")!; + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((0L, 0L))); + Assert.That(checkpoint.Cursor, Is.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task The_moment_a_category_settles_comes_from_the_injected_clock() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var settledAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var clock = new FakeTimeProvider(settledAt); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SettledAt, Is.EqualTo(settledAt.UtcDateTime), "a wall-clock read here would drift from every other time the migration reports"); + } + + [Test] + public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_second_run() + { + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var finishedWithSkips = new MigrationCheckpoint(category.Id, MigrationCategoryState.CompleteWithErrors, "b", 1, 1, 2, + new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 }, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(finishedWithSkips); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(finishedWithSkips with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_category_already_Halted_is_left_alone_on_a_second_run() + { + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, "a", 1, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_category_already_Complete_is_left_alone_on_a_second_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var alreadyDone = new MigrationCheckpoint(category.Id, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(alreadyDone); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(alreadyDone with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task The_merged_and_changed_rows_the_target_reports_add_up_for_each_category_in_this_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d"), Row("e"), Row("f")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, new MergingTarget(checkpointStore), checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + await engine.RunCategoryAsync(category); + + Assert.That(engine.MergedAndChangedThisRun, Is.EquivalentTo(new Dictionary<string, (long Merged, long Changed)> { [category.Id] = (2, 4) }), + "the checkpoint does not hold these counts, so the summary after the copy has only what the engine adds up"); + } + + // Reports one row of every batch merged and two of the rest changed. + sealed class MergingTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(3); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(batch.Rows.Count - 1, 0, 1, null), cancellationToken); + return new MigrationWriteResult(saved, batch.Rows.Count - 1, 0, [], AlreadyPresent: 1, Merged: 1, Changed: 2); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs new file mode 100644 index 0000000000..39ce6653c2 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -0,0 +1,435 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineFailurePathTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + + [Test] + public async Task A_failing_write_halts_the_category_and_records_the_error_instead_of_throwing() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Simulated failure")); + // The first batch committed, so the cursor is real and a later run resumes from it. + Assert.That(checkpoint.Cursor, Is.EqualTo("b")); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task The_halt_is_durable_so_the_health_check_and_the_guard_can_read_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + await engine.RunCategoryAsync(category); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(persisted.LastError, Is.Not.Null); + } + } + + [Test] + public void A_cancelled_run_is_a_shutdown_rather_than_a_failure_and_is_not_swallowed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync<OperationCanceledException>(() => engine.RunCategoryAsync(category, stopping.Token)); + } + + [Test] + public async Task A_halted_category_is_left_alone_until_the_operator_puts_it_back_in_progress() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var halted = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + + // The cause is fixed, but nothing re-runs a Failed category until the operator says so. + target.FailOnCallNumber = null; + var untouched = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + // The reset --migration-retry writes: a re-read from the start that keeps only when the category first started. + await checkpointStore.Upsert(untouched with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(untouched, Is.EqualTo(halted), "a start re-ran a Failed category the operator had not put back"); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "a retried category does not keep the error it failed with"); + Assert.That(finished.SourceTotal, Is.Null); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(new[] { "a", "b", "c", "d" })); + // Rows a and b were committed before the failure, so the re-read finds them already present. + Assert.That((finished.CopiedCount, finished.AlreadyPresentCount), Is.EqualTo((2L, 2L))); + } + } + + [Test] + public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_across_the_restart() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // msg-1's body is unreadable on both runs: every attempt fails on each. + source.FailBodyReads("msg-1", 2 * MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var firstRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var halted = await firstRun.RunCategoryAsync(category); + + // The row a crash just before the failing batch commits leaves, which a start resumes from its cursor. + await checkpointStore.Upsert(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null }); + target.FailOnCallNumber = null; + var secondRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var finished = await secondRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + // The failed batch never committed, so its skip is recorded only when the batch is read again. + Assert.That(halted.SkippedCount, Is.Zero); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(finished.SkippedCount, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + } + } + + [Test] + public async Task An_abandoned_category_is_left_exactly_as_it_is() + { + // Abandoned is the one end a person chooses: this category will never be copied, and the + // guard and the health check both treat it as settled. A later run must not quietly restart it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var abandoned = new MigrationCheckpoint(category.Id, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the body store was unreachable"); + await checkpointStore.Upsert(abandoned); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(abandoned with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_target_with_no_batch_size_for_a_category_halts_it_and_the_next_category_still_runs() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var unmapped = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var next = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(unmapped.Id, Row("k-1")); + source.Seed(next.Id, Row("r-1")); + var target = new InMemoryMigrationTarget(checkpointStore) { NoBatchSizeFor = unmapped.Id }; + var engine = BuildEngine(source, checkpointStore, target); + + var results = await engine.RunCategories([unmapped, next]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].LastError, Does.Contain("No batch size")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public void A_write_failure_halt_logs_its_exception_before_it_settles() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + Assert.ThrowsAsync<TimeoutException>(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Exception?.Message), Does.Contain("Simulated failure on write 1")); + } + + [Test] + public void A_threshold_halt_logs_its_reason_before_it_settles() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + var logger = new CapturingLogger(); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + Assert.ThrowsAsync<TimeoutException>(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); + } + + [Test] + public void An_optional_threshold_halt_whose_save_fails_throws_rather_than_staying_copying() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + Assert.ThrowsAsync<TimeoutException>(() => engine.RunCategoryAsync(category)); + + var stored = checkpointStore.Read(category.Id).GetAwaiter().GetResult(); + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.LastError, Is.Null, "the failed save of a halt was recorded as the category's own error, which leaves a Failed category reading as still copying"); + } + } + + [Test] + public async Task An_optional_category_that_throws_stays_copying_with_the_error_and_its_cursor() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + foreach (var row in new[] { checkpoint, stored! }) + { + Assert.That(row.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("b")); + Assert.That(row.CopiedCount, Is.EqualTo(2)); + Assert.That(row.SettledAt, Is.Null); + Assert.That(row.LastError, Does.Contain("Simulated failure")); + Assert.That(row.LastError, Does.Not.Contain("--migration-abandon").And.Not.Contain("--migration-retry")); + } + } + } + + [Test] + public async Task An_optional_category_an_exception_left_copying_resumes_on_the_next_start_and_clears_its_error() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + target.FailOnCallNumber = null; + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "the error stays on the row only until a start tries again"); + Assert.That(finished.CopiedCount, Is.EqualTo(4)); + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b", "c", "d" }), "the resume did not carry on from the cursor"); + } + } + + [Test] + public async Task A_checkpoint_conflict_leaves_the_other_writer_alone_instead_of_halting_over_it() + { + // Two hosts pointed at one target is what the version token exists for. Settling this as halted + // would write over the progress of whichever host is still copying. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + // Save 1 moves the row to in progress; save 2 is the first batch, by which point the other host has moved it on. + var checkpointStore = new ConflictOnNthSaveCheckpointStore { ConflictOnSave = 2 }; + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + + Assert.ThrowsAsync<MigrationCheckpointConflictException>(() => engine.RunCategoryAsync(category)); + + var persisted = await checkpointStore.Read(category.Id); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "no halted row was written over the conflict"); + } + + [Test] + public async Task A_cancellation_that_is_not_a_shutdown_halts_the_category_like_any_other_failure() + { + // An inner timeout surfaces as the same exception type as a host stopping, and only the token + // says which. Treating a timeout as a shutdown would end the run with no reason on the row. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) + { + FailOnCallNumber = 1, + FailWith = new OperationCanceledException("the query timed out") + }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("OperationCanceledException").And.Contain("the query timed out")); + } + } + + [Test] + public async Task The_halt_reason_names_the_cursor_the_copy_had_reached() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at cursor b"), "the cursor is the only pointer an operator has to where it stopped"); + } + + [Test] + public async Task The_halt_reason_says_at_the_start_when_the_first_batch_never_committed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at the start")); + } + + [Test] + public async Task A_stop_between_batches_leaves_the_row_in_progress_at_the_batch_that_committed() + { + // The stop lands in the source rather than in a write, so nothing is mid-transaction: the row + // still has to describe the batches that did commit, and stay resumable. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, CancelOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync<OperationCanceledException>(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("b")); + Assert.That(persisted.CopiedCount, Is.EqualTo(2)); + Assert.That(persisted.SettledAt, Is.Null); + } + } + + // Another host moved the row on between this host reading it and saving it. + sealed class ConflictOnNthSaveCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + int saves; + + public int ConflictOnSave { get; init; } + + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ++saves == ConflictOnSave + ? throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row has moved on.") + : saved.Upsert(checkpoint, cancellationToken); + } + + // The store shares the target's database, which has become unreachable by the time the halt is saved. + sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + checkpoint.State == MigrationCategoryState.Halted ? throw new TimeoutException("checkpoint store unreachable") : saved.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs new file mode 100644 index 0000000000..9e3a292a46 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -0,0 +1,327 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineHaltTests +{ + // A body already attached, so the engine reads none and a row is lost only where a test rejects it. + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>(), new MigrationBody(new byte[] { 1 }, "application/octet-stream")); + + [Test] + public async Task A_systemic_failure_halts_the_category_partway_through() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // Every 5th of 1,000 rows is rejected: a steady 20% spread evenly rather than clustered at the + // start, past both the 5% proportion and the 100-row floor. + var rows = Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}")).ToArray(); + source.Seed(category.Id, rows); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Halted")); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + // Stopped partway: the 1,000th row was never reached. + Assert.That(target.WrittenRows(category.Id).Count, Is.LessThan(800)); + } + } + + [Test] + public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // The same 20% that halts the category above, except these rows are past the target's retention + // cutoff, so leaving them behind is the copy working rather than failing. + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((800L, 200L))); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(800), "the last row was reached, so nothing halted partway"); + } + } + + [TestCase(25, MigrationCategoryState.CompleteWithErrors, TestName = "A_mix_of_benign_and_fault_skips_runs_on_while_the_faults_stay_under_the_threshold")] + [TestCase(10, MigrationCategoryState.Halted, TestName = "A_mix_of_benign_and_fault_skips_halts_once_the_faults_alone_pass_the_threshold")] + public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_alone(int everyNthIsAFault, MigrationCategoryState expected) + { + // A real archive copy loses rows both ways at once: retention takes some, unreadable bodies take + // others. This is the only shape where the sum has to leave some reasons out. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 5_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + // A fifth of the category is past retention either way, which on its own is four times the threshold. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + // The offset keeps the faults clear of the benign rows: 4% of the category in one case, 10% in the other. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % everyNthIsAFault == 3)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(expected)); + } + + [Test] + public async Task Rows_already_present_keep_a_category_under_the_halt_threshold() + { + // Already-present rows are in the denominator because the run did handle them. Drop them from it + // and this category's 4% fault rate reads as 12%, halting a copy that is merely being re-run. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 3_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 2_000)) + { + target.SeedExistingKey($"row-{i}"); + } + // 120 faults spread through the whole category: past the 100-row floor, and 4% of 3,000. + foreach (var i in Enumerable.Range(1, 3_000).Where(i => i % 25 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(120)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(1_920), "the 80 already-present rows that are also faults are refused before the collision check"); + } + } + + [Test] + public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.SeedExistingKey($"row-{i}"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(800)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(200)); + } + } + + [Test] + public async Task A_resumed_run_counts_only_its_own_skips_and_keeps_the_earlier_ones() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + // What a shutdown after the sixth batch leaves: 120 fault skips in 600 rows, past both the floor and 5% if counted again. + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, "row-600", 480, 120, null, + new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 120 }, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); + Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); + } + } + + [Test] + public async Task A_category_smaller_than_the_floor_that_loses_every_row_ends_Failed_rather_than_Done() + { + // Ninety rows is under the hundred-row floor, so the threshold the engine checks after every batch + // can never fire, however many rows are lost. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.RequiredValueMissing); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.State.IsFinished(), Is.False, "a required category that copied nothing must not let the host open"); + Assert.That(checkpoint.CopiedCount, Is.Zero); + } + } + + [Test] + public async Task A_halted_category_is_returned_untouched_and_its_source_is_not_read() + { + // Failed waits for the operator, so a start that read the category again would copy rows nobody asked for. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, null, 0, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.RowsHandedToWrite(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_small_category_losing_rows_the_product_would_drop_anyway_still_completes() + { + // Harmless skips are rows the target would have deleted anyway, so no number of them may stop a + // category or leave it Failed, however small the category is. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } + + [Test] + public async Task A_run_that_ends_with_one_fault_skip_settles_CompleteWithErrors_however_small_the_share() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + target.RejectKey("row-500", MigrationSkipReason.RequiredValueMissing); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one row in a thousand is far under the threshold, and it is still a row the product wanted"); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((999L, 1L))); + } + } + + [Test] + public async Task A_run_whose_only_skips_are_harmless_settles_Complete_and_keeps_its_reasons() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + target.RejectKey("a", MigrationSkipReason.PastRetention); + target.RejectKey("b", MigrationSkipReason.PastRetention); + target.RejectKey("c", MigrationSkipReason.BlankGroupComment); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((1L, 3L)), "a harmless skip is still a skip, and status and verify count it"); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.PastRetention] = 2, + [MigrationSkipReason.BlankGroupComment] = 1 + })); + } + } + + [Test] + public async Task The_threshold_halt_names_its_counts_and_no_command() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + // 120 skips in 600 rows is the first point past both the floor and 5%. + Assert.That(checkpoint.LastError, Does.Contain("120 of 600").And.Contain("5%").And.Contain("100 rows")); + // The engine cannot know which commands a category may take, so the readers of the row add them. + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon").And.Not.Contain("restart to resume")); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs new file mode 100644 index 0000000000..71e62dac7b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEngineOptionsTests +{ + static readonly SettingsRootNamespace Namespace = new("ServiceControl"); + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); + } + + static MigrationEngineOptions Read() => MigrationEngineOptions.FromSettings(Namespace, EventRetention, ErrorRetention); + + [Test] + public void Defaults_match_the_contract_when_nothing_is_configured() + { + var options = Read(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages }), "both optional categories are copied unless turned off"); + Assert.That(options.EventLogWindow, Is.EqualTo(EventRetention), "the event log window defaults to the event retention period"); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention), "the archived and resolved window defaults to the error retention period"); + Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void Reads_configured_values_from_environment_variables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", "2500"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", "10"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", "50"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "2.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "5.12:00:00"); + + var options = Read(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(2500))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(10)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); + Assert.That(options.EventLogWindow, Is.EqualTo(TimeSpan.FromDays(2))); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(TimeSpan.FromDays(5.5))); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages })); + } + } + + [TestCase("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", MigrationCategoryIds.ArchivedAndResolvedFailedMessages)] + [TestCase("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", MigrationCategoryIds.EventLog)] + public void A_zero_window_turns_its_category_off(string variable, string stillSelected) + { + Environment.SetEnvironmentVariable(variable, "0"); + + var options = Read(); + + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { stillSelected })); + } + + [TestCase("a week")] + [TestCase("-1.00:00:00")] + public void Refuses_a_window_that_is_not_a_time_span_of_zero_or_more(string value) + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", value); + + var ex = Assert.Throws<InvalidOperationException>(() => Read()); + + Assert.That(ex.Message, Does.Contain(value).And.Contain(MigrationSettings.EventLogWindowKey), "the refusal has to name both the value and the setting holding it for the customer to fix it"); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs new file mode 100644 index 0000000000..056805e938 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -0,0 +1,172 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineOrderingTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + + [Test] + public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes_and_says_so_on_its_checkpoint_row() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + // LicensingEndpoints has never run: no checkpoint row for it at all. + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + var persisted = await checkpointStore.Read(throughputCategory.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); + // A row exists, so status can print it. Without one, an operator cannot tell a category + // waiting on another from a category nobody asked for. + Assert.That(persisted, Is.Not.Null); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(persisted.LastError, Does.Contain("LicensingEndpoints")); + } + } + + [Test] + public async Task EndpointSettings_does_not_start_before_KnownEndpoints_completes() + { + // Not a foreign key: settings wait for known endpoints, and a halted required category keeps + // the host, and with it the heartbeat sync, closed. + var settingsCategory = MigrationCategoryRegistry.Find("EndpointSettings")!; + var source = new InMemoryMigrationSource(); + source.Seed(settingsCategory.Id, Row("EndpointSettings/1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(settingsCategory); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(settingsCategory.Id), Is.Empty); + } + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task GroupComments_does_not_start_before_the_unresolved_failed_messages_complete(MigrationCategoryState predecessorState) + { + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); + Assert.That(target.WrittenRows(comments.Id), Is.Empty); + } + } + + [Test] + public async Task A_blocked_category_runs_once_the_category_it_follows_settles() + { + // A block nothing can clear would strand group comments and keep the host closed for good. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var unresolved = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var predecessorRunning = new MigrationCheckpoint(unresolved.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + var saved = await checkpointStore.Upsert(predecessorRunning); + var target = new InMemoryMigrationTarget(checkpointStore); + + var blocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + await checkpointStore.Upsert(saved with { State = MigrationCategoryState.Complete, SettledAt = DateTime.UtcNow }); + var unblocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(unblocked.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(unblocked.LastError, Is.Null, "the block reason does not outlive the block"); + Assert.That(target.WrittenRows(comments.Id), Has.Count.EqualTo(1)); + } + } + + [Test] + public async Task A_blocked_category_has_not_settled_because_it_has_not_finished() + { + // Anything reading SettledAt beside State would otherwise take a blocked category for a finished one. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + var persisted = await checkpointStore.Read(comments.Id); + Assert.That(persisted!.SettledAt, Is.Null); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Blocked)] + public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState predecessorState) + { + // A missing row reads as "not started"; a row that exists says what it actually holds, which is + // how an operator tells a category waiting its turn from one waiting on a chain. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, null, 0, 0, null, null, null, null, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finished_or_abandoned(MigrationCategoryState endpointsState) + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs new file mode 100644 index 0000000000..3a2dca44d7 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -0,0 +1,172 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineResumeTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task A_restart_keeps_the_moment_the_category_first_started() + { + // StartedAt is what an operator reads to see how long a background copy has been going. Stamping + // it again on every restart would report minutes for a copy that has been running for days. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstStart = new DateTimeOffset(2026, 3, 1, 9, 0, 0, TimeSpan.Zero); + var halted = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(firstStart), options, NullLogger<MigrationEngine>.Instance) + .RunCategoryAsync(category); + + // A day later, the cause is fixed and the operator puts the category back as --migration-retry does. + target.FailOnCallNumber = null; + await checkpointStore.Upsert(halted with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); + var restart = firstStart.AddDays(1); + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(restart), options, NullLogger<MigrationEngine>.Instance) + .RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.StartedAt, Is.EqualTo(firstStart.UtcDateTime)); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "a retried category keeps the moment it first started"); + Assert.That(finished.SettledAt, Is.EqualTo(restart.UtcDateTime)); + } + } + + [Test] + public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_and_no_gaps() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + var allIds = Enumerable.Range(1, 6).Select(i => $"row-{i}").ToArray(); + source.Seed(category.Id, [.. allIds.Select(Row)]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + // Batches are 2 rows each; the host stops on the 3rd call to Write, so exactly 2 batches (4 rows) land. + Assert.ThrowsAsync<OperationCanceledException>(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + + var afterStop = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + // Still InProgress, not Halted: a cancelled run is a shutdown, not a failure. + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(afterStop.CopiedCount, Is.EqualTo(4)); + Assert.That(afterStop.Cursor, Is.EqualTo("row-4")); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(4)); + } + + // Restart: same source, same target, same checkpoint store, nothing stopping it this time. + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finalCheckpoint.CopiedCount, Is.EqualTo(6)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(allIds), "no gaps"); + // The target de-duplicates, as the real ones do, so what it kept can never show a row sent twice. + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.Unique, "no duplicates: the restart resumes past the rows the first run committed"); + } + } + + [Test] + public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_batch() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 6).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + // Both in the second batch, the last one to commit before the stop. + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + Assert.ThrowsAsync<OperationCanceledException>(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + var afterStop = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That((afterStop.CopiedCount, afterStop.SkippedCount, afterStop.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present after the stop"); + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + } + + [Test] + public async Task A_hard_crash_after_a_committed_write_loses_nothing_because_the_target_saved_the_real_split() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 4).Select(i => Row($"row-{i}"))]); + var committed = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(committed) { DefaultBatchSize = 2 }; + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var crashingEngine = new MigrationEngine(source, target, new CrashAfterCommitCheckpointStore(committed, crashAfterCursor: "row-4"), new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + await crashingEngine.RunCategoryAsync(category); + + var restartedEngine = new MigrationEngine(source, target, committed, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + var finalCheckpoint = await restartedEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // The engine's own settle was lost, but every count came from the target's own transaction. + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "row-1", "row-2" })); + } + } + + // Once the target has committed crashAfterCursor, the engine's own saves are lost, as if the process died there. + sealed class CrashAfterCommitCheckpointStore(IMigrationCheckpointStore committed, string crashAfterCursor) : IMigrationCheckpointStore + { + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) => committed.ReadAll(cancellationToken); + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) => committed.Read(categoryId, cancellationToken); + + public async Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + (await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor == crashAfterCursor + ? checkpoint + : await committed.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs new file mode 100644 index 0000000000..d08b17f548 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -0,0 +1,158 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineRunCategoriesTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + // The reverse of registry order, so an engine that re-sorted by Order would run KnownEndpoints first. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("MessageRedirects")!, MigrationCategoryRegistry.Find("KnownEndpoints")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + var results = await engine.RunCategories(given); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "MessageRedirects", "KnownEndpoints" })); + Assert.That(results.Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task Every_category_it_is_given_has_a_row_before_the_first_one_copies_anything() + { + // A copy stopped between two categories must not leave a table that reads as finished, because the + // gates that keep a host off an unfinished copy look only at the rows that exist. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + IReadOnlyList<MigrationCheckpoint>? rowsAtFirstWrite = null; + var target = new InMemoryMigrationTarget(checkpointStore) { BeforeWrite = _ => rowsAtFirstWrite ??= checkpointStore.ReadAll().GetAwaiter().GetResult() }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + // Three, so recording only the next category would still leave the last one missing. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!, MigrationCategoryRegistry.Find("MessageRedirects")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + await engine.RunCategories(given); + + Assert.That(rowsAtFirstWrite, Is.Not.Null, "the copy never wrote a batch"); + using (Assert.EnterMultipleScope()) + { + Assert.That(rowsAtFirstWrite!.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "KnownEndpoints", "EndpointSettings", "MessageRedirects" })); + Assert.That(rowsAtFirstWrite!.Where(c => c.CategoryId != "KnownEndpoints").Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.NotStarted)); + } + } + + [Test] + public async Task A_halted_category_is_left_untouched_and_the_next_category_still_runs() + { + var firstStarted = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Halted, "KnownEndpoints-1", 1, 0, null, null, firstStarted, firstStarted, firstStarted, "Halted: earlier run")); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1"), Row("KnownEndpoints-2")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed("MessageRedirects", Row("MessageRedirects-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("MessageRedirects")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty, "a Failed category waits for the operator"); + Assert.That(results[0], Is.EqualTo(halted), "the halted category's row was changed"); + Assert.That(await checkpointStore.Read("KnownEndpoints"), Is.EqualTo(halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows("MessageRedirects").Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_finished_category_is_not_copied_again() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Complete, "KnownEndpoints-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task Running_no_categories_copies_nothing_and_reports_nothing() + { + // What an instance that selected no optional categories asks for on every background pass. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + + var results = await engine.RunCategories([]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results, Is.Empty); + Assert.That(await checkpointStore.ReadAll(), Is.Empty, "a category nobody ran gets no row"); + } + } + + [Test] + public async Task One_category_halting_does_not_stop_the_ones_after_it() + { + // Everything that can still be copied is copied, and the guard decides what an incomplete migration + // may do. Stopping at the first halt would leave later categories untouched with no reason recorded. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1, FailOnCallNumber = 1 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + var first = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var second = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(first.Id, Row("k-1")); + source.Seed(second.Id, Row("r-1")); + + var results = await engine.RunCategories([first, second]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs new file mode 100644 index 0000000000..7726a173f1 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -0,0 +1,234 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineSkipReasonTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkpoint() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // Two reasons over two batches: a and b land in the first, d in the second, so the saved map has to merge both. + target.RejectKey("a", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("b", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("d", MigrationSkipReason.PastRetention); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); + } + } + + [Test] + public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnreadable() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + + [Test] + public async Task A_message_the_source_holds_no_body_for_is_counted_under_BodyMissing() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "msg-1 is the only skip, so a missing body has to count as a fault on its own"); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyMissing] = 1 }), + "the refusal tells the operator which skips no retry can fix only from these reasons"); + Assert.That(logger.Entries.Where(entry => entry.Message.StartsWith("Skipped msg-1")).Select(entry => entry.Message), Has.Exactly(1).Contains("BodyMissing"), + "the checkpoint holds counts, not ids, so the log is the operator's only record of which message was left out"); + } + } + + [Test] + public async Task A_batch_that_loses_rows_two_different_ways_records_both_reasons() + { + // The breakdown is what tells a customer what they lost and why. A merge that overwrote instead of + // summing would leave the total right and the reasons wrong, and verification would still balance. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }, Row("msg-3") with { Body = new MigrationBody(new byte[] { 1 }, "application/octet-stream") }); + // One the engine skips itself because the body will not read, one the target refuses. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + target.RejectKey("msg-2", MigrationSkipReason.PastRetention); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.BodyUnreadable] = 1, + [MigrationSkipReason.PastRetention] = 1 + })); + Assert.That(target.WrittenRows(category.Id).Select(row => row.SourceId), Is.EqualTo(new[] { "msg-3" })); + } + } + + [Test] + public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new UnexplainedSkipTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("reported 1 skipped").And.Contain("reasons for 0")); + Assert.That(checkpoint.Cursor, Is.Null, "the target refused ahead of its commit, so nothing was written and a restart reads the batch again"); + } + } + + [Test] + public async Task A_target_whose_reported_counts_disagree_with_the_checkpoint_it_committed_never_finishes_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new MiscountedCopyTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.LastError, Does.Contain("but the checkpoint it committed moved by"), "the halt has to name which two accounts disagreed, or the operator goes looking for the wrong problem"); + Assert.That(stored.State.IsFinished(), Is.False, "the halt threshold never saw the 150 rows the target dropped, so the category settled finished and the host opened on half a category"); + } + } + + [Test] + public async Task An_optional_category_whose_reported_counts_disagree_with_the_checkpoint_it_committed_settles_halted() + { + // An optional category stays copying after an exception, and this check must not become one of those. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new MiscountedCopyTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored.LastError, Does.Contain("reported copying 10").And.Contain("but the checkpoint it committed moved by")); + Assert.That(stored.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + // Commits half of every batch as skipped and reports the whole batch copied. The saved counts still add up + // to the source total, so nothing later in the run can notice, and the halt threshold sees a clean copy. + sealed class MiscountedCopyTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var skipped = batch.Rows.Count / 2; + var reasons = new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = skipped }; + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(batch.Rows.Count - skipped, skipped, 0, reasons), cancellationToken); + return new MigrationWriteResult(saved, batch.Rows.Count, 0, []); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } + + sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task<int> BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); + + public async Task<MigrationWriteResult> Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, null), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, []); + } + + public Task<long> Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public Task<MigrationCheckpoint> ResetForRetry(MigrationCheckpoint reset, CancellationToken cancellationToken = default) => checkpointStore.Upsert(reset, cancellationToken); + + public IReadOnlyCollection<string> SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary<string, Type> DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs new file mode 100644 index 0000000000..e568edb6a1 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -0,0 +1,97 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineThrottleTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + [Test] + public async Task Optional_categories_pause_between_batches_for_the_configured_duration() + { + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + var pause = TimeSpan.FromSeconds(1); + var options = new MigrationEngineOptions(pause, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger<MigrationEngine>.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // Three batches, so a pause before the second and another before the third. + for (var pauseNumber = 1; pauseNumber <= 2; pauseNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"pause {pauseNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"pause {pauseNumber} should still be pending"); + clock.Advance(pause); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { pause, pause })); + } + } + + [Test] + public async Task A_stop_during_a_pause_ends_the_run_as_a_shutdown() + { + // Most of a throttled background copy's life is spent in this pause, so it is where a host being + // stopped most often lands. + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + using var stopping = new CancellationTokenSource(); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger<MigrationEngine>.Instance); + + var runTask = engine.RunCategoryAsync(category, stopping.Token); + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, "the first pause never started"); + await stopping.CancelAsync(); + + Assert.ThrowsAsync<TaskCanceledException>(() => runTask); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown mid-pause is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("a"), "the batch before the pause committed"); + } + } + + [Test] + public async Task Required_categories_never_pause() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + // A FakeTimeProvider that is never advanced: if the engine tried to pause, this would hang + // until the test runner's own timeout, which WaitAsync turns into a clear failure instead. + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger<MigrationEngine>.Instance); + + var checkpoint = await engine.RunCategoryAsync(category).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs new file mode 100644 index 0000000000..260215cdf7 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationIsReleasedCheckTests +{ + [Test] + public void A_build_without_the_whole_migration_refuses_the_offline_command() + { + var exception = Assert.ThrowsAsync<Exception>(() => new MigrationIsReleasedCheck().Run()); + + Assert.That(exception.Message, Does.Contain("does not yet carry the whole migration").And.Contain("--start-data-migration"), + "the customer has to learn that the offline command is unavailable"); + } + + // A RavenDB target fails the pair check, so this only passes while the release check runs before it. + [Test] + public void An_unreleased_build_refuses_before_any_other_check() + { + var settings = new Settings(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); + using var services = new ServiceCollection().BuildServiceProvider(); + + var exception = Assert.ThrowsAsync<Exception>(() => MigrationStartup.RunRequiredCopy(services, settings)); + + Assert.That(exception.Message, Does.Contain("this build carries the whole migration").And.Not.Contain("supported pair")); + } + + [Test] + public void The_test_marker_lets_a_copy_run_on_an_unreleased_build() + { + Assert.DoesNotThrowAsync(() => new MigrationIsReleasedCheck(new AllowUnreleasedMigration()).Run()); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs new file mode 100644 index 0000000000..c25bf44d0c --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; + +[TestFixture] +class MigrationPairIsSupportedCheckTests +{ + [OneTimeSetUp] + public static void TheTreeIsBuilt() => + Assert.That( + PersistenceFactory.SqlPersistenceNames.Where(name => PersistenceManifestLibrary.Find(name) is null), + Is.Empty, + "These persistence manifests did not resolve, so this fixture cannot tell a supported pair from an unsupported one. Build the tree first with: dotnet build src --configuration Release -graph"); + + [TestCase("SQLServer")] + [TestCase("PostgreSQL")] + public void RavenDB_to_a_SQL_persister_is_supported(string target) => + Assert.DoesNotThrowAsync(() => Check(target).Run()); + + [Test] + public void A_RavenDB_target_is_refused_naming_the_target_setting() + { + var exception = Assert.ThrowsAsync<Exception>(() => Check("RavenDB").Run()); + + Assert.That(exception.Message, Does.Contain("ServiceControl/PersistenceType")); + } + + static MigrationPairIsSupportedCheck Check(string target) => + new(new Settings(transportType: "LearningTransport", persisterType: target, errorRetentionPeriod: TimeSpan.FromDays(10))); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs new file mode 100644 index 0000000000..2767013bb3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs @@ -0,0 +1,75 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationStartupCheckRunnerTests +{ + [Test] + public async Task Checks_run_in_the_order_they_were_given() + { + var ran = new List<string>(); + + await MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the second", _ => ran.Add("second")), + Check("the third", _ => ran.Add("third")) + ]); + + Assert.That(ran, Is.EqualTo(new[] { "first", "second", "third" }), "the databases are opened by checks that sit behind the ones refusing the configuration outright"); + } + + [Test] + public void The_first_check_to_refuse_stops_the_ones_behind_it() + { + var ran = new List<string>(); + var failure = new InvalidOperationException("ServiceControl/RetryHistoryDepth is 0"); + + var exception = Assert.ThrowsAsync<Exception>(() => MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the migration target is ready", _ => throw failure), + Check("the last", _ => ran.Add("last")) + ])); + + using (Assert.EnterMultipleScope()) + { + Assert.That(ran, Is.EqualTo(new[] { "first" }), "a check behind a refusal would run against a configuration already known to be wrong, and the two that open a database are at the end of the list"); + Assert.That(exception!.Message, Does.Contain("the migration target is ready").And.Contain(failure.Message), "the operator needs to know which check refused and why"); + Assert.That(exception.InnerException, Is.SameAs(failure), "the original carries the stack and the detail the summary leaves out"); + } + } + + [Test] + public void A_host_being_stopped_is_not_a_check_refusing() + { + using var stopping = new CancellationTokenSource(); + stopping.Cancel(); + + var exception = Assert.ThrowsAsync<OperationCanceledException>(() => MigrationStartupCheckRunner.Run( + [Check("the first", token => token.ThrowIfCancellationRequested())], stopping.Token)); + + Assert.That(exception!.Message, Does.Not.Contain("Migration startup check"), "a stop dressed up as a refusal sends the operator after a setting that was never wrong"); + } + + static IMigrationStartupCheck Check(string name, Action<CancellationToken> run) => new FakeCheck(name, run); + + sealed class FakeCheck(string name, Action<CancellationToken> run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) + { + run(cancellationToken); + return Task.CompletedTask; + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs b/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs new file mode 100644 index 0000000000..b58c28d598 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/OptionalCategoryCopierTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class OptionalCategoryCopierTests +{ + [Test] + public async Task A_persister_without_checkpoints_needs_no_source_or_target() + { + using var services = new ServiceCollection().BuildServiceProvider(); + await Run(services); + } + + [Test] + public async Task A_database_that_never_migrated_needs_no_source_or_target() + { + using var services = new ServiceCollection() + .AddSingleton<IMigrationCheckpointStore>(new InMemoryMigrationCheckpointStore()) + .BuildServiceProvider(); + await Run(services); + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task A_finished_or_failed_optional_category_does_not_open_the_source(MigrationCategoryState state) + { + var store = new InMemoryMigrationCheckpointStore(); + var checkpoint = await store.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, state, null, 0, 0, null, null, null, null, null, null)); + using var services = new ServiceCollection().AddSingleton<IMigrationCheckpointStore>(store).BuildServiceProvider(); + await Run(services); + Assert.That(await store.Read(checkpoint.CategoryId), Is.EqualTo(checkpoint)); + } + + static async Task Run(ServiceProvider services) + { + var logger = new ErrorLogger(); + using var copier = new OptionalCategoryCopier(services, null, new StartedLifetime(), logger); + await copier.StartAsync(CancellationToken.None); + await copier.ExecuteTask.WaitAsync(TimeSpan.FromSeconds(10)); + await copier.StopAsync(CancellationToken.None); + Assert.That(logger.Errors, Is.Empty, "an idle copier must not attempt to resolve or open a source or target"); + } + + sealed class ErrorLogger : ILogger<OptionalCategoryCopier> + { + public List<string> Errors { get; } = []; + public bool IsEnabled(LogLevel logLevel) => true; + public IDisposable BeginScope<TState>(TState state) where TState : notnull => null; + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter) + { + if (logLevel >= LogLevel.Error) + { + Errors.Add(formatter(state, exception)); + } + } + } + + sealed class StartedLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => new(true); + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() { } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs new file mode 100644 index 0000000000..6f4393ab40 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class OptionalCategoryWindowsAreValidCheckTests +{ + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); + + [TearDown] + public void ClearWindows() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); + } + + [Test] + public async Task Unset_windows_pass_and_default_to_the_retention_periods_it_was_given() + { + var check = new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention); + + await check.Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(check.Options.EventLogWindow, Is.EqualTo(EventRetention)); + Assert.That(check.Options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention)); + } + } + + [Test] + public void A_window_that_is_not_a_time_span_is_refused_and_named() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "a week"); + + var exception = Assert.ThrowsAsync<InvalidOperationException>(async () => + await new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention).Run()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain("a week")); + Assert.That(exception.Message, Does.Contain(MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs new file mode 100644 index 0000000000..cc598257ab --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs @@ -0,0 +1,80 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RecordSourceOutageTests +{ + static readonly MigrationCategory EventLog = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + static readonly MigrationCategory ArchivedAndResolvedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + + readonly InMemoryMigrationCheckpointStore checkpointStore = new(); + + [Test] + public async Task A_category_still_copying_keeps_its_state_and_gains_the_error() + { + await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress, "EventLogItem/40", 40, 0, null, null, null, null, null, null)); + + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("EventLogItem/40")); + Assert.That(row.CopiedCount, Is.EqualTo(40)); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [Test] + public async Task A_category_with_no_row_gets_a_not_started_row_carrying_the_error() + { + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + Assert.That(row, Is.Not.Null, "status and verify read only the rows that exist, so a category with no row would show no error"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(row.CopiedCount, Is.Zero); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.AlreadyPresentCount, Is.Zero); + Assert.That(row.StartedAt, Is.Null, "a category that never started must not look as if it had"); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task A_settled_category_is_left_alone(MigrationCategoryState state) + { + var before = await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, state, null, 0, 0, null, null, null, null, null, "earlier")); + + await RecordOutage(); + + var after = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(after!.Version, Is.EqualTo(before.Version), "a settled row is never saved again"); + Assert.That(after.State, Is.EqualTo(state)); + Assert.That(after.LastError, Is.EqualTo("earlier")); + } + } + + Task RecordOutage() => + MigrationStartup.RecordSourceOutage(checkpointStore, [EventLog, ArchivedAndResolvedFailedMessages], new InvalidOperationException("RavenDB is unreachable")); +} diff --git a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs new file mode 100644 index 0000000000..0bd83978d7 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs @@ -0,0 +1,555 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RequiredCopyGateTests +{ + [TestCase(MigrationCategoryState.Complete)] + public void A_finished_required_category_lets_the_host_open(MigrationCategoryState state) => + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.NotStarted)] + // Blocked happens because EndpointSettings must follow KnownEndpoints. + [TestCase(MigrationCategoryState.Blocked)] + public void A_required_category_that_is_not_finished_keeps_the_host_closed(MigrationCategoryState state) + { + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings).And.Contain(state.ToString())); + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + public void An_unfinished_row_under_an_id_this_build_does_not_know_keeps_the_host_closed(MigrationCategoryState state) + { + var unknown = Checkpoint(state, UnknownCategoryId); + + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy([unknown, Checkpoint(MigrationCategoryState.Complete)], [EndpointSettings]); + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete), .. outside], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is ").And.Contain(state.ToString()), + "an id this build does not know is required until it is known to be optional, as the ingestion gate already decides"); + } + + [Test] + public void An_unfinished_unknown_row_makes_the_copy_not_settled_and_the_refusal_names_it() + { + var unknown = Checkpoint(MigrationCategoryState.InProgress, UnknownCategoryId); + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), unknown }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.False); + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is InProgress")); + }); + } + + [Test] + public void A_finished_unknown_row_leaves_the_copy_settled_and_the_refusal_silent() + { + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), Checkpoint(MigrationCategoryState.Abandoned, UnknownCategoryId) }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.True); + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + }); + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + public void A_finished_row_under_an_id_this_build_does_not_know_holds_nothing(MigrationCategoryState state) => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(state, UnknownCategoryId)], [EndpointSettings]), Is.Empty); + + [Test] + public void An_unfinished_row_known_to_be_optional_holds_nothing() => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(MigrationCategoryState.InProgress, MigrationCategoryIds.EventLog)], [EndpointSettings]), Is.Empty); + + [Test] + public void A_category_that_reported_no_checkpoint_at_all_keeps_the_host_closed() + { + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings), + "an empty result reads as success unless the gate checks what it asked for against what came back"); + } + + [Test] + public void Of_two_attempted_categories_the_one_that_reported_nothing_is_the_one_named() + { + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [KnownEndpoints, EndpointSettings], + [Checkpoint(MigrationCategoryState.Complete, MigrationCategoryIds.KnownEndpoints)], + NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} reported no checkpoint at all").And.Not.Contain(MigrationCategoryIds.KnownEndpoints), + "comparing counts instead of ids would let a two-category run through with one category's fate unknown"); + } + + [Test] + public void The_refusal_reads_to_its_end_as_a_sentence_and_says_how_to_get_back() + { + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Halted)], NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a Failed category waits for the operator, so a refusal naming neither command leaves them restarting for ever"); + Assert.That(exception.Message, Does.Not.Contain("the copy resumes from its last committed batch"), "a restart no longer copies a Failed category"); + Assert.That(exception.Message, Does.Contain("both with ServiceControl stopped. Nothing has opened on SQLServer yet") + .And.EndWith("returns the instance to RavenDB with no loss."), + "the operator reads the whole message, and the rollback instructions are at the end of it"); + }); + } + + [Test] + public void Every_failed_required_category_is_named_with_its_skips_and_both_commands() + { + var withErrors = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) with + { + CopiedCount = 40, + SkippedCount = 6, + SkipReasons = new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 3, + [MigrationSkipReason.BodyMissing] = 1 + } + }; + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.MessageRedirects) with { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [MessageRedirects, UnresolvedAndRetryIssuedFailedMessages], [halted, withErrors], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in new[] { MigrationCategoryIds.MessageRedirects, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages }) + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {categoryId} once the cause is fixed").And.Contain($"--migration-abandon {categoryId} to keep what was copied"), + $"one start gives every required category its go, so the refusal has to name every Failed one, {categoryId} included"); + } + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.MessageRedirects} is Failed (Halted)").And.Contain("the target did not answer")); + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages} is Failed (CompleteWithErrors) after copying 40 and skipping 6")); + Assert.That(exception.Message, Does.Contain("RequiredValueMissing 2, no retry can fix"), "a row no retry can store is a reason to abandon, and the operator has to be told which"); + Assert.That(exception.Message, Does.Contain("BodyUnreadable 3").And.Not.Contain("BodyUnreadable 3, no retry can fix"), "an unreadable body may read on the next try"); + Assert.That(exception.Message, Does.Contain("BodyMissing 1, no retry can fix"), "a body the source does not hold is missing again on every retry"); + } + } + + [Test] + public void A_category_waiting_behind_a_failed_one_is_named_as_waiting() + { + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.KnownEndpoints); + var blocked = Checkpoint(MigrationCategoryState.Blocked) with { LastError = "Blocked: EndpointSettings must follow KnownEndpoints, which is Halted" }; + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([KnownEndpoints, EndpointSettings], [halted, blocked], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Blocked").And.Contain("must follow KnownEndpoints")); + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.KnownEndpoints}")); + Assert.That(exception.Message, Does.Not.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Not.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a category that never got its go has nothing to retry; it runs once the one ahead of it is dealt with"); + } + } + + [Test] + public void A_stalled_category_is_refused_with_the_limit_its_row_records() + { + var stalled = Checkpoint(MigrationCategoryState.Halted) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [stalled], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"committed nothing for {MigrationStartup.ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes") + .And.Contain("looking for a setting to change; run --migration-retry EndpointSettings"), + "the stall is on the row, and the refusal is where the operator reads it"); + } + + // The last thing said before the cutover is one way, and the only place a skipped total appears at all. + [Test] + public void A_failed_category_that_skipped_rows_is_told_a_retry_re_reads_them() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 6, new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 1 + })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning), "rows left behind are not an informational matter"); + Assert.That(entry.Message, Does.Contain("6 skipped"), "the total is the number the operator decides on"); + Assert.That(entry.Message, Does.Contain("PastRetention 3").And.Contain("RequiredValueMissing 2").And.Contain("BodyUnreadable 1"), "a total with no reasons cannot be acted on"); + Assert.That(entry.Message, Does.Contain("stay only in the source database").And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them"), + "without this the operator either waits for a copy that is already over or never learns the rows can still come across"); + Assert.That(entry.Message, Does.Contain("No retry can fix the RequiredValueMissing ones"), "the retry brings the unreadable bodies across but leaves these behind again"); + }); + } + + [Test] + public void A_failed_category_whose_faults_no_retry_can_fix_is_pointed_at_abandon() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 5, new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2 + })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("no retry can fix them").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a retry re-reads the whole category and ends Failed on the same rows"); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "harmless skips are not a reason to retry either"); + }); + } + + [Test] + public void A_halted_category_whose_faults_no_retry_can_fix_is_told_abandon_gives_up_every_row_not_yet_read() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.MalformedValue] = 120 }) + with { LastError = "Halted: 120 of 127 rows skipped in this run exceeds the configured threshold of 5% and 100 rows." }], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("stopped before the end").And.Contain("every row not yet read"), + "a halted category never read the rows after its cursor, so abandon gives up far more than the skipped ones"); + Assert.That(entry.Message, Does.Contain(MigrationSettings.HaltThresholdPercentKey).And.Contain(MigrationSettings.HaltThresholdMinimumKey) + .And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}"), "raising the threshold and retrying copies every row the faults do not stop"); + Assert.That(entry.Message, Does.Contain("No retry can fix the MalformedValue ones")); + Assert.That(entry.Message, Does.Not.Contain("gives up the rest")); + }); + } + + [Test] + public void A_halted_category_with_faults_a_retry_can_fix_is_also_told_what_abandon_gives_up() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Halted, copied: 7, skipped: 120, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.BodyUnreadable] = 120 })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("every row not yet read").And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}")); + Assert.That(entry.Message, Does.Not.Contain("No retry can fix"), "an unreadable body may read on the next try"); + }); + } + + [Test] + public void A_category_that_failed_with_only_harmless_skips_is_still_offered_a_retry() + { + var logger = new CapturingLogger(); + + var stoppedByAnException = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 3, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.PastRetention] = 3 }) + with + { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stoppedByAnException], logger, RunStartedAt, NothingMergedOrChanged); + + Assert.That(logger.Entries.Single().Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them").And.Not.Contain("--migration-abandon"), + "no fault skip stands in the way of a retry, so the exception's cause is what the operator fixes"); + } + + [Test] + public void A_done_category_with_harmless_skips_is_not_offered_a_retry() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 3, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.PastRetention] = 3 })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("3 skipped").And.Contain("PastRetention 3")); + Assert.That(entry.Message, Does.Contain("harmless").And.Contain("stay only in the source database")); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "a Done category cannot be retried, so offering it sends the operator to a command that refuses"); + }); + } + + [Test] + public void Every_harmless_skip_reason_has_its_own_explanation_and_no_other_reason_has_one() => + Assert.That(MigrationStartup.HarmlessSkipExplanations.Keys, Is.EquivalentTo(Enum.GetValues<MigrationSkipReason>().Where(reason => reason.IsBenign())), + "the warning for a Done category explains each harmless reason it skipped, so a harmless reason with no explanation would be left out unexplained"); + + [Test] + public void A_done_category_is_told_why_each_of_its_harmless_reasons_is_harmless_and_nothing_about_the_others() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 8, new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.SourceStorageCheck] = 6, + [MigrationSkipReason.BlankGroupComment] = 2 + })], + logger, + RunStartedAt, + NothingMergedOrChanged); + + var message = logger.Entries.Single().Message; + + Assert.Multiple(() => + { + Assert.That(message, Does.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.SourceStorageCheck]).And.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.BlankGroupComment])); + Assert.That(message, Does.Not.Contain(MigrationStartup.HarmlessSkipExplanations[MigrationSkipReason.PastRetention]), "a reason the category did not skip for would give the operator a false reason"); + }); + } + + [Test] + public void The_refusal_marks_a_harmless_reason_so_it_does_not_read_as_rows_a_retry_could_copy() + { + var failed = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.CustomChecks) with + { + SkippedCount = 7, + SkipReasons = new Dictionary<MigrationSkipReason, long> + { + [MigrationSkipReason.SourceStorageCheck] = 6, + [MigrationSkipReason.MalformedValue] = 1 + } + }; + + var exception = Assert.Throws<Exception>(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([CustomChecks], [failed], NewSettings())); + + Assert.That(exception.Message, Does.Contain("SourceStorageCheck 6, harmless").And.Contain("MalformedValue 1, no retry can fix"), "a retry skips the six harmless rows again"); + } + + [Test] + public void A_blocked_category_is_left_to_the_refusal_rather_than_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Blocked, copied: 0, skipped: 0, null)], logger, RunStartedAt, NothingMergedOrChanged); + + Assert.That(logger.Entries, Is.Empty); + } + + // Every refused start reports it again, directly above the refusal that calls it Failed. + [Test] + public void A_failed_category_that_skipped_nothing_is_not_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt, NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning)); + Assert.That(entry.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Failed (Halted) after 10 copied").And.Not.Contain("nothing skipped")); + }); + } + + [Test] + public void A_category_that_skipped_nothing_says_so_without_raising_a_warning() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null)], logger, RunStartedAt, NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information), "a clean copy warning about nothing trains the operator to ignore the warning that matters"); + Assert.That(entry.Message, Does.Contain("nothing skipped")); + }); + } + + // Restarting a migrated instance used to reprint the whole copy summary in the present tense, so an operator + // restarting to change a setting read "7 copied" and had no way to tell the copy had not run again. + [Test] + public void A_category_an_earlier_run_finished_is_not_reported_as_copied_again() + { + var logger = new CapturingLogger(); + + var alreadyDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([alreadyDone], logger, RunStartedAt, NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("already finished before this start"), "without this the line is indistinguishable from a copy that just ran"); + Assert.That(entry.Message, Does.Contain("This start copied nothing"), "the operator needs to know nothing was written to a target that is already serving"); + Assert.That(entry.Message, Does.Contain("7"), "the historical total is still worth stating, just not as this run's work"); + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information)); + }); + } + + // A category this run actually settled must keep the ordinary wording, or the fix above would silence every report. + [Test] + public void A_category_this_run_finished_is_still_reported_as_copied() + { + var logger = new CapturingLogger(); + + var justDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([justDone], logger, RunStartedAt, NothingMergedOrChanged); + + Assert.That(logger.Entries.Single().Message, Does.Contain("7 copied").And.Not.Contain("already finished")); + } + + // A Failed row comes back untouched on every start, so every refused restart reports it again. + [Test] + public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() + { + var logger = new CapturingLogger(); + + var failedEarlier = Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 2, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 }) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([failedEarlier], logger, RunStartedAt, NothingMergedOrChanged); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Not.Contain("already finished"), "the refusal right after this line calls the same category Failed"); + Assert.That(entry.Message, Does.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}").And.Not.Contain("--migration-retry"), + "the advice has to show on every refused start, not only the first, and no retry can store a row missing a required value"); + }); + } + + [Test] + public void A_category_that_merged_or_changed_rows_says_how_many_apart_from_already_present_and_warns() + { + var logger = new CapturingLogger(); + var done = Settled(MigrationCategoryState.Complete, copied: 15, skipped: 0, null) with { AlreadyPresentCount = 3, SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([done], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (1, 4) }); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Is.EqualTo($"{MigrationCategoryIds.EndpointSettings}: 15 copied (4 changed), 2 already present, 1 merged, nothing skipped"), + "a merged row is counted as already present, so without taking it out the summary says nothing changed"); + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning), "a merged or changed row is one the operator did not expect, and the per-row warnings above name each one"); + }); + } + + [Test] + public void A_category_that_merged_rows_and_skipped_some_counts_both() + { + var logger = new CapturingLogger(); + var failed = Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 2, new Dictionary<MigrationSkipReason, long> { [MigrationSkipReason.RequiredValueMissing] = 2 }) + with + { AlreadyPresentCount = 5 }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([failed], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (2, 0) }); + + Assert.That(logger.Entries.Single().Message, Does.StartWith($"{MigrationCategoryIds.EndpointSettings}: 7 copied (0 changed), 3 already present, 2 merged, 2 skipped (RequiredValueMissing 2).")); + } + + [Test] + public void More_merged_rows_than_rows_already_present_is_a_warning_naming_both_counts() + { + var logger = new CapturingLogger(); + var done = Settled(MigrationCategoryState.Complete, copied: 5, skipped: 0, null) with { AlreadyPresentCount = 1, SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([done], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (3, 0) }); + + var messages = logger.Entries.Select(entry => entry.Message).ToArray(); + + Assert.Multiple(() => + { + Assert.That(messages, Has.Member($"{MigrationCategoryIds.EndpointSettings}: the target reported 3 merged rows but only 1 rows already present. Its key comparer disagrees with the database about which keys are equal, so the merge lines for this category may name the wrong rows."), + "every merged row is also already present, so more merges than that is the comparer predicting wrongly"); + Assert.That(messages, Has.Member($"{MigrationCategoryIds.EndpointSettings}: 5 copied (0 changed), 0 already present, 3 merged, nothing skipped"), "never a negative count"); + }); + } + + [Test] + public void A_failed_category_that_skipped_nothing_still_counts_its_merged_and_changed_rows() + { + var logger = new CapturingLogger(); + var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { AlreadyPresentCount = 3, LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt, new Dictionary<string, (long Merged, long Changed)> { [MigrationCategoryIds.EndpointSettings] = (1, 2) }); + + Assert.That(logger.Entries.Single().Message, Does.StartWith($"{MigrationCategoryIds.EndpointSettings} is Failed (Halted) after 10 copied (2 changed), 2 already present and 1 merged.")); + } + + static readonly IReadOnlyDictionary<string, (long Merged, long Changed)> NothingMergedOrChanged = new Dictionary<string, (long Merged, long Changed)>(); + + static readonly DateTime RunStartedAt = new(2026, 9, 20, 12, 0, 0, DateTimeKind.Utc); + + static MigrationCheckpoint Settled(MigrationCategoryState state, long copied, long skipped, IReadOnlyDictionary<MigrationSkipReason, long>? skipReasons) => + new(MigrationCategoryIds.EndpointSettings, state, null, copied, skipped, null, skipReasons, null, null, null, null); + + static readonly MigrationCategory EndpointSettings = MigrationCategoryRegistry.Find(MigrationCategoryIds.EndpointSettings)!; + static readonly MigrationCategory KnownEndpoints = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + static readonly MigrationCategory MessageRedirects = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + static readonly MigrationCategory CustomChecks = MigrationCategoryRegistry.Find(MigrationCategoryIds.CustomChecks)!; + static readonly MigrationCategory UnresolvedAndRetryIssuedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + + const string UnknownCategoryId = "SomeCategoryFromANewerBuild"; + + static MigrationCheckpoint Checkpoint(MigrationCategoryState state, string categoryId = MigrationCategoryIds.EndpointSettings) => + new(categoryId, state, null, 0, 0, null, null, null, null, null, null); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); +} diff --git a/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs new file mode 100644 index 0000000000..f3aa068120 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Migration.Checks; + +[TestFixture] +class RetryHistoryDepthIsSafeCheckTests +{ + [Test] + public void Passes_at_the_default_depth() => + Assert.DoesNotThrowAsync(() => new RetryHistoryDepthIsSafeCheck(10).Run()); + + [TestCase(0)] + [TestCase(-1)] + public void Refuses_a_depth_that_empties_the_table(int depth) + { + var exception = Assert.ThrowsAsync<Exception>(() => new RetryHistoryDepthIsSafeCheck(depth).Run()); + + Assert.That(exception.Message, Does.Contain("RetryHistoryDepth").And.Contain("HistoricRetryOperations")); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs new file mode 100644 index 0000000000..17f483f0ba --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs @@ -0,0 +1,326 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// The copy can stop three ways that are not the engine's to report: the stall watchdog stopping one category, +// the host shutting down, and a second instance writing to the same database. +[TestFixture] +class StoppedCopyExplanationTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_stall_settles_the_stalled_category_failed_and_the_next_category_still_runs() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { HangOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.CategoryId), Is.EqualTo(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects })); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted), "a stalled category is Failed, so it waits for the operator rather than for the next start"); + Assert.That(results[0].SettledAt, Is.EqualTo(clock.GetUtcNow().UtcDateTime)); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes").And.Not.Contain("--migration-"), + "the row says what happened, and the refusal is what names the commands"); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0]), "the stall was reported but never saved, so the next start would copy it again"); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "one stalled category ended the whole start"); + Assert.That(target.WrittenRows(MigrationCategoryIds.MessageRedirects).Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_stall_in_a_call_that_ignores_its_token_still_settles_the_category_failed_without_waiting_for_the_call() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.CategoryId), Is.EqualTo(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects })); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].SettledAt, Is.EqualTo(clock.GetUtcNow().UtcDateTime)); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "one stuck call ended the whole start"); + } + + target.ReleaseIgnoredCall(); + + Assert.That(async () => await target.IgnoredCall, Throws.InstanceOf<MigrationCheckpointConflictException>(), + "the stuck call saves from the version it read before the stall, so the store must refuse it"); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0]), "a late commit from the stuck call reopened a settled category"); + } + finally + { + target.ReleaseIgnoredCall(); + } + } + + [Test] + public async Task A_stalled_call_that_commits_while_the_stall_is_being_settled_still_leaves_the_category_failed() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + // The stuck call commits after the settle has read the row and before it saves, so the settle's save is from an old version. + store.BeforeUpsert = async checkpoint => + { + if (checkpoint.State == MigrationCategoryState.Halted) + { + target.ReleaseIgnoredCall(); + await target.IgnoredCall; + } + }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].CopiedCount, Is.EqualTo(1), "the settle must keep the batch the stuck call committed"); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes")); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0])); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "a late commit from our own stuck call ended the whole start"); + } + } + finally + { + target.ReleaseIgnoredCall(); + } + } + + [Test] + public async Task A_host_stopping_during_a_call_that_ignores_its_token_propagates_without_waiting_for_the_call() + { + var store = new InMemoryMigrationCheckpointStore(); + using var host = new CancellationTokenSource(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { IgnoreTokenOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + var clock = new FakeTimeProvider(); + + try + { + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger(), host.Token)); + + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await host.CancelAsync(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(async () => await run.WaitAsync(TimeSpan.FromSeconds(10)), Throws.InstanceOf<OperationCanceledException>(), + "a shutdown waited for a call that ignores its token"); + Assert.That((await store.Read(MigrationCategoryIds.KnownEndpoints))!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown settled the category"); + Assert.That((await store.Read(MigrationCategoryIds.MessageRedirects))!.State, Is.EqualTo(MigrationCategoryState.NotStarted), "a stopping host started the next category"); + } + } + finally + { + target.ReleaseIgnoredCall(); + } + } + + [Test] + public async Task A_host_stopping_mid_category_leaves_it_copying_and_the_stop_propagates() + { + var store = new InMemoryMigrationCheckpointStore(); + using var host = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(store) { DefaultBatchSize = 2, StopOnCall = (2, host) }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1"), Row("KnownEndpoints-2"), Row("KnownEndpoints-3"), Row("KnownEndpoints-4")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + var clock = new FakeTimeProvider(); + + var run = MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger(), host.Token); + + using (Assert.EnterMultipleScope()) + { + Assert.That(async () => await run, Throws.InstanceOf<OperationCanceledException>(), "a shutdown is not the category's fault, so it must not come back as a settled row"); + + var stopped = await store.Read(MigrationCategoryIds.KnownEndpoints); + + Assert.That(stopped!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown settled Halted makes every restart refuse until the operator runs --migration-retry"); + Assert.That(stopped.Cursor, Is.EqualTo("KnownEndpoints-2"), "the next start resumes from the first batch's cursor"); + Assert.That(stopped.CopiedCount, Is.EqualTo(2)); + Assert.That(stopped.SettledAt, Is.Null); + Assert.That(stopped.LastError, Is.Null); + Assert.That((await store.Read(MigrationCategoryIds.MessageRedirects))!.State, Is.EqualTo(MigrationCategoryState.NotStarted), "a stopping host started the next category"); + Assert.That(target.RowsHandedToWrite(MigrationCategoryIds.MessageRedirects), Is.Empty); + } + } + + [Test] + public async Task A_stale_in_progress_row_the_start_has_not_reached_is_not_judged_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var twoHoursAgo = clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2); + await store.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, null, 0, 0, null, null, twoHoursAgo, twoHoursAgo, null, null)); + var target = new InMemoryMigrationTarget(store) + { + DefaultBatchSize = 1, + // Each KnownEndpoints batch takes five minutes, and the next one waits until the watchdog has looked at the time. + BeforeWrite = checkpoint => + { + if (checkpoint.CategoryId == MigrationCategoryIds.KnownEndpoints) + { + clock.Advance(TimeSpan.FromMinutes(5)); + store.WaitForAPollAtTheCurrentTime().GetAwaiter().GetResult(); + } + } + }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, [.. Enumerable.Range(1, 8).Select(index => Row($"KnownEndpoints-{index}"))]); + source.Seed(MigrationCategoryIds.EndpointSettings, Row("EndpointSettings-1")); + + var results = await Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings), store, clock, new CapturingLogger())) + .WaitAsync(TimeSpan.FromSeconds(30)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.State), Is.All.EqualTo(MigrationCategoryState.Complete), + "forty minutes copying KnownEndpoints were judged against a row the start had not reached yet"); + Assert.That(target.WrittenRows(MigrationCategoryIds.KnownEndpoints), Has.Count.EqualTo(8)); + Assert.That(target.WrittenRows(MigrationCategoryIds.EndpointSettings).Select(row => row.SourceId), Is.EqualTo(new[] { "EndpointSettings-1" })); + } + } + + [Test] + public void A_cancelled_copy_stays_a_cancellation() => + Assert.That(async () => await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromCanceled<IReadOnlyList<MigrationCheckpoint>>(new CancellationToken(canceled: true)), NewSettings()), + Throws.InstanceOf<OperationCanceledException>(), + "a shutdown is not a failure, so it must not come out as a refusal telling the operator to go looking at the source and the target"); + + [Test] + public void A_checkpoint_saved_by_another_instance_says_which_instance_to_stop() + { + var conflict = new MigrationCheckpointConflictException("Checkpoint KnownEndpoints was saved from version 3, but the stored row is at version 4."); + + var exception = Assert.ThrowsAsync<Exception>(async () => await MigrationStartup.CopyOrExplainWhyItStopped( + Task.FromException<IReadOnlyList<MigrationCheckpoint>>(conflict), + NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception!.Message, Does.Contain("a second ServiceControl pointed at the same SQLServer database"), + "the store's own message names a version, which tells nobody what is actually wrong"); + Assert.That(exception.Message, Does.Contain("Stop the other instance, then run --start-data-migration again"), + "this is the only refusal in the subsystem where doing nothing makes it worse"); + Assert.That(exception.Message, Does.Contain(conflict.Message).And.Contain("Nothing has opened on SQLServer yet")); + Assert.That(exception.InnerException, Is.SameAs(conflict)); + }); + } + + [Test] + public async Task A_copy_that_finished_is_returned_as_it_came_back() + { + IReadOnlyList<MigrationCheckpoint> copied = [new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, null, 3, 0, null, null, null, null, null, null)]; + + var finished = await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromResult(copied), NewSettings()); + + Assert.That(finished, Is.SameAs(copied)); + } + + static MigrationRow Row(string id) => new(id, new object(), new Dictionary<string, object?>()); + + static MigrationCategory[] Categories(params string[] categoryIds) => [.. categoryIds.Select(categoryId => MigrationCategoryRegistry.Find(categoryId)!)]; + + static MigrationEngine Engine(InMemoryMigrationSource source, InMemoryMigrationTarget target, IMigrationCheckpointStore store, TimeProvider clock) => + new(source, target, store, clock, new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger<MigrationEngine>.Instance); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); + + // Refuses a cancelled token as the EF Core store does, so a settle made on the stalled category's token fails here too. + sealed class CancellationHonouringCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore inner = new(); + + /// <summary> + /// Runs before each save that gets past the token check, with the checkpoint about to be saved. + /// </summary> + public Func<MigrationCheckpoint, Task>? BeforeUpsert { get; set; } + + public Task<IReadOnlyList<MigrationCheckpoint>> ReadAll(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.ReadAll(cancellationToken); + } + + public Task<MigrationCheckpoint?> Read(string categoryId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.Read(categoryId, cancellationToken); + } + + public async Task<MigrationCheckpoint> Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (BeforeUpsert is not null) + { + await BeforeUpsert(checkpoint); + } + + return await inner.Upsert(checkpoint, cancellationToken); + } + } +} diff --git a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs index 744fa7bf19..627fee7dee 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs @@ -190,12 +190,10 @@ public async Task A_remote_timeout_does_not_cut_the_slower_remotes_short() factory.Register(settings.RemoteInstances[3], Delayed(delay, Healthy("msg-4"))); var api = new TestApi(settings, factory, Local("local-msg")); - var started = DateTime.UtcNow; var result = await api.Execute(Context(), "/api/messages"); - Assert.That(DateTime.UtcNow - started, Is.GreaterThanOrEqualTo(delay), "the composite must wait for the remotes that are still answering"); - Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"])); + Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"]), "the delay lives in the fake, so these three cannot arrive unless the composite waited for the remotes that are still answering"); Assert.That(result.IncompleteInstances, Is.EqualTo([new IncompleteInstance(settings.RemoteInstances[0].InstanceId, QueryFailure.TimedOut)])); Assert.That(result.QueryStats.TotalCount, Is.EqualTo(4)); } diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index e9a507b944..669218a7af 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -55,6 +55,7 @@ <Project Path="ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj" /> <Project Path="ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj" /> <Project Path="ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj" /> + <Project Path="ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj" /> <Project Path="ServiceControl.Persistence.Tests.InMemory/ServiceControl.Persistence.Tests.InMemory.csproj" /> <Project Path="ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj" /> <Project Path="ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj" /> diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index c294001cf3..83f0bb1afc 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -107,6 +107,7 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S services.AddStartupChecks(); services.AddPersistence(settings); + services.AddMetrics(settings.PrintMetrics); hostBuilder.AddTelemetry(settings); services.AddServiceControlHealthChecks(); diff --git a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs index 00d38bb0b3..bc804646db 100644 --- a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Hosting.Commands using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -13,8 +14,10 @@ namespace ServiceControl.Hosting.Commands using ServiceControl.ExternalIntegrations; using ServiceControl.Hosting.Https; using ServiceControl.Infrastructure.Health; + using ServiceControl.Migration; using ServiceControl.Monitoring; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Recoverability; /// <summary> @@ -25,15 +28,13 @@ namespace ServiceControl.Hosting.Commands /// </summary> class ErrorIngestionOnlyCommand : AbstractCommand { - static readonly string[] SupportedStorageNames = ["SQLServer", "PostgreSQL"]; - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) { EnsureStorageCanScaleOut(settings); var app = BuildHost(settings); - await app.RunAsync(settings.RootUrl); + await app.RunAsync(); } internal static WebApplication BuildHost(Settings settings, Action<WebApplicationBuilder> customize = null) @@ -47,12 +48,24 @@ internal static WebApplication BuildHost(Settings settings, Action<WebApplicatio hostBuilder.AddServiceControlHttps(settings.HttpsSettings); hostBuilder.AddServiceControl(settings, configuration: null, Components); + hostBuilder.Services.AddHostedService(provider => + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService<IMigrationCheckpointStore>(), + "this error ingestion only host")); + + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + customize?.Invoke(hostBuilder); var app = hostBuilder.Build(); app.MapServiceControlHealthChecks(); + // Set here rather than passed to RunAsync, so a caller that starts the host itself gets the + // configured address instead of Kestrel's default port. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + return app; } @@ -60,7 +73,7 @@ static void EnsureStorageCanScaleOut(Settings settings) { var manifest = PersistenceManifestLibrary.Find(settings.PersistenceType); - if (manifest == null || !SupportedStorageNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) + if (manifest == null || !PersistenceFactory.SqlPersistenceNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) { throw new Exception( $"--error-ingestion-only requires SQL Server or PostgreSQL storage, but this instance is configured to use '{settings.PersistenceType}'. Scaling out error ingestion is not supported for this storage type."); diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index d32da70fb1..6dddcd2645 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -17,6 +17,8 @@ using Recoverability; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; + using ServiceControl.Migration; + using ServiceControl.Persistence.DataMigration; class ImportFailedErrorsCommand : AbstractCommand { @@ -62,6 +64,16 @@ internal static IHost BuildHost(Settings settings) hostBuilder.Services.AddSingleton<IEndpointInstanceMonitoring, EndpointInstanceMonitoring>(); hostBuilder.Services.AddErrorMessageEnricher<DetectNewEndpointsFromErrorImportsEnricher>(); + // Importing writes failed messages into the database, so on SQL it waits for an unfinished copy as an ingestion-only worker does. + if (PersistenceFactory.SqlPersistenceNames.Contains(PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name, StringComparer.OrdinalIgnoreCase)) + { + hostBuilder.Services.AddHostedService(provider => + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService<IMigrationCheckpointStore>(), + "--import-failed-errors")); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + } + return hostBuilder.Build(); } } diff --git a/src/ServiceControl/Hosting/Commands/MigrateCommand.cs b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs new file mode 100644 index 0000000000..8681aa14e9 --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrateCommand.cs @@ -0,0 +1,65 @@ +namespace ServiceControl.Hosting.Commands; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Migration; +using ServiceControl.Persistence; + +class MigrateCommand : AbstractCommand +{ + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + ConsoleCancelEventHandler cancel = (_, e) => + { + e.Cancel = true; + cancellation.Cancel(); + }; + Console.CancelKeyPress += cancel; + var migrationToken = cancellation.Token; + + try + { + Console.Out.WriteLine("Stop ServiceControl and all ingestion workers before running the offline migration."); + await Run(settings, cancellationToken: migrationToken); + Console.Out.WriteLine("The offline migration completed. Start the ServiceControl service or container to complete the online migration."); + } + catch (OperationCanceledException) when (migrationToken.IsCancellationRequested) + { + Console.Out.WriteLine("The offline migration stopped. Run --start-data-migration again to resume from the last committed batch."); + Environment.ExitCode = 1; + } + finally + { + Console.CancelKeyPress -= cancel; + } + } + + internal static async Task Run(Settings settings, Action<WebApplicationBuilder> customize = null, CancellationToken cancellationToken = default) + { + var builder = WebApplication.CreateBuilder(); + // Persistence also registers runtime-only services whose dependencies this command does not need. + builder.Host.UseDefaultServiceProvider(options => options.ValidateOnBuild = false); + builder.Logging.ClearProviders(); + builder.Logging.ConfigureLogging(settings.LoggingSettings.LogLevel); + builder.Services.AddSingleton(TimeProvider.System); + builder.Services.AddPersistence(settings); + customize?.Invoke(builder); + +#if DEBUG + RunCommand.AllowUnreleasedMigrationWhenAsked(builder.Services, Environment.GetEnvironmentVariable(RunCommand.AllowUnreleasedMigrationVariable)); +#endif + + // Never start this host: only the copier may write during the offline phase. + await using var app = builder.Build(); + await MigrationStartup.RunRequiredCopy(app.Services, settings, cancellationToken); + } +} diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs new file mode 100644 index 0000000000..40d832e124 --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Hosting.Commands +{ + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence; + + class MigrationSourceReportCommand : AbstractCommand + { + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + try + { + await Report(settings, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + await Console.Error.WriteLineAsync(e.Message); + Environment.ExitCode = 1; + } + } + + static async Task Report(Settings settings, CancellationToken cancellationToken) + { + await using var source = await PersistenceFactory.OpenMigrationSource(settings, cancellationToken); + + var description = await source.Describe(cancellationToken); + + Console.Out.WriteLine("ServiceControl migration source report"); + Console.Out.WriteLine(); + Console.Out.WriteLine($"{"Source persistence",-20}: {PersistenceFactory.MigrationSourcePersistenceType}"); + Console.Out.WriteLine($"{"Version",-20}: {description.Version}"); + + foreach (var fact in description.Facts) + { + var origin = fact.SettingKey is null ? string.Empty : $" (from {fact.SettingKey})"; + Console.Out.WriteLine($"{fact.Label,-20}: {fact.Value}{origin}"); + } + + foreach (var scope in (await source.Inventory(cancellationToken)).GroupBy(entry => entry.Scope)) + { + Console.Out.WriteLine(); + Console.Out.WriteLine($"{scope.Key}:"); + + foreach (var entry in scope.OrderBy(entry => entry.Name, StringComparer.Ordinal)) + { + Console.Out.WriteLine($" {entry.Name,-42}{entry.Count,12:N0}"); + } + } + } + } +} diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs index 33a6361511..0b695e5b94 100644 --- a/src/ServiceControl/Hosting/Commands/RunCommand.cs +++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs @@ -1,9 +1,12 @@ namespace ServiceControl.Hosting.Commands { + using System; using System.Threading; using System.Threading.Tasks; using Infrastructure.WebApi; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -11,11 +14,40 @@ using ServiceControl; using ServiceControl.Hosting.Auth; using ServiceControl.Hosting.Https; + using ServiceControl.Migration; +#if DEBUG + using Microsoft.Extensions.Logging; + using ServiceControl.Infrastructure; + using ServiceControl.Migration.Checks; +#endif using ServicePulse; class RunCommand : AbstractCommand { - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) +#if DEBUG + internal const string AllowUnreleasedMigrationVariable = "SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED"; + + /// <summary> + /// When <paramref name="variableValue"/> is "true" in any letter case, registers <see cref="AllowUnreleasedMigration"/> + /// so a Debug build can copy past the release check for a developer's smoke test, and logs a warning so the bypass shows in the log. + /// </summary> + internal static void AllowUnreleasedMigrationWhenAsked(IServiceCollection services, string variableValue) + { + if (string.Equals(variableValue, "true", StringComparison.OrdinalIgnoreCase)) + { + services.AddSingleton<AllowUnreleasedMigration>(); + LoggerUtil.CreateStaticLogger<RunCommand>().LogWarning("{Variable} is true, so this Debug build skips the check that the build carries the whole migration and will copy for real", AllowUnreleasedMigrationVariable); + } + } +#endif + + public override Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) => + Run(settings, customize: null, cancellationToken); + + /// <summary> + /// Builds and runs the full instance after checking that any offline migration has finished. + /// </summary> + internal static async Task Run(Settings settings, Action<WebApplicationBuilder> customize, CancellationToken cancellationToken = default) { var endpointConfiguration = new EndpointConfiguration(settings.InstanceName); var assemblyScanner = endpointConfiguration.AssemblyScanner(); @@ -31,7 +63,19 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel hostBuilder.AddServiceControl(settings, endpointConfiguration); hostBuilder.AddServiceControlApi(settings.CorsSettings); - var app = hostBuilder.Build(); + customize?.Invoke(hostBuilder); + +#if DEBUG + AllowUnreleasedMigrationWhenAsked(hostBuilder.Services, Environment.GetEnvironmentVariable(AllowUnreleasedMigrationVariable)); +#endif + + hostBuilder.Services.AddHostedService(provider => new MigrationBeforeTheHostOpens(provider)); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + hostBuilder.Services.AddHostedService<OptionalCategoryCopier>(); + + // A start that refuses would otherwise leave everything the host built undisposed. + await using var app = hostBuilder.Build(); + app.UseServiceControl(settings.ForwardedHeadersSettings, settings.HttpsSettings); if (settings.EnableIntegratedServicePulse) { @@ -39,7 +83,11 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel } app.UseServiceControlAuthentication(settings.OpenIdConnectSettings.Enabled); - await app.RunAsync(settings.RootUrl); + // WebApplication's RunAsync(url) takes no cancellation token, so set the url here and call IHost's RunAsync below. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + + await app.RunAsync(cancellationToken); } } } diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index d616180539..633fe38180 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -23,6 +23,16 @@ This mode runs no setup, so combining it with --setup or --setup-and-run is refu Message bodies must be stored somewhere every host can read, so this mode should not be combined with file system body storage unless the path is a shared mount. +MIGRATION + + ServiceControl.exe --start-data-migration + +Copies required data from RavenDB into the configured SQL Server or PostgreSQL database. +Stop ServiceControl and all ingestion workers first. The command never starts the service. +After a successful offline copy, start ServiceControl to copy optional data in the background. +Run --start-data-migration again after an interruption to resume from saved progress. The target database records the migration state. +This build still blocks copying until the complete migration is released. + SERVICE INSTALL AND UNINSTALL AND CONFIGURATION OPTIONS As of Service Control 1.7 the command line uninstall and install switches have been removed. diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index 88fe15a162..7e4571f447 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -62,6 +62,24 @@ public HostArguments(string[] args) } }; + var migrationOptions = new OptionSet + { + { + "start-data-migration", + "Copy required migration data with ServiceControl stopped", + s => Command = typeof(MigrateCommand) + } + }; + + var migrationSourceReportOptions = new OptionSet + { + { + "migration-source-report", + "Report what a migration would read from the source persistence", + s => Command = typeof(MigrationSourceReportCommand) + } + }; + try { // Parsed before setup returns, so setup can refuse to provision an ingestion-only worker. @@ -88,12 +106,30 @@ public HostArguments(string[] args) return; } + migrationOptions.Parse(args); + + if (Command == typeof(MigrateCommand)) + { + if (ErrorIngestionOnly) + { + throw new Exception("--start-data-migration cannot run with --error-ingestion-only. Stop every instance before running the offline migration."); + } + return; + } + if (ErrorIngestionOnly) { Command = typeof(ErrorIngestionOnlyCommand); return; } + migrationSourceReportOptions.Parse(args); + + if (Command == typeof(MigrationSourceReportCommand)) + { + return; + } + defaultOptions.Parse(args); } catch (Exception e) diff --git a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs new file mode 100644 index 0000000000..16831e3add --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Registered by a test host, or by a Debug build whose SERVICECONTROL_MIGRATION_ALLOW_UNRELEASED environment +/// variable is true, to let a copy run on a build that does not yet carry the whole migration. A Release build +/// has no way to register it. +/// </summary> +class AllowUnreleasedMigration; + +/// <summary> +/// Refuses every copy until the whole migration has shipped. A build that can copy the required categories but +/// not yet the background copy, the end-of-migration guard or verification would still commit an instance to the +/// target with no way to finish or check the move. The last phase of the migration deletes this check. +/// </summary> +class MigrationIsReleasedCheck(AllowUnreleasedMigration allowUnreleasedMigration = null) : IMigrationStartupCheck +{ + public string Name => "this build carries the whole migration"; + + public Task Run(CancellationToken cancellationToken = default) + { + if (allowUnreleasedMigration is null) + { + throw new Exception( + $"This build of ServiceControl does not yet carry the whole migration from RavenDB, so it will not copy anything. --start-data-migration is not available in this build. Keep the instance on RavenDB."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs new file mode 100644 index 0000000000..fbdfef0d18 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Refuses any target but SQL Server or PostgreSQL, since the source is always RavenDB. The seams underneath are +/// general enough to copy between any two persisters, and this is the check that says which pair is actually supported. +/// </summary> +class MigrationPairIsSupportedCheck(Settings settings) : IMigrationStartupCheck +{ + public string Name => "the source and target are the supported pair"; + + public Task Run(CancellationToken cancellationToken = default) + { + var targetName = PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name; + + if (!PersistenceFactory.SqlPersistenceNames.Contains(targetName, StringComparer.OrdinalIgnoreCase)) + { + throw new Exception( + $"Migrating from '{PersistenceFactory.MigrationSourcePersistenceType}' to '{settings.PersistenceType}' is not supported. The only supported migration is from RavenDB to SQL Server or PostgreSQL, so set {Settings.SettingsRootNamespace}/PersistenceType to {string.Join(" or ", PersistenceFactory.SqlPersistenceNames)} before running --start-data-migration."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs new file mode 100644 index 0000000000..d8f2ef87bc --- /dev/null +++ b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Reads the engine options out of the settings, which refuses an optional category window that is not a time +/// span before anything is copied rather than part way through the copy. +/// </summary> +class OptionalCategoryWindowsAreValidCheck(TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) : IMigrationStartupCheck +{ + public string Name => "the optional category windows are valid"; + + /// <summary> + /// The options parsed from the settings, which is null until <see cref="Run"/> has returned. + /// </summary> + public MigrationEngineOptions Options { get; private set; } + + public Task Run(CancellationToken cancellationToken = default) + { + Options = MigrationEngineOptions.FromSettings(Settings.SettingsRootNamespace, eventRetentionPeriod, errorRetentionPeriod); + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs new file mode 100644 index 0000000000..8372211f84 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Refuses a copy that the instance's own retry history setting would throw away. At a depth of zero the SQL +/// persisters empty the history table whenever a retry completes, and the migrated rows would go with it. +/// </summary> +/// <param name="retryHistoryDepth">The instance's ServiceControl/RetryHistoryDepth.</param> +class RetryHistoryDepthIsSafeCheck(int retryHistoryDepth) : IMigrationStartupCheck +{ + public string Name => "the retry history depth will not empty a migrated table"; + + public Task Run(CancellationToken cancellationToken = default) + { + // The depth at which RetryHistoryDataStore.TrimHistory deletes every row rather than trimming. + if (retryHistoryDepth <= 0) + { + throw new Exception( + $"ServiceControl/RetryHistoryDepth is {retryHistoryDepth}, and at that depth the SQL persisters delete every row of HistoricRetryOperations when a retry completes. The RetryOperations category copies that table, so the first completed retry after cutover would silently discard the migrated retry history. Set ServiceControl/RetryHistoryDepth to a positive number, or clear it to use the default of 10, before running --start-data-migration."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs new file mode 100644 index 0000000000..90f4f6da59 --- /dev/null +++ b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs @@ -0,0 +1,54 @@ +namespace ServiceControl.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Refuses to start a host that writes failed messages into the database, including the main instance, +/// an error ingestion only host, or --import-failed-errors, until every required category is Done or Abandoned. An optional category, which copies +/// in the background once the main host has opened, never holds it back, but a category under an id this build +/// does not know does. A refusal throws, which fails the start and stops the host. Start such a host only after the +/// main host has opened, because a database with no checkpoint row yet reads as one no migration has touched. +/// </summary> +/// <param name="checkpointStore">Where the checkpoint rows are read. A database no migration has touched holds none, so the host starts as before.</param> +/// <param name="hostDescription">Names the host in the refusal, such as "this error ingestion only host".</param> +sealed class FinishedCopyBeforeAnIngestionNodeOpens( + IMigrationCheckpointStore checkpointStore, + string hostDescription) : IHostedLifecycleService +{ + // Every writer checks persisted progress, because ingesting into a partial copy ends the free abort. + public async Task StartingAsync(CancellationToken cancellationToken = default) + { + var unfinished = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => !checkpoint.State.IsFinished()) + .ToArray(); + + if (unfinished.Length == 0) + { + return; + } + + var detail = string.Join(". ", unfinished.Select(checkpoint => checkpoint.State.IsFailed() + ? MigrationStartup.FailedDetail(checkpoint) + : $"{checkpoint.CategoryId} is Copying ({checkpoint.State})")); + + const string advice = "With ServiceControl and all ingestion workers stopped, resolve any Failed categories and run --start-data-migration again before starting this host."; + + throw new Exception($"A copy into this database has not finished, so {hostDescription} will not start and nothing has been lost. {detail}. {advice}"); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs b/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs new file mode 100644 index 0000000000..b259b2ddcd --- /dev/null +++ b/src/ServiceControl/Migration/MigrationBeforeTheHostOpens.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +sealed class MigrationBeforeTheHostOpens(IServiceProvider services) : IHostedLifecycleService +{ + public async Task StartingAsync(CancellationToken cancellationToken = default) + { + var checkpoints = services.GetService<IMigrationCheckpointStore>(); + if (checkpoints is null) + { + return; + } + + await new FinishedCopyBeforeAnIngestionNodeOpens(checkpoints, "ServiceControl").StartingAsync(cancellationToken); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs new file mode 100644 index 0000000000..9c0626bd7e --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -0,0 +1,608 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// The required copy, from the checks that decide whether it can run at all to the refusal it throws when a +/// category did not finish. Everything here happens with ServiceControl closed, which is the only window in +/// which the copy can be thrown away at no cost. +/// </summary> +static class MigrationStartup +{ + // A category needs both a reader and a writer, so a half-implemented one is never attempted. + internal static IReadOnlyCollection<string> CopyableCategoryIds(IReadOnlyCollection<string> sourceSupports, IReadOnlyCollection<string> targetSupports) => + [.. sourceSupports.Intersect(targetSupports, StringComparer.Ordinal)]; + + /// <summary> + /// Runs the checks and required copy without starting ServiceControl. A completed required copy can succeed + /// without its source; optional checkpoints retain the outage for the online phase. + /// Throws if a check refuses or a required category does not finish. The caller must not start the host. + /// </summary> + public static async Task RunRequiredCopy(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + await MigrationStartupCheckRunner.Run( + [ + new MigrationIsReleasedCheck(services.GetService<AllowUnreleasedMigration>()), + new MigrationPairIsSupportedCheck(settings) + ], cancellationToken); + + var loggerFactory = services.GetRequiredService<ILoggerFactory>(); + var logger = loggerFactory.CreateLogger(typeof(MigrationStartup)); + var target = services.GetRequiredService<IMigrationTarget>(); + var checkpointStore = services.GetRequiredService<IMigrationCheckpointStore>(); + var timeProvider = services.GetRequiredService<TimeProvider>(); + + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var copyable = CopyableCategoryIds(source.SupportedCategoryIds, target.SupportedCategoryIds); + + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + var engine = new MigrationEngine( + source, + target, + checkpointStore, + timeProvider, + options, + loggerFactory.CreateLogger<MigrationEngine>()); + + var toCopy = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => copyable.Contains(category.Id)) + .ToArray(); + + var deferred = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => !copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + var checkpoints = await checkpointStore.ReadAll(cancellationToken); + var unfinishedOutsideTheCopy = UnfinishedRowsOutsideTheCopy(checkpoints, toCopy); + var requiredCopySettled = RequiredCopyIsSettled(checkpoints, toCopy); + + try + { + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Safe to open without the source, because every required category is settled and only the background copy of the optional ones reads it. + catch (Exception exception) when (requiredCopySettled) + { + // The runner's wrapper describes a refused run, which this is not, and it would land on the rows. + var cause = exception.InnerException ?? exception; + + logger.LogError(cause, + "The {SourcePersistenceType} migration source could not be opened. Every required category this build copies is Done or Abandoned and no unknown required row is unfinished, so the offline command can complete. Outstanding optional checkpoints carry the error; start ServiceControl to continue the online migration.", + PersistenceFactory.MigrationSourcePersistenceType); + + await RecordSourceOutage(checkpointStore, engine.SelectCategories(MigrationCategoryKind.Optional), cause, cancellationToken); + return; + } + catch (Exception exception) + { + throw new Exception($"{exception.Message} {NoWayOutWithoutTheSource(settings)}", exception.InnerException); + } + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + logger.LogInformation( + "Offline migration: copying {CopyCount} required categories with ServiceControl stopped ({DeferredCount} not yet implemented: {Deferred})", + toCopy.Length, deferred.Length, string.Join(", ", deferred)); + + // Seed both phases before copying so startup can detect a copy interrupted between categories. + foreach (var category in toCopy.Concat(engine.SelectCategories(MigrationCategoryKind.Optional))) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null), cancellationToken); + } + } + + // Captured before the copy so the report can tell a category this run finished from one an earlier run did. + var runStartedAt = timeProvider.GetUtcNow().UtcDateTime; + + var finished = await CopyOrExplainWhyItStopped( + RunRequiredCategories(engine, toCopy, checkpointStore, timeProvider, logger, cancellationToken), + settings); + + ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt, engine.MergedAndChangedThisRun); + + RefuseIfAnyCategoryDidNotComplete(toCopy, finished, unfinishedOutsideTheCopy, settings); + } + + /// <summary> + /// Runs every startup check in the order they have to run, and opens the target and the source as two of + /// them. The order is what the operator sees: a check that costs nothing comes before one that connects to a + /// database, and the source's own checks run last because they need it open. + /// </summary> + public static async Task<MigrationEngineOptions> RunChecksAndOpen(IServiceProvider services, Settings settings, IMigrationSource source, CancellationToken cancellationToken = default) + { + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + return options; + } + + /// <summary> + /// Runs every startup check that does not need the source, in the order they have to run, ending with + /// opening the target. A check that costs nothing comes before one that connects to a database. The target + /// is open when this returns, and the source has not been touched. + /// </summary> + public static async Task<MigrationEngineOptions> RunChecksAndOpenTarget(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + var target = services.GetRequiredService<IMigrationTarget>(); + var readiness = services.GetRequiredService<IMigrationTargetReadiness>(); + var categories = new OptionalCategoryWindowsAreValidCheck(settings.EventsRetentionPeriod, settings.ErrorRetentionPeriod); + + await MigrationStartupCheckRunner.Run( + [ + categories, + new RetryHistoryDepthIsSafeCheck(settings.RetryHistoryDepth), + .. readiness.ContributedChecks(), + new Step("the migration target opens", target.Open) + ], cancellationToken); + + return categories.Options; + } + + /// <summary> + /// Records on each optional category's checkpoint that the source could not be opened, so status and verify + /// read the outage from the rows. The required copy calls it on a start that + /// opens without the source, and the background copier calls it when its own open fails. It changes no + /// state, cursor or count: a row still copying gains the error as its <see cref="MigrationCheckpoint.LastError" />, + /// a category with no row gets a new not-started row carrying the error, and a row that is Done, Failed or + /// Abandoned is left alone. The next start that resumes a row clears the error. + /// </summary> + internal static async Task RecordSourceOutage(IMigrationCheckpointStore checkpointStore, IReadOnlyList<MigrationCategory> optionalCategories, Exception exception, CancellationToken cancellationToken = default) + { + var error = $"The {PersistenceFactory.MigrationSourcePersistenceType} migration source could not be opened: {exception.GetType().Name}: {exception.Message.TrimEnd('.', ' ')}. This category stays as it is, and the next start tries the source again."; + + foreach (var category in optionalCategories) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken); + + if (checkpoint is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, error), cancellationToken); + } + else if (!checkpoint.State.IsFinished() && !checkpoint.State.IsFailed()) + { + await checkpointStore.Upsert(checkpoint with { LastError = error }, cancellationToken); + } + } + } + + /// <summary> + /// Copies the required categories one after another under the stall watchdog and returns where each one ended, + /// in the same order. A category that commits nothing for <see cref="ClosedWindowProgress.StallLimit" /> is + /// stopped and settled Halted, and the categories after it still get their go. Before copying anything it saves + /// a not-started checkpoint for every category that has none, as <see cref="MigrationEngine.RunCategories" /> does. + /// </summary> + internal static async Task<IReadOnlyList<MigrationCheckpoint>> RunRequiredCategories( + MigrationEngine engine, + IReadOnlyList<MigrationCategory> categories, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + ILogger logger, + CancellationToken cancellationToken = default) + { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null), cancellationToken); + } + } + + await using var progress = new ClosedWindowProgress(checkpointStore, timeProvider, logger, [.. categories.Select(category => category.Id)], cancellationToken); + + var results = new List<MigrationCheckpoint>(categories.Count); + + foreach (var category in categories) + { + using var categoryCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + progress.Watch(category.Id, timeProvider.GetUtcNow().UtcDateTime, categoryCancellation); + var run = engine.RunCategoryAsync(category, categoryCancellation.Token); + + try + { + // Ends the wait on a stall or a host stop even when a call underneath ignores its token. That call keeps running, and the version check refuses its late commit or the settle below re-reads past it. + results.Add(await run.WaitAsync(categoryCancellation.Token)); + } + // Only the watchdog cancels this source without the host's token, so the filter tells a stall from a shutdown. +#pragma warning disable PS0020 + catch (OperationCanceledException) when (categoryCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) +#pragma warning restore PS0020 + { + MigrationCheckpoint settled; + + try + { + settled = await SettleStalled(checkpointStore, category.Id, timeProvider, cancellationToken); + } + // The stalled call can commit between the settle's read and its save, so one re-read settles from that commit. A second conflict is another instance, and it propagates. + catch (MigrationCheckpointConflictException) + { + settled = await SettleStalled(checkpointStore, category.Id, timeProvider, cancellationToken); + } + + results.Add(settled); + + _ = run.ContinueWith( + ended => logger.LogWarning(ended.Exception.GetBaseException(), + "{CategoryId}: the call the stall stopped waiting for ended after the category was settled Halted, with {ExceptionType}. It changed nothing after the settle, because the checkpoint store refuses a save made from before it.", + category.Id, ended.Exception.GetBaseException().GetType().Name), + cancellationToken, TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); + } + } + + return results; + } + + // Takes the host's token, because the stall has already cancelled the category's and a store call on that would fail. + static async Task<MigrationCheckpoint> SettleStalled(IMigrationCheckpointStore checkpointStore, string categoryId, TimeProvider timeProvider, CancellationToken cancellationToken) + { + var stalled = await checkpointStore.Read(categoryId, cancellationToken); + + return await checkpointStore.Upsert(stalled with + { + State = MigrationCategoryState.Halted, + SettledAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = StallExplanation(categoryId) + }, cancellationToken); + } + + /// <summary> + /// Waits for the copy and turns a second instance writing checkpoints to the same database into a refusal the + /// operator can act on. A shutdown and every other failure come out as they are. + /// </summary> +#pragma warning disable PS0018 // The copy it waits on already runs under the host's token, so a token here would have nothing to cancel. + internal static async Task<IReadOnlyList<MigrationCheckpoint>> CopyOrExplainWhyItStopped( + Task<IReadOnlyList<MigrationCheckpoint>> copy, + Settings settings) +#pragma warning restore PS0018 + { + try + { + return await copy; + } + // The engine never turns a conflict into a halt, so without this the copy ends on the checkpoint store's + // own message and none of the advice every other refusal carries. + catch (MigrationCheckpointConflictException exception) + { + throw new Exception( + $"The required copy stopped because another writer saved a migration checkpoint for this instance, which is a second ServiceControl pointed at the same {settings.PersistenceType} database. The offline migration stopped. {exception.Message} Stop the other instance, then run --start-data-migration again; the copy resumes from its last committed batch. " + + RollbackAdvice(settings), exception); + } + } + + /// <summary> + /// Throws unless every attempted category finished. A category that reported no checkpoint at all counts as + /// outstanding too, because nothing says how much of it was copied. + /// </summary> + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList<MigrationCategory> attempted, + IReadOnlyList<MigrationCheckpoint> finished, + Settings settings) + { + var outstanding = finished + .Where(checkpoint => !checkpoint.State.IsFinished()) + .Select(checkpoint => checkpoint.State.IsFailed() ? FailedDetail(checkpoint) : CopyingDetail(checkpoint)) + .ToList(); + + var reported = finished.Select(checkpoint => checkpoint.CategoryId).ToHashSet(StringComparer.Ordinal); + + outstanding.AddRange(attempted + .Where(category => !reported.Contains(category.Id)) + .Select(category => $"{category.Id} reported no checkpoint at all, so whether it copied anything is unknown")); + + if (outstanding.Count == 0) + { + return; + } + + throw new Exception( + $"The required copy did not finish, so ServiceControl cannot start yet, and nothing has been lost. {string.Join(". ", outstanding)}. " + + RollbackAdvice(settings)); + } + + /// <summary> + /// Says whether the host may open without the source: every category this start copies is finished, and no row + /// outside the copy is unfinished. + /// </summary> + internal static bool RequiredCopyIsSettled(IReadOnlyList<MigrationCheckpoint> checkpoints, IReadOnlyList<MigrationCategory> toCopy) => + UnfinishedRowsOutsideTheCopy(checkpoints, toCopy).Count == 0 + && toCopy.All(category => checkpoints.Any(checkpoint => checkpoint.CategoryId == category.Id && checkpoint.State.IsFinished())); + + /// <summary> + /// Throws unless every attempted category finished and no row outside the copy is unfinished, naming each + /// outstanding row in the same words. + /// </summary> + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList<MigrationCategory> attempted, + IReadOnlyList<MigrationCheckpoint> finished, + IReadOnlyList<MigrationCheckpoint> unfinishedOutsideTheCopy, + Settings settings) => + RefuseIfAnyCategoryDidNotComplete(attempted, [.. finished, .. unfinishedOutsideTheCopy], settings); + + /// <summary> + /// Finds the unfinished rows the copy will not run: a row under an id this build does not know, which counts as + /// required because a newer build may have written it for a category that must finish, and a required row for a + /// category this build cannot copy yet. A finished row, and a row known to be optional, hold nothing. + /// </summary> + internal static IReadOnlyList<MigrationCheckpoint> UnfinishedRowsOutsideTheCopy(IReadOnlyList<MigrationCheckpoint> checkpoints, IReadOnlyList<MigrationCategory> toCopy) => + [.. checkpoints + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => toCopy.All(category => category.Id != checkpoint.CategoryId)) + .Where(checkpoint => !checkpoint.State.IsFinished())]; + + internal static string FailedDetail(MigrationCheckpoint checkpoint) + { + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? $" ({string.Join("; ", counts.Select(reason => $"{reason.Key} {reason.Value}{(reason.Key.IsPermanent() ? ", no retry can fix" : reason.Key.IsBenign() ? ", harmless" : "")}"))})" + : ""; + + return $"{checkpoint.CategoryId} is Failed ({checkpoint.State}) after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{reasons}{LastErrorClause(checkpoint)}; " + + $"run --migration-retry {checkpoint.CategoryId} once the cause is fixed, which copies the category again from the start, or --migration-abandon {checkpoint.CategoryId} to keep what was copied and give up the rest, both with ServiceControl stopped"; + } + + static string CopyingDetail(MigrationCheckpoint checkpoint) => + $"{checkpoint.CategoryId} is {checkpoint.State} after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{LastErrorClause(checkpoint)}"; + + // Trimmed because the refusal punctuates each category's sentence itself. + static string LastErrorClause(MigrationCheckpoint checkpoint) => + checkpoint.LastError is null ? "" : $": {checkpoint.LastError.TrimEnd('.', ' ')}"; + + /// <summary> + /// Logs one line per category saying what it copied and what it left behind. Skipped rows are warned about + /// one at a time while the copy runs, and nothing else states the total or says what becomes of them. + /// The merged and changed counts are this run's, by category id, and the line counts merged rows apart from the other rows already present. + /// A category that merged or changed a row is reported as a warning even when it skipped nothing. + /// A category with more merged rows than rows already present gets a warning naming both counts, because the target's key comparer then disagrees with the database. + /// </summary> + internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList<MigrationCheckpoint> finished, ILogger logger, DateTime runStartedAt, IReadOnlyDictionary<string, (long Merged, long Changed)> mergedAndChanged) + { + foreach (var checkpoint in finished) + { + // It did not run, and the refusal that follows names it with the category it waits for. + if (checkpoint.State == MigrationCategoryState.Blocked) + { + continue; + } + + // A category already finished when this run began is returned without being run, so its counts are + // an earlier run's. A Failed one is returned untouched too, but it is not finished, so it falls through. + if (checkpoint.State.IsFinished() && checkpoint.SettledAt is { } settledAt && settledAt < runStartedAt) + { + logger.LogInformation( + "{CategoryId}: already finished before this start, by a run that copied {Copied} and skipped {Skipped}. This start copied nothing.", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.SkippedCount); + continue; + } + + var (merged, changed) = mergedAndChanged.GetValueOrDefault(checkpoint.CategoryId); + + // Every merged row is also counted as already present, so more merges than that means the merge lines were predicted wrongly. + if (merged > checkpoint.AlreadyPresentCount) + { + logger.LogWarning("{CategoryId}: the target reported {Merged} merged rows but only {AlreadyPresent} rows already present. Its key comparer disagrees with the database about which keys are equal, so the merge lines for this category may name the wrong rows.", + checkpoint.CategoryId, merged, checkpoint.AlreadyPresentCount); + } + + var alreadyPresent = Math.Max(0, checkpoint.AlreadyPresentCount - merged); + + // An exception, a stall or unbalanced counts fail a category without a skip, and that is not a clean copy. + if (checkpoint.State.IsFailed() && checkpoint.SkippedCount == 0) + { + logger.LogWarning("{CategoryId} is Failed ({State}) after {Copied} copied ({Changed} changed), {AlreadyPresent} already present and {Merged} merged. The refusal that follows says why and what to run.", + checkpoint.CategoryId, checkpoint.State, checkpoint.CopiedCount, changed, alreadyPresent, merged); + continue; + } + + if (checkpoint.SkippedCount == 0) + { + logger.Log(merged + changed == 0 ? LogLevel.Information : LogLevel.Warning, "{CategoryId}: {Copied} copied ({Changed} changed), {AlreadyPresent} already present, {Merged} merged, nothing skipped", + checkpoint.CategoryId, checkpoint.CopiedCount, changed, alreadyPresent, merged); + continue; + } + + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? string.Join(", ", counts.OrderByDescending(reason => reason.Value).Select(reason => $"{reason.Key} {reason.Value}")) + : "no reason recorded"; + + // Only a Failed category can be retried, so a Done one is never offered the command. + var whatBecomesOfThem = checkpoint.State.IsFailed() + ? WhatARetryCanDo(checkpoint) + : checkpoint.State == MigrationCategoryState.Complete + ? string.Join(" ", ["They were left out as harmless.", .. HarmlessReasonsIn(checkpoint), "They stay only in the source database."]) + : "They stay only in the source database."; + + logger.LogWarning( + "{CategoryId}: {Copied} copied ({Changed} changed), {AlreadyPresent} already present, {Merged} merged, {Skipped} skipped ({Reasons}). {WhatBecomesOfThem}", + checkpoint.CategoryId, checkpoint.CopiedCount, changed, alreadyPresent, merged, checkpoint.SkippedCount, reasons, whatBecomesOfThem); + } + } + + /// <summary> + /// Why each harmless skip reason loses nothing, in one sentence or two that the operator reads in the log. Every reason <see cref="MigrationSkipReasonExtensions.IsBenign" /> accepts has one. + /// </summary> + internal static readonly IReadOnlyDictionary<MigrationSkipReason, string> HarmlessSkipExplanations = new Dictionary<MigrationSkipReason, string> + { + [MigrationSkipReason.PastRetention] = "PastRetention rows were past the retention period, so ServiceControl would have deleted them soon anyway.", + [MigrationSkipReason.BlankGroupComment] = "BlankGroupComment rows are blank group comments, and ServiceControl keeps no row for a blank comment.", + [MigrationSkipReason.SourceStorageCheck] = "SourceStorageCheck rows are results of the checks the source's own storage runs about itself, such as the RavenDB persister's checks on its database. Nothing reports those checks once the primary instance runs on SQL, so a copied result would never change." + }; + + static IEnumerable<string> HarmlessReasonsIn(MigrationCheckpoint checkpoint) => + checkpoint.SkipReasons?.Keys.Where(reason => reason.IsBenign()).Order().Select(reason => HarmlessSkipExplanations[reason]) ?? []; + + // A retry reads the whole category again, so it is offered when a retry could copy some of the skipped rows, or when a halt left rows unread. + static string WhatARetryCanDo(MigrationCheckpoint checkpoint) + { + var faults = checkpoint.SkipReasons?.Keys.Where(reason => !reason.IsBenign()).ToArray() ?? []; + var permanent = faults.Where(reason => reason.IsPermanent()).ToArray(); + var cannotFix = permanent.Length == 0 ? "" : $" No retry can fix the {string.Join(" or ", permanent)} ones."; + + // A Halted category never read the rows after its cursor, so abandoning it gives up far more than the skipped rows. + if (faults.Length > 0 && checkpoint.State == MigrationCategoryState.Halted) + { + return $"The copy stopped before the end of the category, so these and every row not yet read stay only in the source database. " + + $"If it stopped because the skips passed the halt threshold, raise {MigrationSettings.HaltThresholdPercentKey} or {MigrationSettings.HaltThresholdMinimumKey}, or fix their cause, then run --migration-retry {checkpoint.CategoryId}, which copies the category again from the start.{cannotFix} " + + $"--migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up every row not yet read, not only these."; + } + + if (faults.Length > 0 && permanent.Length == faults.Length) + { + return $"They stay only in the source database, and no retry can fix them, so --migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up the rest."; + } + + return $"They stay only in the source database until --migration-retry {checkpoint.CategoryId} re-reads them once the cause is fixed.{cannotFix}"; + } + + internal static string StallExplanation(string stalledCategoryId) => + $"Halted: {stalledCategoryId} committed nothing for {ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes, so it was stopped. The limit is not configurable: check that the source and the target are both responding rather than looking for a setting to change."; + + static string RollbackAdvice(Settings settings) => + $"Nothing has opened on {settings.PersistenceType} yet, so pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType} discards the partial copy and returns the instance to {PersistenceFactory.MigrationSourcePersistenceType} with no loss."; + + static string NoWayOutWithoutTheSource(Settings settings) => + $"If the {PersistenceFactory.MigrationSourcePersistenceType} source is already gone, a required category that is Failed or has started copying can be given up with --migration-abandon <category>, with ServiceControl stopped. " + + $"A required category that never started cannot be abandoned: while one is outstanding ServiceControl has never opened on {settings.PersistenceType}, so pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType}, or starting over against an empty {settings.PersistenceType} database, loses nothing it has served."; + + static Step SourceOpens(IMigrationSource source) => new("the migration source opens", source.Open); + + /// <summary> + /// Makes opening the target or the source look like a startup check, so a failure to connect is reported in + /// the same words as a check that refused, and in its place in the order. + /// </summary> + sealed class Step(string name, Func<CancellationToken, Task> run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) => run(cancellationToken); + } + + /// <summary> + /// Logs how far each category has got while the copy runs, and stops the running category when it commits + /// nothing for <see cref="StallLimit" />. Without it a copy that is waiting on a database nobody is watching + /// holds the instance closed for as long as the operator leaves it. + /// </summary> + internal sealed class ClosedWindowProgress : IAsyncDisposable + { + // Not configurable, and not a total timeout: a deadline would kill a copy that is working. + internal static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30); + internal static readonly TimeSpan StallLimit = TimeSpan.FromMinutes(30); + + readonly CancellationTokenSource cancellation; + readonly HashSet<string> attempted; + readonly Task polling; + volatile RunningCategory running; + + public ClosedWindowProgress(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, IReadOnlyCollection<string> attemptedCategoryIds, CancellationToken cancellationToken = default) + { + cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attempted = attemptedCategoryIds.ToHashSet(StringComparer.Ordinal); + polling = Poll(checkpointStore, timeProvider, logger, cancellation.Token); + } + + /// <summary> + /// Points the watchdog at the category that is starting now, in place of the one before it. Only this + /// category is judged from here on, from the later of its last committed batch and <paramref name="runStartedAt" />. + /// </summary> + public void Watch(string categoryId, DateTime runStartedAt, CancellationTokenSource stop) => + running = new RunningCategory(categoryId, runStartedAt, stop); + + async Task Poll(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(PollInterval, timeProvider); + + try + { + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + try + { + // Only this run's categories, because a row an earlier run left in progress is not this run's to report. + var inProgress = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => checkpoint.State == MigrationCategoryState.InProgress && attempted.Contains(checkpoint.CategoryId)) + .ToArray(); + + foreach (var checkpoint in inProgress) + { + logger.LogInformation( + "{CategoryId}: {Copied} of {Total} copied, {Skipped} skipped, cursor {Cursor}", + checkpoint.CategoryId, + checkpoint.CopiedCount, + checkpoint.SourceTotal is { } total ? total.ToString() : "an unknown number of", + checkpoint.SkippedCount, + checkpoint.Cursor ?? "the start"); + } + + var watched = running; + + // A resumed row carries the previous run's stamp, so the window starts at whichever is + // later: that stamp, or the moment this category's run began. + var stalled = watched is not null && !watched.Stop.IsCancellationRequested && inProgress.Any(checkpoint => + checkpoint.CategoryId == watched.CategoryId + && timeProvider.GetUtcNow().UtcDateTime + - (checkpoint.LastProgressAt is { } lastProgress && lastProgress > watched.RunStartedAt ? lastProgress : watched.RunStartedAt) > StallLimit); + + if (stalled) + { + logger.LogError( + "{CategoryId} has committed nothing for {StallLimit}, so it is being stopped and settled Halted. The categories after it still get their go.", + watched.CategoryId, StallLimit); + await watched.Stop.CancelAsync(); + } + } + // The copy finished or the host is stopping: the outer catch ends the poll. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Log, don't throw: an error here would come out of DisposeAsync and hide what the copy itself failed with. + catch (Exception exception) + { + logger.LogError(exception, "The stall watchdog's poll failed, so a stalled copy will not be noticed until a later poll succeeds. The copy itself is unaffected and is still running."); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Either the copy finished and disposal cancelled the poll, or the host is shutting down. + } + } + + public async ValueTask DisposeAsync() + { + await cancellation.CancelAsync(); + + try + { + await polling; + } + finally + { + cancellation.Dispose(); + } + } + + sealed record RunningCategory(string CategoryId, DateTime RunStartedAt, CancellationTokenSource Stop); + } +} diff --git a/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs new file mode 100644 index 0000000000..68f93d6104 --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Runs startup checks in order and stops at the first one that refuses. +/// </summary> +static class MigrationStartupCheckRunner +{ + /// <summary> + /// Runs each check in turn, and wraps whatever a failing one throws in a message naming the check and + /// saying that this run has copied nothing, because every check runs before this run's copy. + /// </summary> + /// <exception cref="Exception">A check failed. The check's own message is kept, and its exception is the inner one.</exception> + public static async Task Run(IReadOnlyList<IMigrationStartupCheck> checks, CancellationToken cancellationToken = default) + { + foreach (var check in checks) + { + try + { + await check.Run(cancellationToken); + } + // A shutdown is not a check failing, and saying it was would send the customer after the wrong thing. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + throw new Exception( + $"Migration check '{check.Name}' failed, so this run has copied nothing. {exception.Message}", exception); + } + } + } +} diff --git a/src/ServiceControl/Migration/OptionalCategoryCopier.cs b/src/ServiceControl/Migration/OptionalCategoryCopier.cs new file mode 100644 index 0000000000..f7e43480b0 --- /dev/null +++ b/src/ServiceControl/Migration/OptionalCategoryCopier.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +sealed class OptionalCategoryCopier(IServiceProvider services, Settings settings, IHostApplicationLifetime lifetime, ILogger<OptionalCategoryCopier> logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + try + { + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var registration = lifetime.ApplicationStarted.Register(() => started.TrySetResult()); + await started.Task.WaitAsync(cancellationToken); + await Copy(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + logger.LogError(exception, "The online migration stopped. ServiceControl remains available. Check migration progress before restarting."); + } + } + + async Task Copy(CancellationToken cancellationToken) + { + var store = services.GetService<IMigrationCheckpointStore>(); + if (store is null) + { + return; + } + + var checkpoints = await store.ReadAll(cancellationToken); + var pending = checkpoints + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind == MigrationCategoryKind.Optional) + .Where(checkpoint => !checkpoint.State.IsFinished() && !checkpoint.State.IsFailed()) + .Select(checkpoint => checkpoint.CategoryId) + .ToHashSet(StringComparer.Ordinal); + if (pending.Count == 0) + { + return; + } + + await new MigrationIsReleasedCheck(services.GetService<AllowUnreleasedMigration>()).Run(cancellationToken); + var target = services.GetRequiredService<IMigrationTarget>(); + await using var source = PersistenceFactory.CreateMigrationSource(settings); + var copyable = MigrationStartup.CopyableCategoryIds(source.SupportedCategoryIds, target.SupportedCategoryIds); + var options = MigrationEngineOptions.FromSettings(Settings.SettingsRootNamespace, settings.EventsRetentionPeriod, settings.ErrorRetentionPeriod); + var engine = new MigrationEngine(source, target, store, services.GetRequiredService<TimeProvider>(), options, + services.GetRequiredService<ILogger<MigrationEngine>>()); + var categories = engine.SelectCategories(MigrationCategoryKind.Optional) + .Where(category => pending.Contains(category.Id) && copyable.Contains(category.Id)) + .ToArray(); + if (categories.Length == 0) + { + return; + } + + await target.Open(cancellationToken); + try + { + await source.Open(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + await MigrationStartup.RecordSourceOutage(store, categories, exception, cancellationToken); + logger.LogError(exception, "The online migration cannot open its source. ServiceControl remains available. Restart ServiceControl to try again, or abandon the outstanding optional categories."); + return; + } + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + await engine.RunCategories(categories, cancellationToken); + } +} diff --git a/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs new file mode 100644 index 0000000000..62c3c4053e --- /dev/null +++ b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// <summary> +/// Calls <see cref="IMigrationTargetReadiness.RecordHostOpened"/> when a host opens +/// on a database a migration has already written to. +/// </summary> +sealed class RecordHostOpenedOnTarget(IServiceProvider services) : IHostedLifecycleService +{ + // StartedAsync, not StartAsync: the web server binds its port during StartAsync, and a start that dies there + // served nothing. RavenDB registers no checkpoint store and needs no migration marker. + public async Task StartedAsync(CancellationToken cancellationToken = default) + { + var checkpointStore = services.GetService<IMigrationCheckpointStore>(); + + // Only a database a migration has written to gets the stamp, so the marker answers whether a host + // has opened since the copy began rather than whether one ever ran on this database. + if (checkpointStore is not null && (await checkpointStore.ReadAll(cancellationToken)).Count > 0) + { + await services.GetRequiredService<IMigrationTargetReadiness>().RecordHostOpened(cancellationToken); + } + } + + public Task StartingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 892920cf18..5fd045058d 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -2,13 +2,23 @@ namespace ServiceControl.Persistence { using System; using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence.DataMigration; static class PersistenceFactory { + /// <summary> + /// The manifest names of the two persisters built on EF Core. Anything that is true of both of them and + /// of neither RavenDB nor a future persister is decided by this list. + /// </summary> + public static readonly string[] SqlPersistenceNames = ["SQLServer", "PostgreSQL"]; + public static IPersistence Create(Settings settings, bool maintenanceMode = false) { - var persistenceConfiguration = CreatePersistenceConfiguration(settings); + var persistenceConfiguration = CreatePersistenceConfiguration(settings.PersistenceType, settings); if (maintenanceMode && !persistenceConfiguration.SupportsMaintenanceMode) { @@ -24,11 +34,51 @@ public static IPersistence Create(Settings settings, bool maintenanceMode = fals return persistence; } - static IPersistenceConfiguration CreatePersistenceConfiguration(Settings settings) + /// <summary> + /// The persistence a migration copies from. RavenDB is the only source supported today. + /// </summary> + public const string MigrationSourcePersistenceType = "RavenDB"; + + public static IMigrationSource CreateMigrationSource(Settings settings) + { + var sourceFactory = (IMigrationSourceFactory)CreatePersistenceConfiguration(MigrationSourcePersistenceType, settings); + + return sourceFactory.CreateSource(Settings.SettingsRootNamespace); + } + + public static async Task<IMigrationSource> OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) + { + var source = CreateMigrationSource(settings); + var opened = false; + + try + { + await source.Open(cancellationToken); + opened = true; + } + finally + { + if (!opened) + { + await source.DisposeAsync(); + } + } + + return source; + } + + static IPersistenceConfiguration CreatePersistenceConfiguration(string persistenceType, Settings settings) { + var persistenceManifest = PersistenceManifestLibrary.Find(persistenceType) + ?? throw new Exception($"There is no persistence named '{persistenceType}'. Available: {string.Join(", ", PersistenceManifestLibrary.PersistenceManifests.Where(m => m.IsSupported).Select(m => m.Name))}."); + + if (persistenceManifest.TypeName is null) + { + throw new Exception($"The '{persistenceManifest.DisplayName}' persistence no longer ships an assembly and cannot be loaded."); + } + try { - var persistenceManifest = PersistenceManifestLibrary.Find(settings.PersistenceType); var assemblyPath = Path.Combine(persistenceManifest.Location, $"{persistenceManifest.AssemblyName}.dll"); var loadContext = settings.AssemblyLoadContextResolver(assemblyPath); var customizationType = Type.GetType(persistenceManifest.TypeName, loadContext.LoadFromAssemblyName, null, true); @@ -37,7 +87,7 @@ static IPersistenceConfiguration CreatePersistenceConfiguration(Settings setting } catch (Exception e) { - throw new Exception($"Could not load persistence customization type {settings.PersistenceType}.", e); + throw new Exception($"Could not load persistence customization type {persistenceType}.", e); } } } diff --git a/src/ServiceControl/Program.cs b/src/ServiceControl/Program.cs index 7234479cd3..96fb2359df 100644 --- a/src/ServiceControl/Program.cs +++ b/src/ServiceControl/Program.cs @@ -39,7 +39,7 @@ await new CommandRunner(arguments.Command).Execute(arguments, settings); - return 0; + return Environment.ExitCode; } catch (Exception ex) { @@ -52,7 +52,13 @@ LoggingConfigurator.ConfigureNLog("bootstrap.txt", "./", NLog.LogLevel.Fatal); NLog.LogManager.GetCurrentClassLogger().Fatal(ex, "Unrecoverable error"); } - throw; + + // The message goes to the console on its own, because the log above already holds the whole exception. + // Rethrowing instead would print the stack trace a second time and fire the unhandled-exception handler + // for a third, which buries a configuration mistake in what reads like a crash. + await Console.Error.WriteLineAsync(ex.Message); + + return 1; } finally { diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj index 7afd8f88cf..d9b9958c43 100644 --- a/src/ServiceControl/ServiceControl.csproj +++ b/src/ServiceControl/ServiceControl.csproj @@ -68,6 +68,7 @@ <InternalsVisibleTo Include="ServiceControl.AcceptanceTests.PostgreSql" /> <InternalsVisibleTo Include="ServiceControl.AcceptanceTests.RavenDB" /> <InternalsVisibleTo Include="ServiceControl.AcceptanceTests.SqlServer" /> + <InternalsVisibleTo Include="ServiceControl.Migration.Tests" /> <InternalsVisibleTo Include="ServiceControl.MultiInstance.AcceptanceTests" /> <InternalsVisibleTo Include="ServiceControl.Persistence.Tests.PostgreSql" /> <InternalsVisibleTo Include="ServiceControl.Persistence.Tests.RavenDB" />